Merge #878

878: Bump meilisearch v0.13.0 r=MarinPostma a=MarinPostma



Co-authored-by: mpostma <postma.marin@protonmail.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/tracking-issue.md

@@ -0,0 +1,40 @@
+---
+name: Tracking issue
+about: Template for a tracking issue
+title: ''
+labels: tracking-issue
+assignees: ''
+
+---
+
+# Summary
+
+One paragraph to explain the feature.
+
+# Motivations
+
+Why are we doing this? What use cases does it support? What is the expected outcome?
+
+# Explanation
+
+Explain the proposal like it was the final documentation of this proposal.
+
+- What is changing for end-users.
+- How it works.
+- What is breaking?
+- Examples.
+
+# Implementation
+
+Explain the technical specificities that will need to be known or done in order to implement this proposal.
+
+## Steps
+
+Describe each step to create the feature with it's associated issue/PR.
+
+# Related
+
+- [ ] Validated by the team (@people needed)
+- [ ] Test added
+- [ ] [Documentation](https://github.com/meilisearch/documentation/issues/#xxx) //Change xxx or remove the line
+- [ ] [SDK/Integrations](https://github.com/meilisearch/integration-guides/issues/#xxx) //Change xxx or remove the line

.github/is-latest-release.sh

@@ -0,0 +1,132 @@
+#!/bin/sh
+
+# Checks if the current tag should be the latest (in terms of semver and not of release date).
+# Ex: previous tag -> v0.10.1
+#     new tag -> v0.8.12
+#     The new tag should not be the latest
+#     So it returns "false", the CI should not run for the release v0.8.2
+
+# Used in GHA in publish-docker-latest.yml
+# Returns "true" or "false" (as a string) to be used in the `if` in GHA
+
+# GLOBAL
+GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number]
+
+# FUNCTIONS
+
+# semverParseInto and semverLT from https://github.com/cloudflare/semver_bash/blob/master/semver.sh
+
+# usage: semverParseInto version major minor patch special
+# version: the string version
+# major, minor, patch, special: will be assigned by the function
+semverParseInto() {
+    local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)'
+    #MAJOR
+    eval $2=`echo $1 | sed -e "s#$RE#\1#"`
+    #MINOR
+    eval $3=`echo $1 | sed -e "s#$RE#\2#"`
+    #MINOR
+    eval $4=`echo $1 | sed -e "s#$RE#\3#"`
+    #SPECIAL
+    eval $5=`echo $1 | sed -e "s#$RE#\4#"`
+}
+
+# usage: semverLT version1 version2
+semverLT() {
+    local MAJOR_A=0
+    local MINOR_A=0
+    local PATCH_A=0
+    local SPECIAL_A=0
+
+    local MAJOR_B=0
+    local MINOR_B=0
+    local PATCH_B=0
+    local SPECIAL_B=0
+
+    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
+    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
+
+    if [ $MAJOR_A -lt $MAJOR_B ]; then
+        return 0
+    fi
+    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -lt $MINOR_B ]; then
+        return 0
+    fi
+    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then
+        return 0
+    fi
+    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+        return 1
+    fi
+    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  != "_" ] ; then
+        return 1
+    fi
+    if [ "_$SPECIAL_A"  != "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+        return 0
+    fi
+    if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then
+        return 0
+    fi
+
+    return 1
+}
+
+# Returns the tag of the latest stable release (in terms of semver and not of release date)
+get_latest() {
+    temp_file='temp_file' # temp_file needed because the grep would start before the download is over
+    curl -s 'https://api.github.com/repos/meilisearch/MeiliSearch/releases' > "$temp_file"
+    releases=$(cat "$temp_file" | \
+        grep -E "tag_name|draft|prerelease" \
+        | tr -d ',"' | cut -d ':' -f2 | tr -d ' ')
+        # Returns a list of [tag_name draft_boolean prerelease_boolean ...]
+        # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false...
+
+    i=0
+    latest=""
+    current_tag=""
+    for release_info in $releases; do
+        if [ $i -eq 0 ]; then # Cheking tag_name
+            if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release
+                current_tag=$release_info
+            else
+                current_tag=""
+            fi
+            i=1
+        elif [ $i -eq 1 ]; then # Checking draft boolean
+            if [ "$release_info" = "true" ]; then
+                current_tag=""
+            fi
+            i=2
+        elif [ $i -eq 2 ]; then # Checking prerelease boolean
+            if [ "$release_info" = "true" ]; then
+                current_tag=""
+            fi
+            i=0
+            if [ "$current_tag" != "" ]; then # If the current_tag is valid
+                if [ "$latest" = "" ]; then # If there is no latest yet
+                    latest="$current_tag"
+                else
+                    semverLT $current_tag $latest # Comparing latest and the current tag
+                    if [ $? -eq 1 ]; then
+                        latest="$current_tag"
+                    fi
+                fi
+            fi
+        fi
+    done
+
+    rm -f "$temp_file"
+    echo $latest
+}
+
+# MAIN
+current_tag="$(echo $GITHUB_REF | tr -d 'refs/tags/')"
+latest="$(get_latest)"
+
+if [ "$current_tag" != "$latest" ]; then
+    # The current release tag is not the latest
+    echo "false"
+else
+    # The current release tag is the latest
+    echo "true"
+fi

.github/workflows/README.md

@@ -1,4 +1,4 @@
-# GitHub actions workflow for MeiliDB
+# GitHub Actions Workflow for MeiliSearch
 
 > **Note:**
 
@@ -6,12 +6,14 @@
 
 ## Workflow
 
-- On each pull request, we are triggering `cargo test`.
-- On each tag, we are building:
-    - the tagged docker image
+- On each pull request, we trigger `cargo test`.
+- On each tag, we build:
+    - the tagged Docker image and publish it to Docker Hub
     - the binaries for MacOS, Ubuntu, and Windows
-    - the debian package
-- On each stable release, we are build the latest docker image.
+    - the Debian package
+- On each stable release (`v*.*.*` tag):
+    - we build the `latest` Docker image and publish it to Docker Hub
+    - we publish the binary to Hombrew and Gemfury
 
 ## Problems
 

.github/workflows/publish-binaries.yml

@@ -1,9 +1,8 @@
-name: Publish binaries to GitHub release
-
 on:
-  push:
-    tags:
-      - '*'
+  release:
+    types: [published]
+
+name: Publish binaries to release
 
 jobs:
   publish:
@@ -20,8 +19,8 @@ jobs:
             artifact_name: meilisearch
             asset_name: meilisearch-macos-amd64
           - os: windows-latest
-            artifact_name: meilisearch
-            asset_name: meilisearch-windows-amd64
+            artifact_name: meilisearch.exe
+            asset_name: meilisearch-windows-amd64.exe
 
     steps:
     - uses: hecrj/setup-rust-action@master

.github/workflows/publish-deb-brew-pkg.yml

@@ -1,9 +1,8 @@
-name: Publish deb pkg to GitHub release & apt repository & Homebrew
+name: Publish deb pkg to GitHub release & APT repository & Homebrew
 
 on:
-  push:
-    tags:
-      - 'v[0-9]+.[0-9]+.[0-9]+'
+  release:
+    types: [released]
 
 jobs:
   debian:
@@ -32,7 +31,8 @@ jobs:
     name: Bump Homebrew formula
     runs-on: ubuntu-latest
     steps:
-      - uses: mislav/bump-homebrew-formula-action@v1
+      - name: Create PR to Homebrew
+        uses: mislav/bump-homebrew-formula-action@v1
         with:
           formula-name: meilisearch
         env:

.github/workflows/publish-docker-latest.yml

@@ -1,8 +1,7 @@
 ---
 on:
-  push:
-    tags:
-      - 'v[0-9]+.[0-9]+.[0-9]+'
+  release:
+    types: [released]
 
 name: Publish latest image to Docker Hub
 
@@ -10,8 +9,12 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+      - name: Check if current release is latest
+        run: echo "##[set-output name=is_latest;]$(sh .github/is-latest-release.sh)"
+        id: release
       - name: Publish to Registry
+        if: steps.release.outputs.is_latest == 'true'
         uses: elgohr/Publish-Docker-Github-Action@master
         with:
           name: getmeili/meilisearch

.github/workflows/test.yml

@@ -1,5 +1,12 @@
 ---
-on: [pull_request]
+on:
+  push:
+    branches:
+      - release-v*
+      - trying
+      - staging
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+' # this only concerns tags on stable
 
 name: Test binaries with cargo test
 
@@ -10,7 +17,6 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-
     steps:
     - uses: actions/checkout@v1
     - uses: actions-rs/toolchain@v1
@@ -18,8 +24,70 @@ jobs:
         profile: minimal
         toolchain: stable
         override: true
+        components: clippy
     - name: Run cargo test
       uses: actions-rs/cargo@v1
       with:
         command: test
         args: --locked --release
+    - name: Run cargo clippy
+      uses: actions-rs/cargo@v1
+      with:
+        command: clippy
+
+  build-image:
+    name: Test the build of Docker image
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - run: docker build . --file Dockerfile -t meilisearch
+      name: Docker build
+
+  ## A push occurred on a release branch, a prerelease is created and assets are generated
+  prerelease:
+    name: create prerelease
+    needs: [check, build-image]
+    if: ${{ contains(github.ref, 'release-') && github.event_name == 'push' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Get version number
+        id: version-number
+        run: echo "##[set-output name=number;]$(echo ${{ github.ref }} | sed 's/.*\(v.*\)/\1/')"
+      - name: Get commit count
+        id: commit-count
+        run: echo "##[set-output name=count;]$(git rev-list remotes/origin/master..remotes/origin/release-${{ steps.version-number.outputs.number }} --count)"
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.PUBLISH_TOKEN }} # Personal Access Token
+        with:
+          tag_name: ${{ steps.version-number.outputs.number }}rc${{ steps.commit-count.outputs.count }}
+          release_name: Pre-release ${{ steps.version-number.outputs.number }}-rc${{ steps.commit-count.outputs.count }}
+          prerelease: true
+
+  ## If a tag is pushed, a release is created for this tag, and assets will be generated
+  release:
+    name: create release
+    needs: [check, build-image]
+    if:  ${{ contains(github.ref, 'tags/v') }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Get version number
+        id: version-number
+        run: echo "##[set-output name=number;]$(echo ${{ github.ref }} | sed 's/.*\(v.*\)/\1/')" 
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.PUBLISH_TOKEN }} # PAT
+        with:
+          tag_name: ${{ steps.version-number.outputs.number }}
+          release_name: Meilisearch ${{ steps.version-number.outputs.number }}
+          prerelease: false

CHANGELOG.md

@@ -1,4 +1,59 @@
-## v0.10
+## v0.13.0
+
+  - placeholder search (#771)
+  - Add database version mismatch check (#794)
+  - Displayed and searchable attributes wildcard (#846)
+  - Remove sys-info route (#810)
+  - Fix facet distribution case (#797)
+  - Check database version mismatch (#794)
+  - Fix unique docid bug (#841)
+  - Error codes in updates (#792)
+  - Sentry disable argument (#813)
+  - Log analytics if enabled (#825)
+
+## v0.12.0
+
+  - Fix long documents not being indexed completely bug (#816)
+  - Fix distinct attribute returning id instead of name (#800)
+  - error code rename (#805)
+
+## v0.11.1
+
+  - Fix facet cache on document update (#789) 
+  - Improvements on settings consistency (#778)
+
+## v0.11.0
+
+  - Change the HTTP framework, moving from tide to actix-web (#601)
+  - Bump sentry version to 0.18.1 (#690)
+  - Enable max payload size override (#684)
+  - Disable sentry in debug (#681)
+  - Better terminal greeting (#680)
+  - Fix highlight misalignment (#679)
+  - Add support for facet count (#676)
+  - Add support for faceted search (#631)
+  - Add support for configuring the lmdb map size (#646, #647)
+  - Add exposed port for Dockerfile (#654)
+  - Add sentry probe (#664)
+  - Fix url trailing slash and double slash issues (#659)
+  - Fix accept all Content-Type by default (#653)
+  - Return the error message from Serde when a deserialization error is encountered (#661)
+  - Fix NormalizePath middleware to make the dashboard accessible (#695)
+  - Update sentry features to remove openssl (#702)
+  - Add SSL support (#669)
+  - Rename fieldsFrequency into fieldsDistribution in stats (#719)
+  - Add support for error code reporting (#703)
+  - Allow the dashboard to query private servers (#732)
+  - Add telemetry (#720)
+  - Add post route for search (#735)
+
+## v0.10.1
+
+  - Add support for floating points in filters (#640)
+  - Add '@' character as tokenizer separator (#607)
+  - Add support for filtering on arrays of strings (#611)
+
+## v0.10.0
 
   - Refined filtering (#592)
   - Add the number of hits in search result (#541)

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at bonjour@meilisearch.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

Dockerfile

@@ -18,10 +18,12 @@ RUN     $HOME/.cargo/bin/cargo build --release
 # Run
 FROM    alpine:3.10
 
-RUN     apk update --quiet
-RUN     apk add libgcc
+RUN     apk add -q --no-cache libgcc tini
 
 COPY    --from=compiler /meilisearch/target/release/meilisearch .
 
 ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
+EXPOSE  7700/tcp
+
+ENTRYPOINT ["tini", "--"]
 CMD     ./meilisearch

README.md

@@ -2,15 +2,16 @@
   <img src="assets/logo.svg" alt="MeiliSearch" width="200" height="200" />
 </p>
 
+
 <h1 align="center">MeiliSearch</h1>
 
 <h4 align="center">
-  <a href="https://www.meilisearch.com">Website</a> | 
-  <a href="https://blog.meilisearch.com">Blog</a> | 
-  <a href="https://fr.linkedin.com/company/meilisearch">LinkedIn</a> | 
-  <a href="https://twitter.com/meilisearch">Twitter</a> | 
-  <a href="https://docs.meilisearch.com">Documentation</a> | 
-  <a href="https://docs.meilisearch.com/resources/faq.html">FAQ</a>
+  <a href="https://www.meilisearch.com">Website</a> |
+  <a href="https://blog.meilisearch.com">Blog</a> |
+  <a href="https://fr.linkedin.com/company/meilisearch">LinkedIn</a> |
+  <a href="https://twitter.com/meilisearch">Twitter</a> |
+  <a href="https://docs.meilisearch.com">Documentation</a> |
+  <a href="https://docs.meilisearch.com/faq/">FAQ</a>
 </h4>
 
 <p align="center">
@@ -18,6 +19,8 @@
   <a href="https://deps.rs/repo/github/meilisearch/MeiliSearch"><img src="https://deps.rs/repo/github/meilisearch/MeiliSearch/status.svg" alt="Dependency status"></a>
   <a href="https://github.com/meilisearch/MeiliSearch/blob/master/LICENSE"><img src="https://img.shields.io/badge/license-MIT-informational" alt="License"></a>
   <a href="https://slack.meilisearch.com"><img src="https://img.shields.io/badge/slack-MeiliSearch-blue.svg?logo=slack" alt="Slack"></a>
+  <a href="https://github.com/meilisearch/MeiliSearch/discussions" alt="Discussions"><img src="https://img.shields.io/badge/github-discussions-red" /></a>
+  <a href="https://app.bors.tech/repositories/26457"><img src="https://bors.tech/images/badge_small.svg" alt="Bors enabled"></a>
 </p>
 
 <p align="center">⚡ Lightning Fast, Ultra Relevant, and Typo-Tolerant Search Engine 🔍</p>
@@ -29,23 +32,28 @@ For more information about features go to [our documentation](https://docs.meili
   <a href="https://crates.meilisearch.com"><img src="assets/crates-io-demo.gif" alt="crates.io demo gif" /></a>
 </p>
 
-> Meili helps the Rust community find crates on [crates.meilisearch.com](https://crates.meilisearch.com)
+> MeiliSearch helps the Rust community find crates on [crates.meilisearch.com](https://crates.meilisearch.com)
 
 ## Features
 * Search as-you-type experience (answers < 50 milliseconds)
 * Full-text search
 * Typo tolerant (understands typos and miss-spelling)
-* Supports Kanji
+* Supports Kanji characters
 * Supports Synonym
 * Easy to install, deploy, and maintain
 * Whole documents are returned
 * Highly customizable
 * RESTful API
+* Faceted search and filtering
 
 ## Get started
 
 ### Deploy the Server
 
+#### Run it using Digital Ocean
+
+[![DigitalOcean Marketplace](assets/do-btn-blue.svg)](https://marketplace.digitalocean.com/apps/meilisearch?action=deploy&refcode=7c67bd97e101)
+
 #### Run it using Docker
 
 ```bash
@@ -86,6 +94,8 @@ cd MeiliSearch
 In the cloned repository, compile MeiliSearch.
 
 ```bash
+rustup override set stable
+rustup update stable
 cargo run --release
 ```
 
@@ -116,7 +126,7 @@ curl -i -X POST 'http://127.0.0.1:7700/indexes/movies/documents' \
 
 #### In command line
 
-The search engine is now aware of your documents and can serve those via an HTTP server. 
+The search engine is now aware of your documents and can serve those via an HTTP server.
 
 The [`jq` command-line tool](https://stedolan.github.io/jq/) can greatly help you read the server responses.
 
@@ -163,38 +173,6 @@ You can access the web interface in your web browser at the root of the server.
 
 Now that your MeiliSearch server is up and running, you can learn more about how to tune your search engine in [the documentation](https://docs.meilisearch.com).
 
-### Technical features
-
-- Provides [6 default ranking criteria](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/criterion/mod.rs#L106-L111) used to [bucket sort](https://en.wikipedia.org/wiki/Bucket_sort) documents
-- Accepts [custom criteria](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/criterion/mod.rs#L20-L29) and can apply them in any custom order
-- Supports [ranged queries](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/query_builder.rs#L342), useful for paginating results
-- Can [distinct](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/query_builder.rs#L324-L329) and [filter](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/query_builder.rs#L313-L318) returned documents based on context defined rules
-- Searches for [concatenated](https://github.com/meilisearch/MeiliSearch/pull/164) and [splitted query words](https://github.com/meilisearch/MeiliSearch/pull/232) to improve the search quality.
-- Can store complete documents or only [user schema specified fields](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/datasets/movies/schema.toml)
-- The [default tokenizer](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-tokenizer/src/lib.rs) can index Latin and Kanji based languages
-- Returns [the matching text areas](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-types/src/lib.rs#L49-L65), useful to highlight matched words in results
-- Accepts query time search config like the [searchable attributes](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/query_builder.rs#L331-L336)
-- Supports [runtime incremental indexing](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/store/mod.rs#L143-L212)
-
-## Performance
-
-When processing a dataset composed of 5M books, each with their own titles and authors, MeiliSearch is able to carry out more than 553 req/sec with an average response time of 21 ms on an Intel i7-7700 (8) @ 4.2GHz.
-
-Requests are made using [wrk](https://github.com/wg/wrk) and scripted to simulate real users' queries.
-
-```
-Running 10s test @ http://1.2.3.4:7700
-  2 threads and 10 connections
-  Thread Stats   Avg      Stdev     Max   +/- Stdev
-    Latency    21.45ms   15.64ms 214.10ms   85.95%
-    Req/Sec   256.48     37.66   330.00     69.50%
-  5132 requests in 10.05s, 2.31MB read
-Requests/sec:    510.46
-Transfer/sec:    234.77KB
-```
-
-We also indexed a dataset containing about _12 millions_ cities names in _24 minutes_ on a _8 cores_, _64 GB of RAM_, and a _300 GB NMVe_ SSD machine.<br/>
-The size of the resulting database reached _16 GB_ and search results were presented between _30 ms_ and _4 seconds_ for short prefix queries.
 
 ## Contributing
 
@@ -202,8 +180,9 @@ Hey! We're glad you're thinking about contributing to MeiliSearch! If you think
 
 ### Analytic Events
 
-Once a day, events are being sent to our Amplitude instance so we can know how many people are using MeiliSearch.<br/>
-Only information about the platform on which the server runs is stored. No other information is being sent.<br/>
+Every hour, events are being sent to our Amplitude instance so we can know how many people are using MeiliSearch.<br/>
+To see what information we're retrieving, please see the complete list [on the dedicated issue](https://github.com/meilisearch/MeiliSearch/issues/720).<br/>
+We also use Sentry to make us crash and error reports. If you want to know more about what Sentry collects, please visit their [privacy policy website](https://sentry.io/privacy/).<br/>
 If this doesn't suit you, you can disable these analytics by using the `MEILI_NO_ANALYTICS` env variable.
 
 ## Contact
@@ -211,6 +190,7 @@ If this doesn't suit you, you can disable these analytics by using the `MEILI_NO
 Feel free to contact us about any questions you may have:
 * At [bonjour@meilisearch.com](mailto:bonjour@meilisearch.com): English or French is welcome! 🇬🇧 🇫🇷
 * Via the chat box available on every page of [our documentation](https://docs.meilisearch.com/) and on [our landing page](https://www.meilisearch.com/).
+* 🆕 Join our [GitHub Discussions forum](https://github.com/meilisearch/MeiliSearch/discussions) (BETA hype!)
 * Join our [Slack community](https://slack.meilisearch.com/).
 * By opening an issue.
 

assets/do-btn-blue.svg

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="200px" height="42px" viewBox="0 0 200 42" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 52.5 (67469) - http://www.bohemiancoding.com/sketch -->
+    <title>do-btn-blue</title>
+    <desc>Created with Sketch.</desc>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="Partner-welcome-kit-Copy-3" transform="translate(-651.000000, -762.000000)">
+            <g id="do-btn-blue" transform="translate(651.000000, 763.000000)">
+                <rect id="Rectangle-Copy" fill="#0069FF" x="0" y="0" width="200" height="40" rx="6"></rect>
+                <path d="M45,0 L45,40" id="Line-2" stroke="#FFFFFF" stroke-linecap="square"></path>
+                <g id="DO_Logo_horizontal_blue-Copy" transform="translate(13.000000, 11.000000)" fill="#FFFFFF">
+                    <path d="M10.0098493,20 L10.0098493,16.1262429 C14.12457,16.1262429 17.2897398,12.0548452 15.7269372,7.74627862 C15.1334679,6.14538921 13.8674,4.86072487 12.2650328,4.28756693 C7.952489,2.72620566 3.87733294,5.88845634 3.87733294,9.99938223 C3.87733294,9.99938223 3.87733294,9.99938223 3.87733294,9.99938223 L0,9.99938223 C0,3.45747613 6.3303395,-1.64165309 13.1948014,0.492866119 C16.2017127,1.42177726 18.57559,3.81322933 19.5053586,6.79760341 C21.6418482,13.6754986 16.5577943,20 10.0098493,20 Z" id="XMLID_49_"></path>
+                    <polygon id="XMLID_47_" points="9.56521739 15.6521739 6.08695652 15.6521739 6.08695652 12.173913 6.08695652 12.173913 9.56521739 12.173913 9.56521739 12.173913"></polygon>
+                    <polygon id="XMLID_46_" points="6.08695652 19.1304348 3.47826087 19.1304348 3.47826087 19.1304348 3.47826087 16.5217391 6.08695652 16.5217391"></polygon>
+                    <polygon id="XMLID_45_" points="3.47826087 16.5217391 0.869565217 16.5217391 0.869565217 16.5217391 0.869565217 13.9130435 0.869565217 13.9130435 3.47826087 13.9130435 3.47826087 13.9130435"></polygon>
+                </g>
+                <text id="Create-a-Droplet-Copy" font-family="Sailec-Medium, Sailec" font-size="16" font-weight="400" fill="#FFFFFF">
+                    <tspan x="58" y="26">Create a Droplet</tspan>
+                </text>
+            </g>
+        </g>
+    </g>
+</svg>

bors.toml

@@ -0,0 +1,3 @@
+status = ["Test on macos-latest", "Test on ubuntu-latest"]
+# 4 hours timeout
+timeout-sec = 14400

bump.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/bash
+
+NEW_VERSION=$1
+
+if [ -z "$NEW_VERSION" ]
+then
+	echo "error: a version number must be provided"
+	exit 1
+fi
+
+# find current version
+CURRENT_VERSION=$(cat **/*.toml | grep meilisearch | grep version | sed 's/.*\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/' | sed "1q;d")
+
+# bump all version in .toml
+echo "bumping from version $CURRENT_VERSION to version $NEW_VERSION"
+while true
+do
+	read -r -p "Continue (y/n)?" choice
+	case "$choice" in
+		y|Y ) break;;
+		n|N ) echo "aborting bump" && exit 0;;
+		* ) echo "invalid choice";;
+	esac
+done
+# update all crate version
+sed -i "s/version = \"$CURRENT_VERSION\"/version = \"$NEW_VERSION\"/" **/*.toml
+
+printf "running cargo check: "
+
+CARGO_CHECK=$(cargo check 2>&1)
+
+if [ $? != "0" ]
+then
+	printf "\033[31;1m FAIL \033[0m\n"
+	printf "$CARGO_CHECK"
+	exit 1
+fi
+printf "\033[32;1m OK \033[0m\n"

download-latest.sh

@@ -6,8 +6,10 @@ GREEN='\033[32m'
 DEFAULT='\033[0m'
 
 # GLOBALS
-GREP_SEMVER_REGEXP='\"v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\"' # i.e. "v[number].[number].[number]"
 BINARY_NAME='meilisearch'
+GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number]
+
+# FUNCTIONS
 
 # semverParseInto and semverLT from https://github.com/cloudflare/semver_bash/blob/master/semver.sh
 
@@ -66,6 +68,88 @@ semverLT() {
     return 1
 }
 
+# Returns the tag of the latest stable release (in terms of semver and not of release date)
+get_latest() {
+    temp_file='temp_file' # temp_file needed because the grep would start before the download is over
+    curl -s 'https://api.github.com/repos/meilisearch/MeiliSearch/releases' > "$temp_file"
+    releases=$(cat "$temp_file" | \
+        grep -E "tag_name|draft|prerelease" \
+        | tr -d ',"' | cut -d ':' -f2 | tr -d ' ')
+        # Returns a list of [tag_name draft_boolean prerelease_boolean ...]
+        # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false...
+
+    i=0
+    latest=""
+    current_tag=""
+    for release_info in $releases; do
+        if [ $i -eq 0 ]; then # Cheking tag_name
+            if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release
+                current_tag=$release_info
+            else
+                current_tag=""
+            fi
+            i=1
+        elif [ $i -eq 1 ]; then # Checking draft boolean
+            if [ "$release_info" = "true" ]; then
+                current_tag=""
+            fi
+            i=2
+        elif [ $i -eq 2 ]; then # Checking prerelease boolean
+            if [ "$release_info" = "true" ]; then
+                current_tag=""
+            fi
+            i=0
+            if [ "$current_tag" != "" ]; then # If the current_tag is valid
+                if [ "$latest" = "" ]; then # If there is no latest yet
+                    latest="$current_tag"
+                else
+                    semverLT $current_tag $latest # Comparing latest and the current tag
+                    if [ $? -eq 1 ]; then
+                        latest="$current_tag"
+                    fi
+                fi
+            fi
+        fi
+    done
+
+    rm -f "$temp_file"
+    echo $latest
+}
+
+# Gets the OS by setting the $os variable
+# Returns 0 in case of success, 1 otherwise.
+get_os() {
+    os_name=$(uname -s)
+    case "$os_name" in
+    'Darwin')
+        os='macos'
+        ;;
+    'Linux')
+        os='linux'
+        ;;
+    *)
+        return 1
+    esac
+    return 0
+}
+
+# Gets the architecture by setting the $archi variable
+# Returns 0 in case of success, 1 otherwise.
+get_archi() {
+    architecture=$(uname -m)
+    case "$architecture" in
+    'x86_64' | 'amd64')
+        archi='amd64'
+        ;;
+    'aarch64')
+        archi='armv8'
+        ;;
+    *)
+        return 1
+    esac
+    return 0
+}
+
 success_usage() {
     printf "$GREEN%s\n$DEFAULT" "MeiliSearch binary successfully downloaded as '$BINARY_NAME' file."
     echo ''
@@ -76,53 +160,27 @@ success_usage() {
 }
 
 failure_usage() {
-    printf "$RED%s\n$DEFAULT" 'ERROR: MeiliSearch binary is not available for your OS distribution yet.'
+    printf "$RED%s\n$DEFAULT" 'ERROR: MeiliSearch binary is not available for your OS distribution or your architecture yet.'
     echo ''
     echo 'However, you can easily compile the binary from the source files.'
-    echo 'Follow the steps on the docs: https://docs.meilisearch.com/advanced_guides/binary.html#how-to-compile-meilisearch'
+    echo 'Follow the steps at the page ("Source" tab): https://docs.meilisearch.com/guides/advanced_guides/installation.html'
 }
 
-# OS DETECTION
-echo 'Detecting OS distribution...'
-os_name=$(uname -s)
-if [ "$os_name" != "Darwin" ]; then
-    os_name=$(cat /etc/os-release | grep '^ID=' | tr -d '"' | cut -d '=' -f 2)
-fi
-echo "OS distribution detected: $os_name"
-case "$os_name" in
-'Darwin')
-    os='macos'
-    ;;
-'ubuntu' | 'debian')
-    os='linux'
-    ;;
-*)
+# MAIN
+latest="$(get_latest)"
+get_os
+if [ "$?" -eq 1 ]; then
     failure_usage
     exit 1
-esac
-
-# GET LATEST VERSION
-tags=$(curl -s 'https://api.github.com/repos/meilisearch/MeiliSearch/tags' \
-    | grep "$GREP_SEMVER_REGEXP" \
-    | grep 'name' \
-    | tr -d '"' | tr -d ',' | cut -d 'v' -f 2)
-
-latest=""
-for tag in $tags; do
-    if [ "$latest" = "" ]; then
-        latest="$tag"
-    else
-        semverLT $tag $latest
-        if [ $? -eq 1 ]; then
-            latest="$tag"
-        fi
-    fi
-done
-
-# DOWNLOAD THE LATEST
-echo "Downloading MeiliSearch binary v$latest for $os..."
-release_file="meilisearch-$os-amd64"
-link="https://github.com/meilisearch/MeiliSearch/releases/download/v$latest/$release_file"
+fi
+get_archi
+if [ "$?" -eq 1 ]; then
+    failure_usage
+    exit 1
+fi
+echo "Downloading MeiliSearch binary $latest for $os, architecture $archi..."
+release_file="meilisearch-$os-$archi"
+link="https://github.com/meilisearch/MeiliSearch/releases/download/$latest/$release_file"
 curl -OL "$link"
 mv "$release_file" "$BINARY_NAME"
 chmod 744 "$BINARY_NAME"

meilisearch-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "meilisearch-core"
-version = "0.10.0"
+version = "0.13.0"
 license = "MIT"
 authors = ["Kerollmops <clement@meilisearch.com>"]
 edition = "2018"
@@ -11,20 +11,23 @@ bincode = "1.2.1"
 byteorder = "1.3.4"
 chrono = { version = "0.4.11", features = ["serde"] }
 compact_arena = "0.4.0"
+cow-utils = "0.1.2"
 crossbeam-channel = "0.4.2"
 deunicode = "1.1.0"
+either = "1.5.3"
 env_logger = "0.7.1"
-fst = { version = "0.3.5", default-features = false }
+fst = "0.4.4"
 hashbrown = { version = "0.7.1", features = ["serde"] }
-heed = "0.7.0"
+heed = "0.8.0"
 indexmap = { version = "1.3.2", features = ["serde-1"] }
 intervaltree = "0.2.5"
 itertools = "0.9.0"
-levenshtein_automata = { version = "0.1.1", features = ["fst_automaton"] }
+levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }
 log = "0.4.8"
-meilisearch-schema = { path = "../meilisearch-schema", version = "0.10.0" }
-meilisearch-tokenizer = { path = "../meilisearch-tokenizer", version = "0.10.0" }
-meilisearch-types = { path = "../meilisearch-types", version = "0.10.0" }
+meilisearch-error = { path = "../meilisearch-error", version = "0.13.0" }
+meilisearch-schema = { path = "../meilisearch-schema", version = "0.13.0" }
+meilisearch-tokenizer = { path = "../meilisearch-tokenizer", version = "0.13.0" }
+meilisearch-types = { path = "../meilisearch-types", version = "0.13.0" }
 once_cell = "1.3.1"
 ordered-float = { version = "1.0.2", features = ["serde"] }
 pest = { git = "https://github.com/MarinPostma/pest.git", tag = "meilisearch-patch1" }
@@ -32,8 +35,7 @@ pest_derive = "2.0"
 regex = "1.3.6"
 sdset = "0.4.0"
 serde = { version = "1.0.105", features = ["derive"] }
-serde_json = "1.0.50"
-siphasher = "0.3.2"
+serde_json = { version = "1.0.50", features = ["preserve_order"] }
 slice-group-by = "0.2.6"
 unicase = "2.6.0"
 zerocopy = "0.3.0"
@@ -42,12 +44,14 @@ zerocopy = "0.3.0"
 assert_matches = "1.3.0"
 criterion = "0.3.1"
 csv = "1.1.3"
-jemallocator = "0.3.2"
 rustyline = { version = "6.0.0", default-features = false }
 structopt = "0.3.12"
 tempfile = "3.1.0"
 termcolor = "1.1.0"
 
+[target.'cfg(unix)'.dev-dependencies]
+jemallocator = "0.3.2"
+
 [[bench]]
 name = "search_benchmark"
 harness = false

meilisearch-core/benches/search_benchmark.rs

@@ -2,13 +2,14 @@
 #[macro_use]
 extern crate assert_matches;
 
-use std::sync::mpsc;
-use std::path::Path;
+use std::error::Error;
 use std::fs::File;
 use std::io::BufReader;
 use std::iter;
+use std::path::Path;
+use std::sync::mpsc;
 
-use meilisearch_core::Database;
+use meilisearch_core::{Database, DatabaseOptions};
 use meilisearch_core::{ProcessedUpdateResult, UpdateStatus};
 use meilisearch_core::settings::{Settings, SettingsUpdate};
 use meilisearch_schema::Schema;
@@ -17,7 +18,7 @@ use serde_json::Value;
 use criterion::{criterion_group, criterion_main, Criterion, BenchmarkId};
 
 fn prepare_database(path: &Path) -> Database {
-    let database = Database::open_or_create(path).unwrap();
+    let database = Database::open_or_create(path, DatabaseOptions::default()).unwrap();
     let db = &database;
 
     let (sender, receiver) = mpsc::sync_channel(100);
@@ -27,22 +28,24 @@ fn prepare_database(path: &Path) -> Database {
     let index = database.create_index("bench").unwrap();
 
     database.set_update_callback(Box::new(update_fn));
-    
-    let mut writer = db.main_write_txn().unwrap();
-    index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
-    writer.commit().unwrap();
+
+    db.main_write::<_, _, Box<dyn Error>>(|writer| {
+        index.main.put_schema(writer, &Schema::with_primary_key("id")).unwrap();
+        Ok(())
+    }).unwrap();
 
     let settings_update: SettingsUpdate = {
         let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../datasets/movies/settings.json");
         let file = File::open(path).unwrap();
         let reader = BufReader::new(file);
         let settings: Settings = serde_json::from_reader(reader).unwrap();
-        settings.into_update().unwrap()
+        settings.to_update().unwrap()
     };
 
-    let mut update_writer = db.update_write_txn().unwrap();
-    let _update_id = index.settings_update(&mut update_writer, settings_update).unwrap();
-    update_writer.commit().unwrap();
+    db.update_write::<_, _, Box<dyn Error>>(|writer| {
+        let _update_id = index.settings_update(writer, settings_update).unwrap();
+        Ok(())
+    }).unwrap();
 
     let mut additions = index.documents_addition();
 
@@ -58,9 +61,10 @@ fn prepare_database(path: &Path) -> Database {
         additions.update_document(document);
     }
 
-    let mut update_writer = db.update_write_txn().unwrap();
-    let update_id = additions.finalize(&mut update_writer).unwrap();
-    update_writer.commit().unwrap();
+    let update_id = db.update_write::<_, _, Box<dyn Error>>(|writer| {
+        let update_id = additions.finalize(writer).unwrap();
+        Ok(update_id)
+    }).unwrap();
 
     // block until the transaction is processed
     let _ = receiver.into_iter().find(|id| *id == update_id);

meilisearch-core/examples/from_file.rs

@@ -12,11 +12,11 @@ use serde::{Deserialize, Serialize};
 use structopt::StructOpt;
 use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
 
-use meilisearch_core::{Database, Highlight, ProcessedUpdateResult};
+use meilisearch_core::{Database, DatabaseOptions, Highlight, ProcessedUpdateResult};
 use meilisearch_core::settings::Settings;
 use meilisearch_schema::FieldId;
 
-// #[cfg(target_os = "linux")]
+#[cfg(target_os = "linux")]
 #[global_allocator]
 static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
 
@@ -123,12 +123,10 @@ fn index_command(command: IndexCommand, database: Database) -> Result<(), Box<dy
     let settings = {
         let string = fs::read_to_string(&command.settings)?;
         let settings: Settings = serde_json::from_str(&string).unwrap();
-        settings.into_update().unwrap()
+        settings.to_update().unwrap()
     };
 
-    let mut update_writer = db.update_write_txn().unwrap();
-    index.settings_update(&mut update_writer, settings)?;
-    update_writer.commit().unwrap();
+    db.update_write(|w| index.settings_update(w, settings))?;
 
     let mut rdr = if command.csv_data_path.as_os_str() == "-" {
         csv::Reader::from_reader(Box::new(io::stdin()) as Box<dyn Read>)
@@ -175,10 +173,9 @@ fn index_command(command: IndexCommand, database: Database) -> Result<(), Box<dy
 
         println!();
 
-        let mut update_writer = db.update_write_txn().unwrap();
+        let update_id = db.update_write(|w| additions.finalize(w))?;
+
         println!("committing update...");
-        let update_id = additions.finalize(&mut update_writer)?;
-        update_writer.commit().unwrap();
         max_update_id = max_update_id.max(update_id);
         println!("committed update {}", update_id);
     }
@@ -325,7 +322,7 @@ fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<
 
     let reader = db.main_read_txn().unwrap();
     let schema = index.main.schema(&reader)?;
-    reader.abort();
+    reader.abort().unwrap();
 
     let schema = schema.ok_or(meilisearch_core::Error::SchemaMissing)?;
 
@@ -371,12 +368,12 @@ fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<
                     });
                 }
 
-                let (documents, _nb_hits) = builder.query(ref_reader, &query, 0..command.number_results)?;
+                let result = builder.query(ref_reader, Some(&query), 0..command.number_results)?;
 
                 let mut retrieve_duration = Duration::default();
 
-                let number_of_documents = documents.len();
-                for mut doc in documents {
+                let number_of_documents = result.documents.len();
+                for mut doc in result.documents {
                     doc.highlights
                         .sort_unstable_by_key(|m| (m.char_index, m.char_length));
 
@@ -454,7 +451,7 @@ fn show_updates_command(
     let reader = db.update_read_txn().unwrap();
     let updates = index.all_updates_status(&reader)?;
     println!("{:#?}", updates);
-    reader.abort();
+    reader.abort().unwrap();
 
     Ok(())
 }
@@ -463,7 +460,7 @@ fn main() -> Result<(), Box<dyn Error>> {
     env_logger::init();
 
     let opt = Command::from_args();
-    let database = Database::open_or_create(opt.path())?;
+    let database = Database::open_or_create(opt.path(), DatabaseOptions::default())?;
 
     match opt {
         Command::Index(command) => index_command(command, database),

meilisearch-core/src/bucket_sort.rs

@@ -10,34 +10,41 @@ use std::fmt;
 
 use compact_arena::{SmallArena, Idx32, mk_arena};
 use log::debug;
-use meilisearch_types::DocIndex;
-use sdset::{Set, SetBuf, exponential_search};
+use sdset::{Set, SetBuf, exponential_search, SetOperation, Counter, duo::OpBuilder};
 use slice_group_by::{GroupBy, GroupByMut};
 
-use crate::error::Error;
+use meilisearch_types::DocIndex;
+
 use crate::criterion::{Criteria, Context, ContextMut};
 use crate::distinct_map::{BufferedDistinctMap, DistinctMap};
 use crate::raw_document::RawDocument;
 use crate::{database::MainT, reordered_attrs::ReorderedAttrs};
-use crate::{store, Document, DocumentId, MResult};
+use crate::{store, Document, DocumentId, MResult, Index, RankedMap, MainReader, Error};
 use crate::query_tree::{create_query_tree, traverse_query_tree};
 use crate::query_tree::{Operation, QueryResult, QueryKind, QueryId, PostingsKey};
 use crate::query_tree::Context as QTContext;
 
+#[derive(Debug, Default)]
+pub struct SortResult {
+    pub documents: Vec<Document>,
+    pub nb_hits: usize,
+    pub exhaustive_nb_hit: bool,
+    pub facets: Option<HashMap<String, HashMap<String, usize>>>,
+    pub exhaustive_facets_count: Option<bool>,
+}
+
+#[allow(clippy::too_many_arguments)]
 pub fn bucket_sort<'c, FI>(
     reader: &heed::RoTxn<MainT>,
     query: &str,
     range: Range<usize>,
+    facets_docids: Option<SetBuf<DocumentId>>,
+    facet_count_docids: Option<HashMap<String, HashMap<String, Cow<Set<DocumentId>>>>>,
     filter: Option<FI>,
     criteria: Criteria<'c>,
     searchable_attrs: Option<ReorderedAttrs>,
-    main_store: store::Main,
-    postings_lists_store: store::PostingsLists,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    synonyms_store: store::Synonyms,
-    prefix_documents_cache_store: store::PrefixDocumentsCache,
-    prefix_postings_lists_cache_store: store::PrefixPostingsListsCache,
-) -> MResult<(Vec<Document>, usize)>
+    index: &Index,
+) -> MResult<SortResult>
 where
     FI: Fn(DocumentId) -> bool,
 {
@@ -50,33 +57,28 @@ where
             reader,
             query,
             range,
+            facets_docids,
+            facet_count_docids,
             filter,
             distinct,
             distinct_size,
             criteria,
             searchable_attrs,
-            main_store,
-            postings_lists_store,
-            documents_fields_counts_store,
-            synonyms_store,
-            prefix_documents_cache_store,
-            prefix_postings_lists_cache_store,
+            index,
         );
     }
 
-    let words_set = match unsafe { main_store.static_words_fst(reader)? } {
-        Some(words) => words,
-        None => return Ok((Vec::new(), 0)),
-    };
+    let mut result = SortResult::default();
 
-    let stop_words = main_store.stop_words_fst(reader)?.unwrap_or_default();
+    let words_set = index.main.words_fst(reader)?;
+    let stop_words = index.main.stop_words_fst(reader)?;
 
     let context = QTContext {
         words_set,
         stop_words,
-        synonyms: synonyms_store,
-        postings_lists: postings_lists_store,
-        prefix_postings_lists: prefix_postings_lists_cache_store,
+        synonyms: index.synonyms,
+        postings_lists: index.postings_lists,
+        prefix_postings_lists: index.prefix_postings_lists_cache,
     };
 
     let (operation, mapping) = create_query_tree(reader, &context, query)?;
@@ -94,10 +96,23 @@ where
     let mut queries_kinds = HashMap::new();
     recurs_operation(&mut queries_kinds, &operation);
 
-    let QueryResult { docids, queries } = traverse_query_tree(reader, &context, &operation)?;
+    let QueryResult { mut docids, queries } = traverse_query_tree(reader, &context, &operation)?;
     debug!("found {} documents", docids.len());
     debug!("number of postings {:?}", queries.len());
 
+    if let Some(facets_docids) = facets_docids {
+        let intersection = sdset::duo::OpBuilder::new(docids.as_ref(), facets_docids.as_set())
+            .intersection()
+            .into_set_buf();
+        docids = Cow::Owned(intersection);
+    }
+
+    if let Some(f) = facet_count_docids {
+        // hardcoded value, until approximation optimization
+        result.exhaustive_facets_count = Some(true);
+        result.facets = Some(facet_count(f, &docids));
+    }
+
     let before = Instant::now();
     mk_arena!(arena);
     let mut bare_matches = cleanup_bare_matches(&mut arena, &docids, queries);
@@ -132,7 +147,7 @@ where
                 reader,
                 postings_lists: &mut arena,
                 query_mapping: &mapping,
-                documents_fields_counts_store,
+                documents_fields_counts_store: index.documents_fields_counts,
             };
 
             criterion.prepare(ctx, &mut group)?;
@@ -165,49 +180,48 @@ where
     debug!("criterion loop took {:.02?}", before_criterion_loop.elapsed());
     debug!("proximity evaluation called {} times", proximity_count.load(Ordering::Relaxed));
 
-    let schema = main_store.schema(reader)?.ok_or(Error::SchemaMissing)?;
+    let schema = index.main.schema(reader)?.ok_or(Error::SchemaMissing)?;
     let iter = raw_documents.into_iter().skip(range.start).take(range.len());
     let iter = iter.map(|rd| Document::from_raw(rd, &queries_kinds, &arena, searchable_attrs.as_ref(), &schema));
     let documents = iter.collect();
 
     debug!("bucket sort took {:.02?}", before_bucket_sort.elapsed());
 
-    Ok((documents, docids.len()))
+    result.documents = documents;
+    result.nb_hits = docids.len();
+
+    Ok(result)
 }
 
+#[allow(clippy::too_many_arguments)]
 pub fn bucket_sort_with_distinct<'c, FI, FD>(
     reader: &heed::RoTxn<MainT>,
     query: &str,
     range: Range<usize>,
+    facets_docids: Option<SetBuf<DocumentId>>,
+    facet_count_docids: Option<HashMap<String, HashMap<String, Cow<Set<DocumentId>>>>>,
     filter: Option<FI>,
     distinct: FD,
     distinct_size: usize,
     criteria: Criteria<'c>,
     searchable_attrs: Option<ReorderedAttrs>,
-    main_store: store::Main,
-    postings_lists_store: store::PostingsLists,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    synonyms_store: store::Synonyms,
-    _prefix_documents_cache_store: store::PrefixDocumentsCache,
-    prefix_postings_lists_cache_store: store::PrefixPostingsListsCache,
-) -> MResult<(Vec<Document>, usize)>
+    index: &Index,
+) -> MResult<SortResult>
 where
     FI: Fn(DocumentId) -> bool,
     FD: Fn(DocumentId) -> Option<u64>,
 {
-    let words_set = match unsafe { main_store.static_words_fst(reader)? } {
-        Some(words) => words,
-        None => return Ok((Vec::new(), 0)),
-    };
+    let mut result = SortResult::default();
 
-    let stop_words = main_store.stop_words_fst(reader)?.unwrap_or_default();
+    let words_set = index.main.words_fst(reader)?;
+    let stop_words = index.main.stop_words_fst(reader)?;
 
     let context = QTContext {
         words_set,
         stop_words,
-        synonyms: synonyms_store,
-        postings_lists: postings_lists_store,
-        prefix_postings_lists: prefix_postings_lists_cache_store,
+        synonyms: index.synonyms,
+        postings_lists: index.postings_lists,
+        prefix_postings_lists: index.prefix_postings_lists_cache,
     };
 
     let (operation, mapping) = create_query_tree(reader, &context, query)?;
@@ -225,10 +239,23 @@ where
     let mut queries_kinds = HashMap::new();
     recurs_operation(&mut queries_kinds, &operation);
 
-    let QueryResult { docids, queries } = traverse_query_tree(reader, &context, &operation)?;
+    let QueryResult { mut docids, queries } = traverse_query_tree(reader, &context, &operation)?;
     debug!("found {} documents", docids.len());
     debug!("number of postings {:?}", queries.len());
 
+    if let Some(facets_docids) = facets_docids {
+        let intersection = OpBuilder::new(docids.as_ref(), facets_docids.as_set())
+            .intersection()
+            .into_set_buf();
+        docids = Cow::Owned(intersection);
+    }
+
+    if let Some(f) = facet_count_docids {
+        // hardcoded value, until approximation optimization
+        result.exhaustive_facets_count = Some(true);
+        result.facets = Some(facet_count(f, &docids));
+    }
+
     let before = Instant::now();
     mk_arena!(arena);
     let mut bare_matches = cleanup_bare_matches(&mut arena, &docids, queries);
@@ -273,7 +300,7 @@ where
                 reader,
                 postings_lists: &mut arena,
                 query_mapping: &mapping,
-                documents_fields_counts_store,
+                documents_fields_counts_store: index.documents_fields_counts,
             };
 
             let before_criterion_preparation = Instant::now();
@@ -338,7 +365,7 @@ where
     // once we classified the documents related to the current
     // automatons we save that as the next valid result
     let mut seen = BufferedDistinctMap::new(&mut distinct_map);
-    let schema = main_store.schema(reader)?.ok_or(Error::SchemaMissing)?;
+    let schema = index.main.schema(reader)?.ok_or(Error::SchemaMissing)?;
 
     let mut documents = Vec::with_capacity(range.len());
     for raw_document in raw_documents.into_iter().skip(distinct_raw_offset) {
@@ -362,8 +389,10 @@ where
             }
         }
     }
+    result.documents = documents;
+    result.nb_hits = docids.len();
 
-    Ok((documents, docids.len()))
+    Ok(result)
 }
 
 fn cleanup_bare_matches<'tag, 'txn>(
@@ -558,3 +587,69 @@ impl Deref for PostingsListView<'_> {
         }
     }
 }
+
+/// sorts documents ids according to user defined ranking rules.
+pub fn placeholder_document_sort(
+    document_ids: &mut [DocumentId],
+    index: &store::Index,
+    reader: &MainReader,
+    ranked_map: &RankedMap
+) -> MResult<()> {
+    use crate::settings::RankingRule;
+    use std::cmp::Ordering;
+
+    enum SortOrder {
+        Asc,
+        Desc,
+    }
+
+    if let Some(ranking_rules) = index.main.ranking_rules(reader)? {
+        let schema = index.main.schema(reader)?
+            .ok_or(Error::SchemaMissing)?;
+
+        // Select custom rules from ranking rules, and map them to custom rules
+        // containing a field_id
+        let ranking_rules = ranking_rules.iter().filter_map(|r|
+            match r {
+                RankingRule::Asc(name) => schema.id(name).map(|f| (f, SortOrder::Asc)),
+                RankingRule::Desc(name) => schema.id(name).map(|f| (f, SortOrder::Desc)),
+                _ => None,
+            }).collect::<Vec<_>>();
+
+        document_ids.sort_unstable_by(|a, b| {
+            for (field_id, order) in &ranking_rules {
+                let a_value = ranked_map.get(*a, *field_id);
+                let b_value = ranked_map.get(*b, *field_id);
+                let (a, b) = match order {
+                    SortOrder::Asc => (a_value, b_value),
+                    SortOrder::Desc => (b_value, a_value),
+                };
+                match a.cmp(&b) {
+                    Ordering::Equal => continue,
+                    ordering => return ordering,
+                }
+            }
+            Ordering::Equal
+        });
+    }
+    Ok(())
+}
+
+/// For each entry in facet_docids, calculates the number of documents in the intersection with candidate_docids.
+pub fn facet_count(
+    facet_docids: HashMap<String, HashMap<String, Cow<Set<DocumentId>>>>,
+    candidate_docids: &Set<DocumentId>,
+) -> HashMap<String, HashMap<String, usize>> {
+    let mut facets_counts = HashMap::with_capacity(facet_docids.len());
+    for (key, doc_map) in facet_docids {
+        let mut count_map = HashMap::with_capacity(doc_map.len());
+        for (value, docids) in doc_map {
+            let mut counter = Counter::new();
+            let op = OpBuilder::new(docids.as_ref(), candidate_docids).intersection();
+            SetOperation::<DocumentId>::extend_collection(op, &mut counter);
+            count_map.insert(value, counter.0);
+        }
+        facets_counts.insert(key, count_map);
+    }
+    facets_counts
+}

meilisearch-core/src/criterion/mod.rs

@@ -92,6 +92,7 @@ impl<'a> CriteriaBuilder<'a> {
         self.inner.reserve(additional)
     }
 
+    #[allow(clippy::should_implement_trait)]
     pub fn add<C: 'a>(mut self, criterion: C) -> CriteriaBuilder<'a>
     where
         C: Criterion,

meilisearch-core/src/criterion/typo.rs

@@ -22,6 +22,7 @@ impl Criterion for Typo {
         // It is safe to panic on input number higher than 3,
         // the number of typos is never bigger than that.
         #[inline]
+        #[allow(clippy::approx_constant)]
         fn custom_log10(n: u8) -> f32 {
             match n {
                 0 => 0.0,     // log(1)

meilisearch-core/src/database.rs

@@ -3,18 +3,33 @@ use std::fs::File;
 use std::path::Path;
 use std::sync::{Arc, RwLock};
 use std::{fs, thread};
+use std::io::{Read, Write, ErrorKind};
 
+use chrono::{DateTime, Utc};
 use crossbeam_channel::{Receiver, Sender};
-use heed::types::{Str, Unit};
-use heed::{CompactionOption, Result as ZResult};
-use log::debug;
+use heed::CompactionOption;
+use heed::types::{Str, Unit, SerdeBincode};
+use log::{debug, error};
 use meilisearch_schema::Schema;
+use regex::Regex;
 
-use crate::{store, update, Index, MResult};
+use crate::{store, update, Index, MResult, Error};
 
 pub type BoxUpdateFn = Box<dyn Fn(&str, update::ProcessedUpdateResult) + Send + Sync + 'static>;
+
 type ArcSwapFn = arc_swap::ArcSwapOption<BoxUpdateFn>;
 
+type SerdeDatetime = SerdeBincode<DateTime<Utc>>;
+
+pub type MainWriter<'a> = heed::RwTxn<'a, MainT>;
+pub type MainReader = heed::RoTxn<MainT>;
+
+pub type UpdateWriter<'a> = heed::RwTxn<'a, UpdateT>;
+pub type UpdateReader = heed::RoTxn<UpdateT>;
+
+const UNHEALTHY_KEY: &str = "_is_unhealthy";
+const LAST_UPDATE_KEY: &str = "last-update";
+
 pub struct MainT;
 pub struct UpdateT;
 
@@ -27,6 +42,20 @@ pub struct Database {
     update_fn: Arc<ArcSwapFn>,
 }
 
+pub struct DatabaseOptions {
+    pub main_map_size: usize,
+    pub update_map_size: usize,
+}
+
+impl Default for DatabaseOptions {
+    fn default() -> DatabaseOptions {
+        DatabaseOptions {
+            main_map_size: 100 * 1024 * 1024 * 1024, //100Gb
+            update_map_size: 100 * 1024 * 1024 * 1024, //100Gb
+        }
+    }
+}
+
 macro_rules! r#break_try {
     ($expr:expr, $msg:tt) => {
         match $expr {
@@ -55,8 +84,7 @@ fn update_awaiter(
     update_fn: Arc<ArcSwapFn>,
     index: Index,
 ) -> MResult<()> {
-    let mut receiver = receiver.into_iter();
-    while let Some(event) = receiver.next() {
+    for event in receiver {
 
         // if we receive a *MustClear* event, clear the index and break the loop
         if let UpdateEvent::MustClear = event {
@@ -90,7 +118,7 @@ fn update_awaiter(
             };
 
             // do not keep the reader for too long
-            update_reader.abort();
+            break_try!(update_reader.abort(), "aborting update transaction failed");
 
             // instantiate a transaction to touch to the main env
             let result = env.typed_write_txn::<MainT>();
@@ -104,7 +132,7 @@ fn update_awaiter(
             if status.error.is_none() {
                 break_try!(main_writer.commit(), "commit nested transaction failed");
             } else {
-                main_writer.abort()
+                break_try!(main_writer.abort(), "abborting nested transaction failed");
             }
 
             // now that the update has been processed we can instantiate
@@ -135,20 +163,78 @@ fn update_awaiter(
     Ok(())
 }
 
+/// Ensures Meilisearch version is compatible with the database, returns an error versions mismatch.
+/// If create is set to true, a VERSION file is created with the current version.
+fn version_guard(path: &Path, create: bool) -> MResult<()> {
+    let current_version_major = env!("CARGO_PKG_VERSION_MAJOR");
+    let current_version_minor = env!("CARGO_PKG_VERSION_MINOR");
+    let current_version_patch = env!("CARGO_PKG_VERSION_PATCH");
+    let version_path = path.join("VERSION");
+
+    match File::open(&version_path) {
+        Ok(mut file) => {
+            let mut version = String::new();
+            file.read_to_string(&mut version)?;
+            // Matches strings like XX.XX.XX
+            let re = Regex::new(r"(\d+).(\d+).(\d+)").unwrap();
+
+            // Make sure there is a result
+            let version = re
+                .captures_iter(&version)
+                .next()
+                .ok_or(Error::VersionMismatch("bad VERSION file".to_string()))?;
+            // the first is always the complete match, safe to unwrap because we have a match
+            let version_major = version.get(1).unwrap().as_str();
+            let version_minor = version.get(2).unwrap().as_str();
+
+            if version_major != current_version_major || version_minor != current_version_minor {
+                return Err(Error::VersionMismatch(format!("{}.{}.XX", version_major, version_minor)));
+            }
+        }
+        Err(error) => {
+            match error.kind() {
+                ErrorKind::NotFound => {
+                    if create {
+                        // when no version file is found, and we've been told to create one,
+                        // create a new file with the current version in it.
+                        let mut version_file = File::create(&version_path)?;
+                        version_file.write_all(format!("{}.{}.{}",
+                                current_version_major,
+                                current_version_minor,
+                                current_version_patch).as_bytes())?;
+                    } else {
+                        // when no version file is found and we were not told to create one, this
+                        // means that the version is inferior to the one this feature was added in.
+                        return Err(Error::VersionMismatch(format!("<0.12.0")));
+                    }
+                }
+                _ => return Err(error.into())
+            }
+        }
+    }
+    Ok(())
+}
+
 impl Database {
-    pub fn open_or_create(path: impl AsRef<Path>) -> MResult<Database> {
+    pub fn open_or_create(path: impl AsRef<Path>, options: DatabaseOptions) -> MResult<Database> {
         let main_path = path.as_ref().join("main");
         let update_path = path.as_ref().join("update");
 
+        //create db directory
+        fs::create_dir_all(&path)?;
+
+        // create file only if main db wasn't created before (first run)
+        version_guard(path.as_ref(), !main_path.exists() && !update_path.exists())?;
+
         fs::create_dir_all(&main_path)?;
         let env = heed::EnvOpenOptions::new()
-            .map_size(100 * 1024 * 1024 * 1024) // 100GB
+            .map_size(options.main_map_size)
             .max_dbs(3000)
             .open(main_path)?;
 
         fs::create_dir_all(&update_path)?;
         let update_env = heed::EnvOpenOptions::new()
-            .map_size(100 * 1024 * 1024 * 1024) // 100GB
+            .map_size(options.update_map_size)
             .max_dbs(3000)
             .open(update_path)?;
 
@@ -164,7 +250,7 @@ impl Database {
             must_open.push(index_uid.to_owned());
         }
 
-        reader.abort();
+        reader.abort()?;
 
         // open the previously aggregated indexes
         let mut indexes = HashMap::new();
@@ -227,6 +313,13 @@ impl Database {
         }
     }
 
+    pub fn is_indexing(&self, reader: &UpdateReader, index: &str) -> MResult<Option<bool>> {
+        match self.open_index(&index) {
+            Some(index) => index.current_update_id(&reader).map(|u| Some(u.is_some())),
+            None => Ok(None),
+        }
+    }
+
     pub fn create_index(&self, name: impl AsRef<str>) -> MResult<Index> {
         let name = name.as_ref();
         let mut indexes_lock = self.indexes.write().unwrap();
@@ -305,23 +398,73 @@ impl Database {
         self.update_fn.swap(None);
     }
 
-    pub fn main_read_txn(&self) -> heed::Result<heed::RoTxn<MainT>> {
-        self.env.typed_read_txn::<MainT>()
+    pub fn main_read_txn(&self) -> MResult<MainReader> {
+        Ok(self.env.typed_read_txn::<MainT>()?)
     }
 
-    pub fn main_write_txn(&self) -> heed::Result<heed::RwTxn<MainT>> {
-        self.env.typed_write_txn::<MainT>()
+    pub(crate) fn main_write_txn(&self) -> MResult<MainWriter> {
+        Ok(self.env.typed_write_txn::<MainT>()?)
     }
 
-    pub fn update_read_txn(&self) -> heed::Result<heed::RoTxn<UpdateT>> {
-        self.update_env.typed_read_txn::<UpdateT>()
+    /// Calls f providing it with a writer to the main database. After f is called, makes sure the
+    /// transaction is commited. Returns whatever result f returns.
+    pub fn main_write<F, R, E>(&self, f: F) -> Result<R, E>
+    where
+        F: FnOnce(&mut MainWriter) -> Result<R, E>,
+        E: From<Error>,
+    {
+        let mut writer = self.main_write_txn()?;
+        let result = f(&mut writer)?;
+        writer.commit().map_err(Error::Heed)?;
+        Ok(result)
     }
 
-    pub fn update_write_txn(&self) -> heed::Result<heed::RwTxn<UpdateT>> {
-        self.update_env.typed_write_txn::<UpdateT>()
+    /// provides a context with a reader to the main database. experimental.
+    pub fn main_read<F, R, E>(&self, f: F) -> Result<R, E>
+    where
+        F: FnOnce(&MainReader) -> Result<R, E>,
+        E: From<Error>,
+    {
+        let reader = self.main_read_txn()?;
+        let result = f(&reader)?;
+        reader.abort().map_err(Error::Heed)?;
+        Ok(result)
     }
 
-    pub fn copy_and_compact_to_path<P: AsRef<Path>>(&self, path: P) -> ZResult<(File, File)> {
+    pub fn update_read_txn(&self) -> MResult<UpdateReader> {
+        Ok(self.update_env.typed_read_txn::<UpdateT>()?)
+    }
+
+    pub(crate) fn update_write_txn(&self) -> MResult<heed::RwTxn<UpdateT>> {
+        Ok(self.update_env.typed_write_txn::<UpdateT>()?)
+    }
+
+    /// Calls f providing it with a writer to the main database. After f is called, makes sure the
+    /// transaction is commited. Returns whatever result f returns.
+    pub fn update_write<F, R, E>(&self, f: F) -> Result<R, E>
+    where
+        F: FnOnce(&mut UpdateWriter) -> Result<R, E>,
+        E: From<Error>,
+    {
+        let mut writer = self.update_write_txn()?;
+        let result = f(&mut writer)?;
+        writer.commit().map_err(Error::Heed)?;
+        Ok(result)
+    }
+
+    /// provides a context with a reader to the update database. experimental.
+    pub fn update_read<F, R, E>(&self, f: F) -> Result<R, E>
+    where
+        F: FnOnce(&UpdateReader) -> Result<R, E>,
+        E: From<Error>,
+    {
+        let reader = self.update_read_txn()?;
+        let result = f(&reader)?;
+        reader.abort().map_err(Error::Heed)?;
+        Ok(result)
+    }
+
+    pub fn copy_and_compact_to_path<P: AsRef<Path>>(&self, path: P) -> MResult<(File, File)> {
         let path = path.as_ref();
 
         let env_path = path.join("main");
@@ -338,7 +481,7 @@ impl Database {
             Ok(update_env_file) => Ok((env_file, update_env_file)),
             Err(e) => {
                 fs::remove_file(env_path)?;
-                Err(e)
+                Err(e.into())
             },
         }
     }
@@ -348,15 +491,85 @@ impl Database {
         indexes.keys().cloned().collect()
     }
 
-    pub fn common_store(&self) -> heed::PolyDatabase {
+    pub(crate) fn common_store(&self) -> heed::PolyDatabase {
         self.common_store
     }
+
+    pub fn last_update(&self, reader: &heed::RoTxn<MainT>) -> MResult<Option<DateTime<Utc>>> {
+        match self.common_store()
+            .get::<_, Str, SerdeDatetime>(reader, LAST_UPDATE_KEY)? {
+                Some(datetime) => Ok(Some(datetime)),
+                None => Ok(None),
+            }
+    }
+
+    pub fn set_last_update(&self, writer: &mut heed::RwTxn<MainT>, time: &DateTime<Utc>) -> MResult<()> {
+        self.common_store()
+            .put::<_, Str, SerdeDatetime>(writer, LAST_UPDATE_KEY, time)?;
+        Ok(())
+    }
+
+    pub fn set_healthy(&self, writer: &mut heed::RwTxn<MainT>) -> MResult<()> {
+        let common_store = self.common_store();
+        common_store.delete::<_, Str>(writer, UNHEALTHY_KEY)?;
+        Ok(())
+    }
+
+    pub fn set_unhealthy(&self, writer: &mut heed::RwTxn<MainT>) -> MResult<()> {
+        let common_store = self.common_store();
+        common_store.put::<_, Str, Unit>(writer, UNHEALTHY_KEY, &())?;
+        Ok(())
+    }
+
+    pub fn get_health(&self, reader: &heed::RoTxn<MainT>) -> MResult<Option<()>> {
+        let common_store = self.common_store();
+        Ok(common_store.get::<_, Str, Unit>(&reader, UNHEALTHY_KEY)?)
+    }
+
+    pub fn compute_stats(&self, writer: &mut MainWriter, index_uid: &str) -> MResult<()> {
+        let index = match self.open_index(&index_uid) {
+            Some(index) => index,
+            None => {
+                error!("Impossible to retrieve index {}", index_uid);
+                return Ok(());
+            }
+        };
+
+        let schema = match index.main.schema(&writer)? {
+            Some(schema) => schema,
+            None => return Ok(()),
+        };
+
+        let all_documents_fields = index
+            .documents_fields_counts
+            .all_documents_fields_counts(&writer)?;
+
+        // count fields frequencies
+        let mut fields_frequency = HashMap::<_, usize>::new();
+        for result in all_documents_fields {
+            let (_, attr, _) = result?;
+            if let Some(field_id) = schema.indexed_pos_to_field_id(attr) {
+                *fields_frequency.entry(field_id).or_default() += 1;
+            }
+        }
+
+        // convert attributes to their names
+        let frequency: HashMap<_, _> = fields_frequency
+            .into_iter()
+            .filter_map(|(a, c)| schema.name(a).map(|name| (name.to_string(), c)))
+            .collect();
+
+        index
+            .main
+            .put_fields_distribution(writer, &frequency)
+    }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    use crate::bucket_sort::SortResult;
     use crate::criterion::{self, CriteriaBuilder};
     use crate::update::{ProcessedUpdateResult, UpdateStatus};
     use crate::settings::Settings;
@@ -368,7 +581,7 @@ mod tests {
     fn valid_updates() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -393,7 +606,7 @@ mod tests {
                 }
             "#;
             let settings: Settings = serde_json::from_str(data).unwrap();
-            settings.into_update().unwrap()
+            settings.to_update().unwrap()
         };
 
         let mut update_writer = db.update_write_txn().unwrap();
@@ -433,7 +646,7 @@ mod tests {
     fn invalid_updates() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -456,7 +669,7 @@ mod tests {
                 }
             "#;
             let settings: Settings = serde_json::from_str(data).unwrap();
-            settings.into_update().unwrap()
+            settings.to_update().unwrap()
         };
 
         let mut update_writer = db.update_write_txn().unwrap();
@@ -495,7 +708,7 @@ mod tests {
     fn ignored_words_too_long() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -518,7 +731,7 @@ mod tests {
                 }
             "#;
             let settings: Settings = serde_json::from_str(data).unwrap();
-            settings.into_update().unwrap()
+            settings.to_update().unwrap()
         };
 
         let mut update_writer = db.update_write_txn().unwrap();
@@ -550,7 +763,7 @@ mod tests {
     fn add_schema_attributes_at_end() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -573,7 +786,7 @@ mod tests {
                 }
             "#;
             let settings: Settings = serde_json::from_str(data).unwrap();
-            settings.into_update().unwrap()
+            settings.to_update().unwrap()
         };
 
         let mut update_writer = db.update_write_txn().unwrap();
@@ -609,7 +822,7 @@ mod tests {
                 }
             "#;
             let settings: Settings = serde_json::from_str(data).unwrap();
-            settings.into_update().unwrap()
+            settings.to_update().unwrap()
         };
 
         let mut writer = db.update_write_txn().unwrap();
@@ -623,7 +836,7 @@ mod tests {
         let update_reader = db.update_read_txn().unwrap();
         let result = index.update_status(&update_reader, update_id).unwrap();
         assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-        update_reader.abort();
+        update_reader.abort().unwrap();
 
         let mut additions = index.documents_addition();
 
@@ -657,14 +870,14 @@ mod tests {
         let update_reader = db.update_read_txn().unwrap();
         let result = index.update_status(&update_reader, update_id).unwrap();
         assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-        update_reader.abort();
+        update_reader.abort().unwrap();
 
         // even try to search for a document
         let reader = db.main_read_txn().unwrap();
-        let (results, _nb_hits) = index.query_builder().query(&reader, "21 ", 0..20).unwrap();
-        assert_matches!(results.len(), 1);
+        let SortResult {documents, .. } = index.query_builder().query(&reader, Some("21 "), 0..20).unwrap();
+        assert_matches!(documents.len(), 1);
 
-        reader.abort();
+        reader.abort().unwrap();
 
         // try to introduce attributes in the middle of the schema
         let settings = {
@@ -675,7 +888,7 @@ mod tests {
                 }
             "#;
             let settings: Settings = serde_json::from_str(data).unwrap();
-            settings.into_update().unwrap()
+            settings.to_update().unwrap()
         };
 
         let mut writer = db.update_write_txn().unwrap();
@@ -694,7 +907,7 @@ mod tests {
     fn deserialize_documents() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -717,7 +930,7 @@ mod tests {
                 }
             "#;
             let settings: Settings = serde_json::from_str(data).unwrap();
-            settings.into_update().unwrap()
+            settings.to_update().unwrap()
         };
 
         let mut writer = db.update_write_txn().unwrap();
@@ -753,19 +966,19 @@ mod tests {
         let update_reader = db.update_read_txn().unwrap();
         let result = index.update_status(&update_reader, update_id).unwrap();
         assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-        update_reader.abort();
+        update_reader.abort().unwrap();
 
         let reader = db.main_read_txn().unwrap();
         let document: Option<IgnoredAny> = index.document(&reader, None, DocumentId(25)).unwrap();
         assert!(document.is_none());
 
         let document: Option<IgnoredAny> = index
-            .document(&reader, None, DocumentId(7_900_334_843_754_999_545))
+            .document(&reader, None, DocumentId(0))
             .unwrap();
         assert!(document.is_some());
 
         let document: Option<IgnoredAny> = index
-            .document(&reader, None, DocumentId(8_367_468_610_878_465_872))
+            .document(&reader, None, DocumentId(1))
             .unwrap();
         assert!(document.is_some());
     }
@@ -774,7 +987,7 @@ mod tests {
     fn partial_document_update() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -797,7 +1010,7 @@ mod tests {
                 }
             "#;
             let settings: Settings = serde_json::from_str(data).unwrap();
-            settings.into_update().unwrap()
+            settings.to_update().unwrap()
         };
 
         let mut writer = db.update_write_txn().unwrap();
@@ -833,23 +1046,23 @@ mod tests {
         let update_reader = db.update_read_txn().unwrap();
         let result = index.update_status(&update_reader, update_id).unwrap();
         assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-        update_reader.abort();
+        update_reader.abort().unwrap();
 
         let reader = db.main_read_txn().unwrap();
         let document: Option<IgnoredAny> = index.document(&reader, None, DocumentId(25)).unwrap();
         assert!(document.is_none());
 
         let document: Option<IgnoredAny> = index
-            .document(&reader, None, DocumentId(7_900_334_843_754_999_545))
+            .document(&reader, None, DocumentId(0))
             .unwrap();
         assert!(document.is_some());
 
         let document: Option<IgnoredAny> = index
-            .document(&reader, None, DocumentId(8_367_468_610_878_465_872))
+            .document(&reader, None, DocumentId(1))
             .unwrap();
         assert!(document.is_some());
 
-        reader.abort();
+        reader.abort().unwrap();
 
         let mut partial_additions = index.documents_partial_addition();
 
@@ -878,11 +1091,11 @@ mod tests {
         let update_reader = db.update_read_txn().unwrap();
         let result = index.update_status(&update_reader, update_id).unwrap();
         assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-        update_reader.abort();
+        update_reader.abort().unwrap();
 
         let reader = db.main_read_txn().unwrap();
         let document: Option<serde_json::Value> = index
-            .document(&reader, None, DocumentId(7_900_334_843_754_999_545))
+            .document(&reader, None, DocumentId(0))
             .unwrap();
 
         let new_doc1 = serde_json::json!({
@@ -893,7 +1106,7 @@ mod tests {
         assert_eq!(document, Some(new_doc1));
 
         let document: Option<serde_json::Value> = index
-            .document(&reader, None, DocumentId(8_367_468_610_878_465_872))
+            .document(&reader, None, DocumentId(1))
             .unwrap();
 
         let new_doc2 = serde_json::json!({
@@ -908,7 +1121,7 @@ mod tests {
     fn delete_index() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Arc::new(Database::open_or_create(dir.path()).unwrap());
+        let database = Arc::new(Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap());
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -936,7 +1149,7 @@ mod tests {
                 }
             "#;
             let settings: Settings = serde_json::from_str(data).unwrap();
-            settings.into_update().unwrap()
+            settings.to_update().unwrap()
         };
 
         let mut writer = db.update_write_txn().unwrap();
@@ -980,7 +1193,7 @@ mod tests {
     fn check_number_ordering() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -1012,7 +1225,7 @@ mod tests {
                 }
             "#;
             let settings: Settings = serde_json::from_str(data).unwrap();
-            settings.into_update().unwrap()
+            settings.to_update().unwrap()
         };
 
         let mut writer = db.update_write_txn().unwrap();
@@ -1059,20 +1272,20 @@ mod tests {
 
         let builder = index.query_builder_with_criteria(criteria);
 
-        let (results, _nb_hits) = builder.query(&reader, "Kevin", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult {documents, .. } = builder.query(&reader, Some("Kevin"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(
             iter.next(),
             Some(Document {
-                id: DocumentId(7_900_334_843_754_999_545),
+                id: DocumentId(0),
                 ..
             })
         );
         assert_matches!(
             iter.next(),
             Some(Document {
-                id: DocumentId(8_367_468_610_878_465_872),
+                id: DocumentId(1),
                 ..
             })
         );

meilisearch-core/src/error.rs

@@ -9,26 +9,55 @@ pub use fst::Error as FstError;
 pub use heed::Error as HeedError;
 pub use pest::error as pest_error;
 
+use meilisearch_error::{ErrorCode, Code};
+
 pub type MResult<T> = Result<T, Error>;
 
 #[derive(Debug)]
 pub enum Error {
-    Io(io::Error),
-    IndexAlreadyExists,
-    MissingPrimaryKey,
-    SchemaMissing,
-    WordIndexMissing,
-    MissingDocumentId,
-    MaxFieldsLimitExceeded,
-    Schema(meilisearch_schema::Error),
-    Zlmdb(heed::Error),
-    Fst(fst::Error),
-    SerdeJson(SerdeJsonError),
     Bincode(bincode::Error),
-    Serializer(SerializerError),
     Deserializer(DeserializerError),
-    UnsupportedOperation(UnsupportedOperation),
-    FilterParseError(PestError<Rule>)
+    FacetError(FacetError),
+    FilterParseError(PestError<Rule>),
+    Fst(fst::Error),
+    Heed(heed::Error),
+    IndexAlreadyExists,
+    Io(io::Error),
+    MaxFieldsLimitExceeded,
+    MissingDocumentId,
+    MissingPrimaryKey,
+    Schema(meilisearch_schema::Error),
+    SchemaMissing,
+    SerdeJson(SerdeJsonError),
+    Serializer(SerializerError),
+    VersionMismatch(String),
+    WordIndexMissing,
+}
+
+impl ErrorCode for Error {
+    fn error_code(&self) -> Code {
+        use Error::*;
+
+        match self {
+            FacetError(_) => Code::Facet,
+            FilterParseError(_) => Code::Filter,
+            IndexAlreadyExists => Code::IndexAlreadyExists,
+            MissingPrimaryKey => Code::MissingPrimaryKey,
+            MissingDocumentId => Code::MissingDocumentId,
+            MaxFieldsLimitExceeded => Code::MaxFieldsLimitExceeded,
+            Schema(s) =>  s.error_code(),
+            WordIndexMissing
+            | SchemaMissing => Code::InvalidState,
+            Heed(_)
+            | Fst(_)
+            | SerdeJson(_)
+            | Bincode(_)
+            | Serializer(_)
+            | Deserializer(_)
+            | VersionMismatch(_)
+            | Io(_) => Code::Internal,
+        }
+    }
 }
 
 impl From<io::Error> for Error {
@@ -57,7 +86,13 @@ impl From<PestError<Rule>> for Error {
             s.to_string()
         }))
     }
-} 
+}
+
+impl From<FacetError> for Error {
+    fn from(error: FacetError) -> Error {
+        Error::FacetError(error)
+    }
+}
 
 impl From<meilisearch_schema::Error> for Error {
     fn from(error: meilisearch_schema::Error) -> Error {
@@ -67,7 +102,7 @@ impl From<meilisearch_schema::Error> for Error {
 
 impl From<HeedError> for Error {
     fn from(error: HeedError) -> Error {
-        Error::Zlmdb(error)
+        Error::Heed(error)
     }
 }
 
@@ -91,7 +126,10 @@ impl From<BincodeError> for Error {
 
 impl From<SerializerError> for Error {
     fn from(error: SerializerError) -> Error {
-        Error::Serializer(error)
+        match error {
+            SerializerError::DocumentIdNotFound => Error::MissingDocumentId,
+            e => Error::Serializer(e),
+        }
     }
 }
 
@@ -101,58 +139,86 @@ impl From<DeserializerError> for Error {
     }
 }
 
-impl From<UnsupportedOperation> for Error {
-    fn from(op: UnsupportedOperation) -> Error {
-        Error::UnsupportedOperation(op)
-    }
-}
-
 impl fmt::Display for Error {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         use self::Error::*;
         match self {
-            Io(e) => write!(f, "{}", e),
-            IndexAlreadyExists => write!(f, "index already exists"),
-            MissingPrimaryKey => write!(f, "schema cannot be built without a primary key"),
-            SchemaMissing => write!(f, "this index does not have a schema"),
-            WordIndexMissing => write!(f, "this index does not have a word index"),
-            MissingDocumentId => write!(f, "document id is missing"),
-            MaxFieldsLimitExceeded => write!(f, "maximum number of fields in a document exceeded"),
-            Schema(e) => write!(f, "schema error; {}", e),
-            Zlmdb(e) => write!(f, "heed error; {}", e),
-            Fst(e) => write!(f, "fst error; {}", e),
-            SerdeJson(e) => write!(f, "serde json error; {}", e),
             Bincode(e) => write!(f, "bincode error; {}", e),
-            Serializer(e) => write!(f, "serializer error; {}", e),
             Deserializer(e) => write!(f, "deserializer error; {}", e),
-            UnsupportedOperation(op) => write!(f, "unsupported operation; {}", op),
+            FacetError(e) => write!(f, "error processing facet filter: {}", e),
             FilterParseError(e) => write!(f, "error parsing filter; {}", e),
+            Fst(e) => write!(f, "fst error; {}", e),
+            Heed(e) => write!(f, "heed error; {}", e),
+            IndexAlreadyExists => write!(f, "index already exists"),
+            Io(e) => write!(f, "{}", e),
+            MaxFieldsLimitExceeded => write!(f, "maximum number of fields in a document exceeded"),
+            MissingDocumentId => write!(f, "document id is missing"),
+            MissingPrimaryKey => write!(f, "schema cannot be built without a primary key"),
+            Schema(e) => write!(f, "schema error; {}", e),
+            SchemaMissing => write!(f, "this index does not have a schema"),
+            SerdeJson(e) => write!(f, "serde json error; {}", e),
+            Serializer(e) => write!(f, "serializer error; {}", e),
+            VersionMismatch(version) => write!(f, "Cannot open database, expected MeiliSearch engine version: {}, current engine version: {}.{}.{}",
+                version,
+                env!("CARGO_PKG_VERSION_MAJOR"),
+                env!("CARGO_PKG_VERSION_MINOR"),
+                env!("CARGO_PKG_VERSION_PATCH")),
+            WordIndexMissing => write!(f, "this index does not have a word index"),
         }
     }
 }
 
 impl error::Error for Error {}
 
-#[derive(Debug)]
-pub enum UnsupportedOperation {
-    SchemaAlreadyExists,
-    CannotUpdateSchemaPrimaryKey,
-    CannotReorderSchemaAttribute,
-    CanOnlyIntroduceNewSchemaAttributesAtEnd,
-    CannotRemoveSchemaAttribute,
+struct FilterParseError(PestError<Rule>);
+
+impl fmt::Display for FilterParseError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use crate::pest_error::LineColLocation::*;
+
+        let (line, column) = match self.0.line_col {
+            Span((line, _), (column, _)) => (line, column),
+            Pos((line, column)) => (line, column),
+        };
+        write!(f, "parsing error on line {} at column {}: {}", line, column, self.0.variant.message())
+    }
 }
 
-impl fmt::Display for UnsupportedOperation {
+#[derive(Debug)]
+pub enum FacetError {
+    EmptyArray,
+    ParsingError(String),
+    UnexpectedToken { expected: &'static [&'static str], found: String },
+    InvalidFormat(String),
+    AttributeNotFound(String),
+    AttributeNotSet { expected: Vec<String>, found: String },
+    InvalidDocumentAttribute(String),
+    NoAttributesForFaceting,
+}
+
+impl FacetError {
+    pub fn unexpected_token(expected: &'static [&'static str], found: impl ToString) -> FacetError {
+        FacetError::UnexpectedToken{ expected, found: found.to_string() }
+    }
+
+    pub fn attribute_not_set(expected: Vec<String>, found: impl ToString) -> FacetError {
+        FacetError::AttributeNotSet{ expected, found: found.to_string() }
+    }
+}
+
+impl fmt::Display for FacetError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        use self::UnsupportedOperation::*;
+        use FacetError::*;
+
         match self {
-            SchemaAlreadyExists => write!(f, "Cannot update index which already have a schema"),
-            CannotUpdateSchemaPrimaryKey => write!(f, "Cannot update the primary key of a schema"),
-            CannotReorderSchemaAttribute => write!(f, "Cannot reorder the attributes of a schema"),
-            CanOnlyIntroduceNewSchemaAttributesAtEnd => {
-                write!(f, "Can only introduce new attributes at end of a schema")
-            }
-            CannotRemoveSchemaAttribute => write!(f, "Cannot remove attributes from a schema"),
+            EmptyArray => write!(f, "empty array in facet filter is unspecified behavior"),
+            ParsingError(msg) => write!(f, "parsing error: {}", msg),
+            UnexpectedToken { expected, found } => write!(f, "unexpected token {}, expected {}", found, expected.join("or")),
+            InvalidFormat(found) => write!(f, "invalid facet: {}, facets should be \"facetName:facetValue\"", found),
+            AttributeNotFound(attr) => write!(f, "unknown {:?} attribute", attr),
+            AttributeNotSet { found, expected } => write!(f, "`{}` is not set as a faceted attribute. available facet attributes: {}", found, expected.join(", ")),
+            InvalidDocumentAttribute(attr) => write!(f, "invalid document attribute {}, accepted types: String and [String]", attr),
+            NoAttributesForFaceting => write!(f, "impossible to perform faceted search, no attributes for faceting are set"),
         }
     }
 }

meilisearch-core/src/facets.rs

@@ -0,0 +1,355 @@
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::hash::Hash;
+use std::ops::Deref;
+
+use cow_utils::CowUtils;
+use either::Either;
+use heed::types::{Str, OwnedType};
+use indexmap::IndexMap;
+use serde_json::Value;
+
+use meilisearch_schema::{FieldId, Schema};
+use meilisearch_types::DocumentId;
+
+use crate::database::MainT;
+use crate::error::{FacetError, MResult};
+use crate::store::BEU16;
+
+/// Data structure used to represent a boolean expression in the form of nested arrays.
+/// Values in the outer array are and-ed together, values in the inner arrays are or-ed together.
+#[derive(Debug, PartialEq)]
+pub struct FacetFilter(Vec<Either<Vec<FacetKey>, FacetKey>>);
+
+impl Deref for FacetFilter {
+    type Target = Vec<Either<Vec<FacetKey>, FacetKey>>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl FacetFilter {
+    pub fn from_str(
+        s: &str,
+        schema: &Schema,
+        attributes_for_faceting: &[FieldId],
+    ) -> MResult<FacetFilter> {
+        if attributes_for_faceting.is_empty() {
+            return Err(FacetError::NoAttributesForFaceting.into());
+        }
+        let parsed = serde_json::from_str::<Value>(s).map_err(|e| FacetError::ParsingError(e.to_string()))?;
+        let mut filter = Vec::new();
+        match parsed {
+            Value::Array(and_exprs) => {
+                if and_exprs.is_empty() {
+                    return Err(FacetError::EmptyArray.into());
+                }
+                for expr in and_exprs {
+                    match expr {
+                        Value::String(s) => {
+                            let key = FacetKey::from_str( &s, schema, attributes_for_faceting)?;
+                            filter.push(Either::Right(key));
+                        }
+                        Value::Array(or_exprs) => {
+                            if or_exprs.is_empty() {
+                                return Err(FacetError::EmptyArray.into());
+                            }
+                            let mut inner = Vec::new();
+                            for expr in or_exprs {
+                                match expr {
+                                    Value::String(s) => {
+                                        let key = FacetKey::from_str( &s, schema, attributes_for_faceting)?;
+                                        inner.push(key);
+                                    }
+                                    bad_value => return Err(FacetError::unexpected_token(&["String"], bad_value).into()),
+                                }
+                            }
+                            filter.push(Either::Left(inner));
+                        }
+                        bad_value => return Err(FacetError::unexpected_token(&["Array", "String"], bad_value).into()),
+                    }
+                }
+                Ok(Self(filter))
+            }
+            bad_value => Err(FacetError::unexpected_token(&["Array"], bad_value).into()),
+        }
+    }
+}
+
+#[derive(Debug, Eq, PartialEq, Hash)]
+#[repr(C)]
+pub struct FacetKey(FieldId, String);
+
+impl FacetKey {
+    pub fn new(field_id: FieldId, value: String) -> Self {
+        let value = match value.cow_to_lowercase() {
+                Cow::Borrowed(_) => value,
+                Cow::Owned(s) => s,
+        };
+        Self(field_id, value)
+    }
+
+    pub fn key(&self) -> FieldId {
+        self.0
+    }
+
+    pub fn value(&self) -> &str {
+        &self.1
+    }
+
+    // TODO improve parser
+    fn from_str(
+        s: &str,
+        schema: &Schema,
+        attributes_for_faceting: &[FieldId],
+    ) -> Result<Self, FacetError> {
+        let mut split = s.splitn(2, ':');
+        let key = split
+            .next()
+            .ok_or_else(|| FacetError::InvalidFormat(s.to_string()))?
+            .trim();
+        let field_id = schema
+            .id(key)
+            .ok_or_else(|| FacetError::AttributeNotFound(key.to_string()))?;
+
+        if !attributes_for_faceting.contains(&field_id) {
+            return Err(FacetError::attribute_not_set(
+                    attributes_for_faceting
+                    .iter()
+                    .filter_map(|&id| schema.name(id))
+                    .map(str::to_string)
+                    .collect::<Vec<_>>(),
+                    key))
+        }
+        let value = split
+            .next()
+            .ok_or_else(|| FacetError::InvalidFormat(s.to_string()))?
+            .trim();
+        // unquoting the string if need be:
+        let mut indices = value.char_indices();
+        let value =  match (indices.next(), indices.last()) {
+            (Some((s, '\'')), Some((e, '\''))) |
+            (Some((s, '\"')), Some((e, '\"'))) => value[s + 1..e].to_string(),
+            _ => value.to_string(),
+        };
+        Ok(Self::new(field_id, value))
+    }
+}
+
+impl<'a> heed::BytesEncode<'a> for FacetKey {
+    type EItem = FacetKey;
+
+    fn bytes_encode(item: &'a Self::EItem) -> Option<Cow<'a, [u8]>> {
+        let mut buffer = Vec::with_capacity(2 + item.1.len());
+        let id = BEU16::new(item.key().into());
+        let id_bytes = OwnedType::bytes_encode(&id)?;
+        let value_bytes = Str::bytes_encode(item.value())?;
+        buffer.extend_from_slice(id_bytes.as_ref());
+        buffer.extend_from_slice(value_bytes.as_ref());
+        Some(Cow::Owned(buffer))
+    }
+}
+
+impl<'a> heed::BytesDecode<'a> for FacetKey {
+    type DItem = FacetKey;
+
+    fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
+        let (id_bytes, value_bytes) = bytes.split_at(2);
+        let id = OwnedType::<BEU16>::bytes_decode(id_bytes)?;
+        let id = id.get().into();
+        let string = Str::bytes_decode(&value_bytes)?;
+        Some(FacetKey(id, string.to_string()))
+    }
+}
+
+pub fn add_to_facet_map(
+    facet_map: &mut HashMap<FacetKey, Vec<DocumentId>>,
+    field_id: FieldId,
+    value: Value,
+    document_id: DocumentId,
+) -> Result<(), FacetError> {
+    let value = match value {
+        Value::String(s) => s,
+        // ignore null
+        Value::Null => return Ok(()),
+        value => return Err(FacetError::InvalidDocumentAttribute(value.to_string())),
+    };
+    let key = FacetKey::new(field_id, value);
+    facet_map.entry(key).or_insert_with(Vec::new).push(document_id);
+    Ok(())
+}
+
+pub fn facet_map_from_docids(
+    rtxn: &heed::RoTxn<MainT>,
+    index: &crate::Index,
+    document_ids: &[DocumentId],
+    attributes_for_facetting: &[FieldId],
+) -> MResult<HashMap<FacetKey, Vec<DocumentId>>> {
+    let mut facet_map = HashMap::new();
+    for document_id in document_ids {
+        for result in index
+            .documents_fields
+            .document_fields(rtxn, *document_id)?
+        {
+            let (field_id, bytes) = result?;
+            if attributes_for_facetting.contains(&field_id) {
+                match serde_json::from_slice(bytes)? {
+                    Value::Array(values) => {
+                        for v in values {
+                            add_to_facet_map(&mut facet_map, field_id, v, *document_id)?;
+                        }
+                    }
+                    v => add_to_facet_map(&mut facet_map, field_id, v, *document_id)?,
+                };
+            }
+        }
+    }
+    Ok(facet_map)
+}
+
+pub fn facet_map_from_docs(
+    schema: &Schema,
+    documents: &HashMap<DocumentId, IndexMap<String, Value>>,
+    attributes_for_facetting: &[FieldId],
+) -> MResult<HashMap<FacetKey, Vec<DocumentId>>> {
+    let mut facet_map = HashMap::new();
+    let attributes_for_facetting = attributes_for_facetting
+        .iter()
+        .filter_map(|&id| schema.name(id).map(|name| (id, name)))
+        .collect::<Vec<_>>();
+
+    for (id, document) in documents {
+        for (field_id, name) in &attributes_for_facetting {
+            if let Some(value) = document.get(*name) {
+                match value {
+                    Value::Array(values) => {
+                        for v in values {
+                            add_to_facet_map(&mut facet_map, *field_id, v.clone(), *id)?;
+                        }
+                    }
+                    v => add_to_facet_map(&mut facet_map, *field_id, v.clone(), *id)?,
+                }
+            }
+        }
+    }
+    Ok(facet_map)
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use meilisearch_schema::Schema;
+
+    #[test]
+    fn test_facet_key() {
+        let mut schema = Schema::new();
+        let id = schema.insert_and_index("hello").unwrap();
+        let facet_list = [schema.id("hello").unwrap()];
+        assert_eq!(
+            FacetKey::from_str("hello:12", &schema, &facet_list).unwrap(),
+            FacetKey::new(id, "12".to_string())
+        );
+        assert_eq!(
+            FacetKey::from_str("hello:\"foo bar\"", &schema, &facet_list).unwrap(),
+            FacetKey::new(id, "foo bar".to_string())
+        );
+        assert_eq!(
+            FacetKey::from_str("hello:'foo bar'", &schema, &facet_list).unwrap(),
+            FacetKey::new(id, "foo bar".to_string())
+        );
+        // weird case
+        assert_eq!(
+            FacetKey::from_str("hello:blabla:machin", &schema, &facet_list).unwrap(),
+            FacetKey::new(id, "blabla:machin".to_string())
+        );
+
+        assert_eq!(
+            FacetKey::from_str("hello:\"\"", &schema, &facet_list).unwrap(),
+            FacetKey::new(id, "".to_string())
+        );
+
+        assert_eq!(
+            FacetKey::from_str("hello:'", &schema, &facet_list).unwrap(),
+            FacetKey::new(id, "'".to_string())
+        );
+        assert_eq!(
+            FacetKey::from_str("hello:''", &schema, &facet_list).unwrap(),
+            FacetKey::new(id, "".to_string())
+        );
+        assert!(FacetKey::from_str("hello", &schema, &facet_list).is_err());
+        assert!(FacetKey::from_str("toto:12", &schema, &facet_list).is_err());
+    }
+
+    #[test]
+    fn test_parse_facet_array() {
+        use either::Either::{Left, Right};
+        let mut schema = Schema::new();
+        let _id = schema.insert_and_index("hello").unwrap();
+        let facet_list = [schema.id("hello").unwrap()];
+        assert_eq!(
+            FacetFilter::from_str("[[\"hello:12\"]]", &schema, &facet_list).unwrap(),
+            FacetFilter(vec![Left(vec![FacetKey(FieldId(0), "12".to_string())])])
+        );
+        assert_eq!(
+            FacetFilter::from_str("[\"hello:12\"]", &schema, &facet_list).unwrap(),
+            FacetFilter(vec![Right(FacetKey(FieldId(0), "12".to_string()))])
+        );
+        assert_eq!(
+            FacetFilter::from_str("[\"hello:12\", \"hello:13\"]", &schema, &facet_list).unwrap(),
+            FacetFilter(vec![
+                Right(FacetKey(FieldId(0), "12".to_string())),
+                Right(FacetKey(FieldId(0), "13".to_string()))
+            ])
+        );
+        assert_eq!(
+            FacetFilter::from_str("[[\"hello:12\", \"hello:13\"]]", &schema, &facet_list).unwrap(),
+            FacetFilter(vec![Left(vec![
+                FacetKey(FieldId(0), "12".to_string()),
+                FacetKey(FieldId(0), "13".to_string())
+            ])])
+        );
+        assert_eq!(
+            FacetFilter::from_str(
+                "[[\"hello:12\", \"hello:13\"], \"hello:14\"]",
+                &schema,
+                &facet_list
+            )
+            .unwrap(),
+            FacetFilter(vec![
+                Left(vec![
+                    FacetKey(FieldId(0), "12".to_string()),
+                    FacetKey(FieldId(0), "13".to_string())
+                ]),
+                Right(FacetKey(FieldId(0), "14".to_string()))
+            ])
+        );
+
+        // invalid array depths
+        assert!(FacetFilter::from_str(
+            "[[[\"hello:12\", \"hello:13\"], \"hello:14\"]]",
+            &schema,
+            &facet_list
+        )
+        .is_err());
+        assert!(FacetFilter::from_str(
+            "[[[\"hello:12\", \"hello:13\"]], \"hello:14\"]]",
+            &schema,
+            &facet_list
+        )
+        .is_err());
+        assert!(FacetFilter::from_str("\"hello:14\"", &schema, &facet_list).is_err());
+
+        // unexisting key
+        assert!(FacetFilter::from_str("[\"foo:12\"]", &schema, &facet_list).is_err());
+
+        // invalid facet key
+        assert!(FacetFilter::from_str("[\"foo=12\"]", &schema, &facet_list).is_err());
+        assert!(FacetFilter::from_str("[\"foo12\"]", &schema, &facet_list).is_err());
+        assert!(FacetFilter::from_str("[\"\"]", &schema, &facet_list).is_err());
+
+        // empty array error
+        assert!(FacetFilter::from_str("[]", &schema, &facet_list).is_err());
+        assert!(FacetFilter::from_str("[\"hello:12\", []]", &schema, &facet_list).is_err());
+    }
+}

meilisearch-core/src/filters/condition.rs

@@ -10,7 +10,7 @@ use pest::iterators::Pair;
 use serde_json::{Value, Number};
 use super::parser::Rule;
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
 enum ConditionType {
     Greater,
     Less,
@@ -31,7 +31,7 @@ struct ConditionValue<'a> {
 
 impl<'a> ConditionValue<'a> {
     pub fn new(value: &Pair<'a, Rule>) -> Self {
-        let value = match value.as_rule() {
+        match value.as_rule() {
             Rule::string | Rule::word => {
                 let string =  value.as_str();
                 let boolean = match value.as_str() {
@@ -43,12 +43,11 @@ impl<'a> ConditionValue<'a> {
                 ConditionValue { string, boolean, number }
             },
             _ => unreachable!(),
-        };
-        value
+        }
     }
 
     pub fn as_str(&self) -> &str {
-        self.string.as_ref()
+        self.string
     }
 
     pub fn as_number(&self) -> Option<&Number> {
@@ -73,7 +72,7 @@ fn get_field_value<'a>(schema: &Schema, pair: Pair<'a, Rule>) -> Result<(FieldId
     let key = items.next().unwrap();
     let field = schema
         .id(key.as_str())
-        .ok_or::<PestError<Rule>>(PestError::new_from_span(
+        .ok_or_else(|| PestError::new_from_span(
                 ErrorVariant::CustomError {
                     message: format!(
                                  "attribute `{}` not found, available attributes are: {}",
@@ -160,12 +159,19 @@ impl<'a> Condition<'a> {
         document_id: DocumentId,
     ) -> Result<bool, Error> {
         match index.document_attribute::<Value>(reader, document_id, self.field)? {
+            Some(Value::Array(values)) => Ok(values.iter().any(|v| self.match_value(Some(v)))),
+            other => Ok(self.match_value(other.as_ref())),
+        }
+    }
+
+    fn match_value(&self, value: Option<&Value>) -> bool {
+        match value {
             Some(Value::String(s)) => {
                 let value = self.value.as_str();
                 match self.condition {
-                    ConditionType::Equal => Ok(unicase::eq(value, &s)),
-                    ConditionType::NotEqual => Ok(!unicase::eq(value, &s)),
-                    _ => Ok(false)
+                    ConditionType::Equal => unicase::eq(value, &s),
+                    ConditionType::NotEqual => !unicase::eq(value, &s),
+                    _ => false
                 }
             },
             Some(Value::Number(n)) => { 
@@ -179,22 +185,25 @@ impl<'a> Condition<'a> {
                             ConditionType::Greater => ord == Ordering::Greater,
                             ConditionType::Less => ord == Ordering::Less,
                         };
-                        return Ok(res)
+                        return res
                     } 
                 } 
-                Ok(false)
+                false
             },
             Some(Value::Bool(b)) => {
                 if let Some(value) = self.value.as_bool() {
-                    return match self.condition {
-                        ConditionType::Equal => Ok(b == value),
-                        ConditionType::NotEqual => Ok(b != value),
-                        _ => Ok(false)
-                    }
+                    let res = match self.condition {
+                        ConditionType::Equal => *b == value,
+                        ConditionType::NotEqual => *b != value,
+                        _ => false
+                    };
+                    return res
                 }
-                Ok(false)
+                false
             },
-            _ => Ok(false),
+            // if field is not supported (or not found), all values are different from it,
+            // so != should always return true in this case.
+            _ => self.condition == ConditionType::NotEqual,
         }
     }
 }

meilisearch-core/src/filters/mod.rs

@@ -26,7 +26,7 @@ pub enum Filter<'a> {
 
 impl<'a> Filter<'a> {
     pub fn parse(expr: &'a str, schema: &'a Schema) -> FilterResult<'a> {
-        let mut lexed = FilterParser::parse(Rule::prgm, expr.as_ref())?;
+        let mut lexed = FilterParser::parse(Rule::prgm, expr)?;
         Self::build(lexed.next().unwrap().into_inner(), schema)
     }
 
@@ -60,6 +60,7 @@ impl<'a> Filter<'a> {
                 Rule::geq => Ok(Filter::Condition(Condition::geq(pair, schema)?)),
                 Rule::leq => Ok(Filter::Condition(Condition::leq(pair, schema)?)),
                 Rule::prgm => Self::build(pair.into_inner(), schema),
+                Rule::term => Self::build(pair.into_inner(), schema),
                 Rule::not => Ok(Filter::Not(Box::new(Self::build(
                     pair.into_inner(),
                     schema,
@@ -109,6 +110,7 @@ mod test {
         assert!(FilterParser::parse(Rule::prgm, r#"field > 10"#).is_ok());
         assert!(FilterParser::parse(Rule::prgm, r#"field < 10"#).is_ok());
         assert!(FilterParser::parse(Rule::prgm, r#"field < 10 AND NOT field=5"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field < 10 AND NOT field > 7.5"#).is_ok());
         assert!(FilterParser::parse(Rule::prgm, r#"field=true OR NOT field=5"#).is_ok());
         assert!(FilterParser::parse(Rule::prgm, r#"NOT field=true OR NOT field=5"#).is_ok());
         assert!(FilterParser::parse(Rule::prgm, r#"field='hello world' OR ( NOT field=true OR NOT field=5 )"#).is_ok());

meilisearch-core/src/filters/parser/grammar.pest

@@ -2,7 +2,7 @@ key = _{quoted | word}
 value = _{quoted | word}
 quoted = _{ (PUSH("'") | PUSH("\"")) ~ string ~ POP  }
 string = {char*}
-word = ${(LETTER | NUMBER | "_" | "-")+}
+word = ${(LETTER | NUMBER | "_" | "-" | ".")+}
 
 char =  _{ !(PEEK | "\\") ~ ANY
     | "\\" ~ (PEEK | "\\" | "/" | "b" | "f" | "n" | "r" | "t")
@@ -18,7 +18,7 @@ less = {key ~ "<" ~ value}
 
 prgm = {SOI ~ expr ~ EOI}
 expr = _{ ( term ~ (operation ~ term)* ) }
-term = _{ ("(" ~ expr ~ ")") | condition | not }
+term = { ("(" ~ expr ~ ")") | condition | not }
 operation = _{ and | or }
 	and = {"AND"}
 	or = {"OR"}

meilisearch-core/src/lib.rs

@@ -1,3 +1,5 @@
+#![allow(clippy::type_complexity)]
+
 #[cfg(test)]
 #[macro_use]
 extern crate assert_matches;
@@ -18,15 +20,16 @@ mod query_words_mapper;
 mod ranked_map;
 mod raw_document;
 mod reordered_attrs;
-mod update;
-pub mod settings;
 pub mod criterion;
+pub mod facets;
 pub mod raw_indexer;
 pub mod serde;
+pub mod settings;
 pub mod store;
+pub mod update;
 
-pub use self::database::{BoxUpdateFn, Database, MainT, UpdateT};
-pub use self::error::{Error, HeedError, FstError, MResult, pest_error};
+pub use self::database::{BoxUpdateFn, Database, DatabaseOptions, MainT, UpdateT, MainWriter, MainReader, UpdateWriter, UpdateReader};
+pub use self::error::{Error, HeedError, FstError, MResult, pest_error, FacetError};
 pub use self::filters::Filter;
 pub use self::number::{Number, ParseNumberError};
 pub use self::ranked_map::RankedMap;
@@ -37,16 +40,20 @@ pub use meilisearch_types::{DocIndex, DocumentId, Highlight};
 pub use meilisearch_schema::Schema;
 pub use query_words_mapper::QueryWordsMapper;
 
-use std::convert::TryFrom;
-use std::collections::HashMap;
 use compact_arena::SmallArena;
 use log::{error, trace};
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::convert::TryFrom;
 
 use crate::bucket_sort::PostingsListView;
 use crate::levenshtein::prefix_damerau_levenshtein;
 use crate::query_tree::{QueryId, QueryKind};
 use crate::reordered_attrs::ReorderedAttrs;
 
+type FstSetCow<'a> = fst::Set<Cow<'a, [u8]>>;
+type FstMapCow<'a> = fst::Map<Cow<'a, [u8]>>;
+
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
 pub struct Document {
     pub id: DocumentId,
@@ -190,6 +197,6 @@ mod tests {
 
     #[test]
     fn docindex_mem_size() {
-        assert_eq!(mem::size_of::<DocIndex>(), 16);
+        assert_eq!(mem::size_of::<DocIndex>(), 12);
     }
 }

meilisearch-core/src/number.rs

@@ -6,7 +6,7 @@ use std::str::FromStr;
 use ordered_float::OrderedFloat;
 use serde::{Deserialize, Serialize};
 
-#[derive(Serialize, Deserialize, Debug, Copy, Clone, Hash)]
+#[derive(Serialize, Deserialize, Debug, Copy, Clone)]
 pub enum Number {
     Unsigned(u64),
     Signed(i64),

meilisearch-core/src/query_builder.rs

@@ -1,66 +1,57 @@
-use std::ops::Range;
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::ops::{Deref, Range};
 use std::time::Duration;
 
-use crate::database::MainT;
-use crate::bucket_sort::{bucket_sort, bucket_sort_with_distinct};
-use crate::{criterion::Criteria, Document, DocumentId};
-use crate::{reordered_attrs::ReorderedAttrs, store, MResult};
+use either::Either;
+use sdset::{SetOperation, SetBuf, Set};
 
-pub struct QueryBuilder<'c, 'f, 'd> {
+use meilisearch_schema::FieldId;
+
+use crate::bucket_sort::{bucket_sort, bucket_sort_with_distinct, SortResult, placeholder_document_sort, facet_count};
+use crate::database::MainT;
+use crate::facets::FacetFilter;
+use crate::distinct_map::{DistinctMap, BufferedDistinctMap};
+use crate::Document;
+use crate::{criterion::Criteria, DocumentId};
+use crate::{reordered_attrs::ReorderedAttrs, store, MResult, MainReader};
+
+pub struct QueryBuilder<'c, 'f, 'd, 'i> {
     criteria: Criteria<'c>,
     searchable_attrs: Option<ReorderedAttrs>,
     filter: Option<Box<dyn Fn(DocumentId) -> bool + 'f>>,
     distinct: Option<(Box<dyn Fn(DocumentId) -> Option<u64> + 'd>, usize)>,
     timeout: Option<Duration>,
-    main_store: store::Main,
-    postings_lists_store: store::PostingsLists,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    synonyms_store: store::Synonyms,
-    prefix_documents_cache_store: store::PrefixDocumentsCache,
-    prefix_postings_lists_cache_store: store::PrefixPostingsListsCache,
+    index: &'i store::Index,
+    facet_filter: Option<FacetFilter>,
+    facets: Option<Vec<(FieldId, String)>>,
 }
 
-impl<'c, 'f, 'd> QueryBuilder<'c, 'f, 'd> {
-    pub fn new(
-        main: store::Main,
-        postings_lists: store::PostingsLists,
-        documents_fields_counts: store::DocumentsFieldsCounts,
-        synonyms: store::Synonyms,
-        prefix_documents_cache: store::PrefixDocumentsCache,
-        prefix_postings_lists_cache: store::PrefixPostingsListsCache,
-    ) -> QueryBuilder<'c, 'f, 'd> {
-        QueryBuilder::with_criteria(
-            main,
-            postings_lists,
-            documents_fields_counts,
-            synonyms,
-            prefix_documents_cache,
-            prefix_postings_lists_cache,
-            Criteria::default(),
-        )
+impl<'c, 'f, 'd, 'i> QueryBuilder<'c, 'f, 'd, 'i> {
+    pub fn new(index: &'i store::Index) -> Self {
+        QueryBuilder::with_criteria(index, Criteria::default())
     }
 
-    pub fn with_criteria(
-        main: store::Main,
-        postings_lists: store::PostingsLists,
-        documents_fields_counts: store::DocumentsFieldsCounts,
-        synonyms: store::Synonyms,
-        prefix_documents_cache: store::PrefixDocumentsCache,
-        prefix_postings_lists_cache: store::PrefixPostingsListsCache,
-        criteria: Criteria<'c>,
-    ) -> QueryBuilder<'c, 'f, 'd> {
+    /// sets facet attributes to filter on
+    pub fn set_facet_filter(&mut self, facets: Option<FacetFilter>) {
+        self.facet_filter = facets;
+    }
+
+    /// sets facet attributes for which to return the count
+    pub fn set_facets(&mut self, facets: Option<Vec<(FieldId, String)>>) {
+        self.facets = facets;
+    }
+
+    pub fn with_criteria(index: &'i store::Index, criteria: Criteria<'c>) -> Self {
         QueryBuilder {
             criteria,
             searchable_attrs: None,
             filter: None,
             distinct: None,
             timeout: None,
-            main_store: main,
-            postings_lists_store: postings_lists,
-            documents_fields_counts_store: documents_fields_counts,
-            synonyms_store: synonyms,
-            prefix_documents_cache_store: prefix_documents_cache,
-            prefix_postings_lists_cache_store: prefix_postings_lists_cache,
+            index,
+            facet_filter: None,
+            facets: None,
         }
     }
 
@@ -87,45 +78,201 @@ impl<'c, 'f, 'd> QueryBuilder<'c, 'f, 'd> {
         reorders.insert_attribute(attribute);
     }
 
-    pub fn query(
-        self,
-        reader: &heed::RoTxn<MainT>,
-        query: &str,
-        range: Range<usize>,
-    ) -> MResult<(Vec<Document>, usize)> {
+    /// returns the documents ids associated with a facet filter by computing the union and
+    /// intersection of the document sets
+    fn facets_docids(&self, reader: &MainReader) -> MResult<Option<SetBuf<DocumentId>>> {
+        let facet_docids = match self.facet_filter {
+            Some(ref facets) => {
+                let mut ands = Vec::with_capacity(facets.len());
+                let mut ors = Vec::new();
+                for f in facets.deref() {
+                    match f {
+                        Either::Left(keys) => {
+                            ors.reserve(keys.len());
+                            for key in keys {
+                                let docids = self
+                                    .index
+                                    .facets
+                                    .facet_document_ids(reader, &key)?
+                                    .unwrap_or_default();
+                                ors.push(docids);
+                            }
+                            let sets: Vec<_> = ors.iter().map(Cow::deref).collect();
+                            let or_result = sdset::multi::OpBuilder::from_vec(sets)
+                                .union()
+                                .into_set_buf();
+                            ands.push(Cow::Owned(or_result));
+                            ors.clear();
+                        }
+                        Either::Right(key) => {
+                            match self.index.facets.facet_document_ids(reader, &key)? {
+                                Some(docids) => ands.push(docids),
+                                // no candidates for search, early return.
+                                None => return Ok(Some(SetBuf::default())),
+                            }
+                        }
+                    };
+                }
+                let ands: Vec<_> = ands.iter().map(Cow::deref).collect();
+                Some(
+                    sdset::multi::OpBuilder::from_vec(ands)
+                    .intersection()
+                    .into_set_buf(),
+                )
+            }
+            None => None,
+        };
+        Ok(facet_docids)
+    }
+
+    fn standard_query(self, reader: &MainReader, query: &str, range: Range<usize>) -> MResult<SortResult> {
+        let facets_docids = match self.facets_docids(reader)? {
+            Some(ids) if ids.is_empty() => return Ok(SortResult::default()),
+            other => other
+        };
+        // for each field to retrieve the count for, create an HashMap associating the attribute
+        // value to a set of matching documents. The HashMaps are them collected in another
+        // HashMap, associating each HashMap to it's field.
+        let facet_count_docids = self.facet_count_docids(reader)?;
+
         match self.distinct {
             Some((distinct, distinct_size)) => bucket_sort_with_distinct(
                 reader,
                 query,
                 range,
+                facets_docids,
+                facet_count_docids,
                 self.filter,
                 distinct,
                 distinct_size,
                 self.criteria,
                 self.searchable_attrs,
-                self.main_store,
-                self.postings_lists_store,
-                self.documents_fields_counts_store,
-                self.synonyms_store,
-                self.prefix_documents_cache_store,
-                self.prefix_postings_lists_cache_store,
+                self.index,
             ),
             None => bucket_sort(
                 reader,
                 query,
                 range,
+                facets_docids,
+                facet_count_docids,
                 self.filter,
                 self.criteria,
                 self.searchable_attrs,
-                self.main_store,
-                self.postings_lists_store,
-                self.documents_fields_counts_store,
-                self.synonyms_store,
-                self.prefix_documents_cache_store,
-                self.prefix_postings_lists_cache_store,
+                self.index,
             ),
         }
     }
+
+    fn placeholder_query(self, reader: &heed::RoTxn<MainT>, range: Range<usize>) -> MResult<SortResult> {
+        match self.facets_docids(reader)? {
+            Some(docids) => {
+                // We sort the docids from facets according to the criteria set by the user
+                let mut sorted_docids = docids.clone().into_vec();
+                let mut sort_result = match self.index.main.ranked_map(reader)? {
+                    Some(ranked_map) => {
+                        placeholder_document_sort(&mut sorted_docids, self.index, reader, &ranked_map)?;
+                        self.sort_result_from_docids(&sorted_docids, range)
+                    },
+                    // if we can't perform a sort, we return documents unordered
+                    None => self.sort_result_from_docids(&docids, range),
+                };
+
+                if let Some(f) = self.facet_count_docids(reader)? {
+                    sort_result.exhaustive_facets_count = Some(true);
+                    sort_result.facets = Some(facet_count(f, &docids));
+                }
+
+                Ok(sort_result)
+            },
+            None => {
+                match self.index.main.sorted_document_ids_cache(reader)? {
+                    // build result from cached document ids
+                    Some(docids) => {
+                        let mut sort_result = self.sort_result_from_docids(&docids, range);
+
+                        if let Some(f) = self.facet_count_docids(reader)? {
+                            sort_result.exhaustive_facets_count = Some(true);
+                            // document ids are not sorted in natural order, we need to construct a new set
+                            let document_set = SetBuf::from_dirty(Vec::from(docids));
+                            sort_result.facets = Some(facet_count(f, &document_set));
+                        }
+
+                        Ok(sort_result)
+                    },
+                    // no document id cached, return empty result
+                    None => Ok(SortResult::default()),
+                }
+            }
+        }
+    }
+
+    fn facet_count_docids<'a>(&self, reader: &'a MainReader) -> MResult<Option<HashMap<String, HashMap<String, Cow<'a, Set<DocumentId>>>>>> {
+        match self.facets {
+            Some(ref field_ids) => {
+                let mut facet_count_map = HashMap::new();
+                for (field_id, field_name) in field_ids {
+                    let mut key_map = HashMap::new();
+                    for pair in self.index.facets.field_document_ids(reader, *field_id)? {
+                        let (facet_key, document_ids) = pair?;
+                        let value = facet_key.value();
+                        key_map.insert(value.to_string(), document_ids);
+                    }
+                    facet_count_map.insert(field_name.clone(), key_map);
+                }
+                Ok(Some(facet_count_map))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn sort_result_from_docids(&self, docids: &[DocumentId], range: Range<usize>) -> SortResult {
+        let mut sort_result = SortResult::default();
+        let mut result = match self.filter {
+            Some(ref filter) => docids
+                .iter()
+                .filter(|item| (filter)(**item))
+                .skip(range.start)
+                .take(range.end - range.start)
+                .map(|&id| Document::from_highlights(id, &[]))
+                .collect::<Vec<_>>(),
+            None => docids
+                .iter()
+                .skip(range.start)
+                .take(range.end - range.start)
+                .map(|&id| Document::from_highlights(id, &[]))
+                .collect::<Vec<_>>(),
+        };
+
+        // distinct is set, remove duplicates with disctinct function
+        if let Some((distinct, distinct_size)) = &self.distinct {
+            let mut distinct_map = DistinctMap::new(*distinct_size);
+            let mut distinct_map = BufferedDistinctMap::new(&mut distinct_map);
+            result.retain(|doc| {
+                let id = doc.id;
+                let key = (distinct)(id);
+                match key {
+                    Some(key) => distinct_map.register(key),
+                    None => distinct_map.register_without_key(),
+                }
+            });
+        }
+
+        sort_result.documents = result;
+        sort_result.nb_hits = docids.len();
+        sort_result
+    }
+
+    pub fn query(
+        self,
+        reader: &heed::RoTxn<MainT>,
+        query: Option<&str>,
+        range: Range<usize>,
+    ) -> MResult<SortResult> {
+        match query {
+            Some(query) => self.standard_query(reader, query, range),
+            None => self.placeholder_query(reader, range),
+        }
+    }
 }
 
 #[cfg(test)]
@@ -135,33 +282,34 @@ mod tests {
     use std::collections::{BTreeSet, HashMap};
     use std::iter::FromIterator;
 
-    use fst::{IntoStreamer, Set};
+    use fst::IntoStreamer;
     use meilisearch_schema::IndexedPos;
     use sdset::SetBuf;
     use tempfile::TempDir;
 
-    use crate::DocIndex;
     use crate::automaton::normalize_str;
     use crate::bucket_sort::SimpleMatch;
-    use crate::database::Database;
+    use crate::database::{Database, DatabaseOptions};
     use crate::store::Index;
+    use crate::DocIndex;
+    use crate::Document;
     use meilisearch_schema::Schema;
 
-    fn set_from_stream<'f, I, S>(stream: I) -> Set
+    fn set_from_stream<'f, I, S>(stream: I) -> fst::Set<Vec<u8>>
     where
         I: for<'a> fst::IntoStreamer<'a, Into = S, Item = &'a [u8]>,
         S: 'f + for<'a> fst::Streamer<'a, Item = &'a [u8]>,
     {
         let mut builder = fst::SetBuilder::memory();
         builder.extend_stream(stream).unwrap();
-        builder.into_inner().and_then(Set::from_bytes).unwrap()
+        builder.into_set()
     }
 
-    fn insert_key(set: &Set, key: &[u8]) -> Set {
+    fn insert_key<A: AsRef<[u8]>>(set: &fst::Set<A>, key: &[u8]) -> fst::Set<Vec<u8>> {
         let unique_key = {
             let mut builder = fst::SetBuilder::memory();
             builder.insert(key).unwrap();
-            builder.into_inner().and_then(Set::from_bytes).unwrap()
+            builder.into_set()
         };
 
         let union_ = set.op().add(unique_key.into_stream()).r#union();
@@ -169,14 +317,14 @@ mod tests {
         set_from_stream(union_)
     }
 
-    fn sdset_into_fstset(set: &sdset::Set<&str>) -> Set {
+    fn sdset_into_fstset(set: &sdset::Set<&str>) -> fst::Set<Vec<u8>> {
         let mut builder = fst::SetBuilder::memory();
         let set = SetBuf::from_dirty(set.into_iter().map(|s| normalize_str(s)).collect());
         builder.extend_iter(set.into_iter()).unwrap();
-        builder.into_inner().and_then(Set::from_bytes).unwrap()
+        builder.into_set()
     }
 
-    const fn doc_index(document_id: u64, word_index: u16) -> DocIndex {
+    const fn doc_index(document_id: u32, word_index: u16) -> DocIndex {
         DocIndex {
             document_id: DocumentId(document_id),
             attribute: 0,
@@ -186,7 +334,7 @@ mod tests {
         }
     }
 
-    const fn doc_char_index(document_id: u64, word_index: u16, char_index: u16) -> DocIndex {
+    const fn doc_char_index(document_id: u32, word_index: u16, char_index: u16) -> DocIndex {
         DocIndex {
             document_id: DocumentId(document_id),
             attribute: 0,
@@ -213,15 +361,11 @@ mod tests {
 
             let word = normalize_str(word);
 
-            let alternatives = match self
+            let alternatives = self
                 .index
                 .synonyms
-                .synonyms(&writer, word.as_bytes())
-                .unwrap()
-            {
-                Some(alternatives) => alternatives,
-                None => fst::Set::default(),
-            };
+                .synonyms_fst(&writer, word.as_bytes())
+                .unwrap();
 
             let new = sdset_into_fstset(&new);
             let new_alternatives =
@@ -231,10 +375,7 @@ mod tests {
                 .put_synonyms(&mut writer, word.as_bytes(), &new_alternatives)
                 .unwrap();
 
-            let synonyms = match self.index.main.synonyms_fst(&writer).unwrap() {
-                Some(synonyms) => synonyms,
-                None => fst::Set::default(),
-            };
+            let synonyms = self.index.main.synonyms_fst(&writer).unwrap();
 
             let synonyms_fst = insert_key(&synonyms, word.as_bytes());
             self.index
@@ -249,7 +390,7 @@ mod tests {
     impl<'a> FromIterator<(&'a str, &'a [DocIndex])> for TempDatabase {
         fn from_iter<I: IntoIterator<Item = (&'a str, &'a [DocIndex])>>(iter: I) -> Self {
             let tempdir = TempDir::new().unwrap();
-            let database = Database::open_or_create(&tempdir).unwrap();
+            let database = Database::open_or_create(&tempdir, DatabaseOptions::default()).unwrap();
             let index = database.create_index("default").unwrap();
 
             let db = &database;
@@ -287,7 +428,7 @@ mod tests {
 
             index.main.put_schema(&mut writer, &schema).unwrap();
 
-            let words_fst = Set::from_iter(words_fst).unwrap();
+            let words_fst = fst::Set::from_iter(words_fst).unwrap();
 
             index.main.put_words_fst(&mut writer, &words_fst).unwrap();
 
@@ -331,8 +472,8 @@ mod tests {
         let reader = db.main_read_txn().unwrap();
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "iphone from apple", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("iphone from apple"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut matches = matches.into_iter();
@@ -354,8 +495,8 @@ mod tests {
         let reader = db.main_read_txn().unwrap();
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "hello", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("hello"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut matches = matches.into_iter();
@@ -365,8 +506,8 @@ mod tests {
         assert_matches!(iter.next(), None);
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "bonjour", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("bonjour"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut matches = matches.into_iter();
@@ -388,7 +529,7 @@ mod tests {
 
     //     let builder = store.query_builder();
     //     let results = builder.query(&reader, "sal", 0..20).unwrap();
-    //     let mut iter = results.into_iter();
+    //     let mut iter = documents.into_iter();
 
     //     assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
     //         let mut matches = matches.into_iter();
@@ -399,7 +540,7 @@ mod tests {
 
     //     let builder = store.query_builder();
     //     let results = builder.query(&reader, "bonj", 0..20).unwrap();
-    //     let mut iter = results.into_iter();
+    //     let mut iter = documents.into_iter();
 
     //     assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
     //         let mut matches = matches.into_iter();
@@ -410,13 +551,13 @@ mod tests {
 
     //     let builder = store.query_builder();
     //     let results = builder.query(&reader, "sal blabla", 0..20).unwrap();
-    //     let mut iter = results.into_iter();
+    //     let mut iter = documents.into_iter();
 
     //     assert_matches!(iter.next(), None);
 
     //     let builder = store.query_builder();
     //     let results = builder.query(&reader, "bonj blabla", 0..20).unwrap();
-    //     let mut iter = results.into_iter();
+    //     let mut iter = documents.into_iter();
 
     //     assert_matches!(iter.next(), None);
     // }
@@ -432,7 +573,7 @@ mod tests {
 
     //     let builder = store.query_builder();
     //     let results = builder.query(&reader, "salutution", 0..20).unwrap();
-    //     let mut iter = results.into_iter();
+    //     let mut iter = documents.into_iter();
 
     //     assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
     //         let mut matches = matches.into_iter();
@@ -443,7 +584,7 @@ mod tests {
 
     //     let builder = store.query_builder();
     //     let results = builder.query(&reader, "saluttion", 0..20).unwrap();
-    //     let mut iter = results.into_iter();
+    //     let mut iter = documents.into_iter();
 
     //     assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
     //         let mut matches = matches.into_iter();
@@ -469,8 +610,8 @@ mod tests {
         let reader = db.main_read_txn().unwrap();
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "hello", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("hello"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut matches = matches.into_iter();
@@ -490,8 +631,8 @@ mod tests {
         assert_matches!(iter.next(), None);
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "bonjour", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("bonjour"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut matches = matches.into_iter();
@@ -511,8 +652,8 @@ mod tests {
         assert_matches!(iter.next(), None);
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "salut", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("salut"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut matches = matches.into_iter();
@@ -557,8 +698,8 @@ mod tests {
         let reader = db.main_read_txn().unwrap();
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "NY subway", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("NY subway"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => {
             let mut iter = matches.into_iter();
@@ -579,8 +720,8 @@ mod tests {
         assert_matches!(iter.next(), None);
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "NYC subway", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("NYC subway"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => {
             let mut iter = matches.into_iter();
@@ -621,8 +762,8 @@ mod tests {
         let reader = db.main_read_txn().unwrap();
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "NY", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("NY"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(2), matches, .. }) => {
             let mut matches = matches.into_iter();
@@ -645,8 +786,8 @@ mod tests {
         assert_matches!(iter.next(), None);
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "new york", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("new york"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut matches = matches.into_iter();
@@ -679,8 +820,8 @@ mod tests {
         let reader = db.main_read_txn().unwrap();
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "NY subway", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("NY subway"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut matches = matches.into_iter();
@@ -696,8 +837,9 @@ mod tests {
         assert_matches!(iter.next(), None);
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "new york subway", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } =
+            builder.query(&reader, Some("new york subway"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => {
             let mut matches = matches.into_iter();
@@ -744,8 +886,8 @@ mod tests {
         let reader = db.main_read_txn().unwrap();
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "NY subway", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("NY subway"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => {
             let mut iter = matches.into_iter();
@@ -766,8 +908,8 @@ mod tests {
         assert_matches!(iter.next(), None);
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "NYC subway", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("NYC subway"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => {
             let mut iter = matches.into_iter();
@@ -819,8 +961,8 @@ mod tests {
         let reader = db.main_read_txn().unwrap();
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "NY subway broken", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult {documents, .. } = builder.query(&reader, Some("NY subway broken"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut iter = matches.into_iter();
@@ -835,8 +977,8 @@ mod tests {
         assert_matches!(iter.next(), None);
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "NYC subway", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("NYC subway"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => {
             let mut iter = matches.into_iter();
@@ -891,10 +1033,10 @@ mod tests {
         let reader = db.main_read_txn().unwrap();
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder
-            .query(&reader, "new york underground train broken", 0..20)
+        let SortResult { documents, .. } = builder
+            .query(&reader, Some("new york underground train broken"), 0..20)
             .unwrap();
-        let mut iter = results.into_iter();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(2), matches, .. }) => {
             let mut matches = matches.into_iter();
@@ -921,10 +1063,10 @@ mod tests {
         assert_matches!(iter.next(), None);
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder
-            .query(&reader, "new york city underground train broken", 0..20)
+        let SortResult { documents, .. } = builder
+            .query(&reader, Some("new york city underground train broken"), 0..20)
             .unwrap();
-        let mut iter = results.into_iter();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(2), matches, .. }) => {
             let mut matches = matches.into_iter();
@@ -965,8 +1107,8 @@ mod tests {
         let reader = db.main_read_txn().unwrap();
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "new york big ", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("new york big "), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut matches = matches.into_iter();
@@ -999,8 +1141,8 @@ mod tests {
         let reader = db.main_read_txn().unwrap();
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "NY subway ", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("NY subway "), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut matches = matches.into_iter();
@@ -1049,10 +1191,10 @@ mod tests {
         let reader = db.main_read_txn().unwrap();
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder
-            .query(&reader, "new york city long subway cool ", 0..20)
+        let SortResult { documents, .. } = builder
+            .query(&reader, Some("new york city long subway cool "), 0..20)
             .unwrap();
-        let mut iter = results.into_iter();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut matches = matches.into_iter();
@@ -1082,8 +1224,8 @@ mod tests {
         let reader = db.main_read_txn().unwrap();
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "telephone", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("telephone"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut iter = matches.into_iter();
@@ -1099,8 +1241,8 @@ mod tests {
         assert_matches!(iter.next(), None);
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "téléphone", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("téléphone"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut iter = matches.into_iter();
@@ -1116,8 +1258,8 @@ mod tests {
         assert_matches!(iter.next(), None);
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "télephone", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("télephone"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => {
             let mut iter = matches.into_iter();
@@ -1143,8 +1285,8 @@ mod tests {
         let reader = db.main_read_txn().unwrap();
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "i phone case", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("i phone case"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut iter = matches.into_iter();
@@ -1172,8 +1314,8 @@ mod tests {
         let reader = db.main_read_txn().unwrap();
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "searchengine", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("searchengine"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut iter = matches.into_iter();
@@ -1212,8 +1354,8 @@ mod tests {
         let reader = db.main_read_txn().unwrap();
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "searchengine", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("searchengine"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut iter = matches.into_iter();
@@ -1244,8 +1386,8 @@ mod tests {
         let reader = db.main_read_txn().unwrap();
 
         let builder = store.query_builder();
-        let (results, _nb_hits) = builder.query(&reader, "searchengine", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult { documents, .. } = builder.query(&reader, Some("searchengine"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut iter = matches.into_iter();

meilisearch-core/src/query_tree.rs

@@ -12,7 +12,7 @@ use sdset::{Set, SetBuf, SetOperation};
 use log::debug;
 
 use crate::database::MainT;
-use crate::{store, DocumentId, DocIndex, MResult};
+use crate::{store, DocumentId, DocIndex, MResult, FstSetCow};
 use crate::automaton::{normalize_str, build_dfa, build_prefix_dfa, build_exact_dfa};
 use crate::QueryWordsMapper;
 
@@ -112,9 +112,9 @@ pub struct PostingsList {
     matches: SetBuf<DocIndex>,
 }
 
-pub struct Context {
-    pub words_set: fst::Set,
-    pub stop_words: fst::Set,
+pub struct Context<'a> {
+    pub words_set: FstSetCow<'a>,
+    pub stop_words: FstSetCow<'a>,
     pub synonyms: store::Synonyms,
     pub postings_lists: store::PostingsLists,
     pub prefix_postings_lists: store::PrefixPostingsListsCache,
@@ -147,7 +147,7 @@ fn split_best_frequency<'a>(reader: &heed::RoTxn<MainT>, ctx: &Context, word: &'
 
 fn fetch_synonyms(reader: &heed::RoTxn<MainT>, ctx: &Context, words: &[&str]) -> MResult<Vec<Vec<String>>> {
     let words = normalize_str(&words.join(" "));
-    let set = ctx.synonyms.synonyms(reader, words.as_bytes())?.unwrap_or_default();
+    let set = ctx.synonyms.synonyms_fst(reader, words.as_bytes())?;
 
     let mut strings = Vec::new();
     let mut stream = set.stream();

meilisearch-core/src/query_words_mapper.rs

@@ -19,6 +19,7 @@ impl QueryWordsMapper {
         QueryWordsMapper { originals, mappings: HashMap::new() }
     }
 
+    #[allow(clippy::len_zero)]
     pub fn declare<I, A>(&mut self, range: Range<usize>, id: QueryId, replacement: I)
     where I: IntoIterator<Item = A>,
           A: ToString,
@@ -53,7 +54,7 @@ impl QueryWordsMapper {
         }
 
         {
-            let replacement = replacement[common_left..replacement.len() - common_right].iter().cloned().collect();
+            let replacement = replacement[common_left..replacement.len() - common_right].to_vec();
             self.mappings.insert(id + common_left, (range.clone(), replacement));
         }
 

meilisearch-core/src/raw_indexer.rs

@@ -1,34 +1,37 @@
+use std::borrow::Cow;
 use std::collections::{BTreeMap, HashMap};
 use std::convert::TryFrom;
 
-use crate::{DocIndex, DocumentId};
 use deunicode::deunicode_with_tofu;
 use meilisearch_schema::IndexedPos;
 use meilisearch_tokenizer::{is_cjk, SeqTokenizer, Token, Tokenizer};
 use sdset::SetBuf;
 
+use crate::{DocIndex, DocumentId};
+use crate::FstSetCow;
+
 const WORD_LENGTH_LIMIT: usize = 80;
 
 type Word = Vec<u8>; // TODO make it be a SmallVec
 
-pub struct RawIndexer {
+pub struct RawIndexer<A> {
     word_limit: usize, // the maximum number of indexed words
-    stop_words: fst::Set,
+    stop_words: fst::Set<A>,
     words_doc_indexes: BTreeMap<Word, Vec<DocIndex>>,
     docs_words: HashMap<DocumentId, Vec<Word>>,
 }
 
-pub struct Indexed {
+pub struct Indexed<'a> {
     pub words_doc_indexes: BTreeMap<Word, SetBuf<DocIndex>>,
-    pub docs_words: HashMap<DocumentId, fst::Set>,
+    pub docs_words: HashMap<DocumentId, FstSetCow<'a>>,
 }
 
-impl RawIndexer {
-    pub fn new(stop_words: fst::Set) -> RawIndexer {
+impl<A> RawIndexer<A> {
+    pub fn new(stop_words: fst::Set<A>) -> RawIndexer<A> {
         RawIndexer::with_word_limit(stop_words, 1000)
     }
 
-    pub fn with_word_limit(stop_words: fst::Set, limit: usize) -> RawIndexer {
+    pub fn with_word_limit(stop_words: fst::Set<A>, limit: usize) -> RawIndexer<A> {
         RawIndexer {
             word_limit: limit,
             stop_words,
@@ -36,7 +39,9 @@ impl RawIndexer {
             docs_words: HashMap::new(),
         }
     }
+}
 
+impl<A: AsRef<[u8]>> RawIndexer<A> {
     pub fn index_text(&mut self, id: DocumentId, indexed_pos: IndexedPos, text: &str) -> usize {
         let mut number_of_words = 0;
 
@@ -61,9 +66,9 @@ impl RawIndexer {
         number_of_words
     }
 
-    pub fn index_text_seq<'a, I>(&mut self, id: DocumentId, indexed_pos: IndexedPos, iter: I)
+    pub fn index_text_seq<'s, I>(&mut self, id: DocumentId, indexed_pos: IndexedPos, iter: I)
     where
-        I: IntoIterator<Item = &'a str>,
+        I: IntoIterator<Item = &'s str>,
     {
         let iter = iter.into_iter();
         for token in SeqTokenizer::new(iter) {
@@ -83,7 +88,7 @@ impl RawIndexer {
         }
     }
 
-    pub fn build(self) -> Indexed {
+    pub fn build(self) -> Indexed<'static> {
         let words_doc_indexes = self
             .words_doc_indexes
             .into_iter()
@@ -96,7 +101,8 @@ impl RawIndexer {
             .map(|(id, mut words)| {
                 words.sort_unstable();
                 words.dedup();
-                (id, fst::Set::from_iter(words).unwrap())
+                let fst = fst::Set::from_iter(words).unwrap().map_data(Cow::Owned).unwrap();
+                (id, fst)
             })
             .collect();
 
@@ -107,16 +113,18 @@ impl RawIndexer {
     }
 }
 
-fn index_token(
+fn index_token<A>(
     token: Token,
     id: DocumentId,
     indexed_pos: IndexedPos,
     word_limit: usize,
-    stop_words: &fst::Set,
+    stop_words: &fst::Set<A>,
     words_doc_indexes: &mut BTreeMap<Word, Vec<DocIndex>>,
     docs_words: &mut HashMap<DocumentId, Vec<Word>>,
-) -> bool {
-    if token.word_index >= word_limit {
+) -> bool
+where A: AsRef<[u8]>,
+{
+    if token.index >= word_limit {
         return false;
     }
 
@@ -269,4 +277,36 @@ mod tests {
             .get(&"🇯🇵".to_owned().into_bytes())
             .is_some());
     }
+
+    #[test]
+    // test sample from 807
+    fn very_long_text() {
+        let mut indexer = RawIndexer::new(fst::Set::default());
+        let indexed_pos = IndexedPos(0);
+        let docid = DocumentId(0);
+        let text = " The locations block is the most powerful, and potentially most involved, section of the .platform.app.yaml file. It allows you to control how the application container responds to incoming requests at a very fine-grained level. Common patterns also vary between language containers due to the way PHP-FPM handles incoming requests.\nEach entry of the locations block is an absolute URI path (with leading /) and its value includes the configuration directives for how the web server should handle matching requests. That is, if your domain is example.com then '/' means &ldquo;requests for example.com/&rdquo;, while '/admin' means &ldquo;requests for example.com/admin&rdquo;. If multiple blocks could match an incoming request then the most-specific will apply.\nweb:locations:&#39;/&#39;:# Rules for all requests that don&#39;t otherwise match....&#39;/sites/default/files&#39;:# Rules for any requests that begin with /sites/default/files....The simplest possible locations configuration is one that simply passes all requests on to your application unconditionally:\nweb:locations:&#39;/&#39;:passthru:trueThat is, all requests to /* should be forwarded to the process started by web.commands.start above. Note that for PHP containers the passthru key must specify what PHP file the request should be forwarded to, and must also specify a docroot under which the file lives. For example:\nweb:locations:&#39;/&#39;:root:&#39;web&#39;passthru:&#39;/app.php&#39;This block will serve requests to / from the web directory in the application, and if a file doesn&rsquo;t exist on disk then the request will be forwarded to the /app.php script.\nA full list of the possible subkeys for locations is below.\n  root: The folder from which to serve static assets for this location relative to the application root. The application root is the directory in which the .platform.app.yaml file is located. Typical values for this property include public or web. Setting it to '' is not recommended, and its behavior may vary depending on the type of application. Absolute paths are not supported.\n  passthru: Whether to forward disallowed and missing resources from this location to the application and can be true, false or an absolute URI path (with leading /). The default value is false. For non-PHP applications it will generally be just true or false. In a PHP application this will typically be the front controller such as /index.php or /app.php. This entry works similar to mod_rewrite under Apache. Note: If the value of passthru does not begin with the same value as the location key it is under, the passthru may evaluate to another entry. That may be useful when you want different cache settings for different paths, for instance, but want missing files in all of them to map back to the same front controller. See the example block below.\n  index: The files to consider when serving a request for a directory: an array of file names or null. (typically ['index.html']). Note that in order for this to work, access to the static files named must be allowed by the allow or rules keys for this location.\n  expires: How long to allow static assets from this location to be cached (this enables the Cache-Control and Expires headers) and can be a time or -1 for no caching (default). Times can be suffixed with &ldquo;ms&rdquo; (milliseconds), &ldquo;s&rdquo; (seconds), &ldquo;m&rdquo; (minutes), &ldquo;h&rdquo; (hours), &ldquo;d&rdquo; (days), &ldquo;w&rdquo; (weeks), &ldquo;M&rdquo; (months, 30d) or &ldquo;y&rdquo; (years, 365d).\n  scripts: Whether to allow loading scripts in that location (true or false). This directive is only meaningful on PHP.\n  allow: Whether to allow serving files which don&rsquo;t match a rule (true or false, default: true).\n  headers: Any additional headers to apply to static assets. This section is a mapping of header names to header values. Responses from the application aren&rsquo;t affected, to avoid overlap with the application&rsquo;s own ability to include custom headers in the response.\n  rules: Specific overrides for a specific location. The key is a PCRE (regular expression) that is matched against the full request path.\n  request_buffering: Most application servers do not support chunked requests (e.g. fpm, uwsgi), so Platform.sh enables request_buffering by default to handle them. That default configuration would look like this if it was present in .platform.app.yaml:\nweb:locations:&#39;/&#39;:passthru:truerequest_buffering:enabled:truemax_request_size:250mIf the application server can already efficiently handle chunked requests, the request_buffering subkey can be modified to disable it entirely (enabled: false). Additionally, applications that frequently deal with uploads greater than 250MB in size can update the max_request_size key to the application&rsquo;s needs. Note that modifications to request_buffering will need to be specified at each location where it is desired.\n ";
+        indexer.index_text(docid, indexed_pos, text);
+        let Indexed {
+            words_doc_indexes, ..
+        } = indexer.build();
+        assert!(words_doc_indexes.get(&"buffering".to_owned().into_bytes()).is_some());
+    }
+
+    #[test]
+    fn words_over_index_1000_not_indexed() {
+        let mut indexer = RawIndexer::new(fst::Set::default());
+        let indexed_pos = IndexedPos(0);
+        let docid = DocumentId(0);
+        let mut text = String::with_capacity(5000);
+        for _ in 0..1000 {
+            text.push_str("less ");
+        }
+        text.push_str("more");
+        indexer.index_text(docid, indexed_pos, &text);
+        let Indexed {
+            words_doc_indexes, ..
+        } = indexer.build();
+        assert!(words_doc_indexes.get(&"less".to_owned().into_bytes()).is_some());
+        assert!(words_doc_indexes.get(&"more".to_owned().into_bytes()).is_none());
+    }
 }

meilisearch-core/src/serde/convert_to_number.rs

@@ -1,198 +0,0 @@
-use std::str::FromStr;
-
-use ordered_float::OrderedFloat;
-use serde::ser;
-use serde::Serialize;
-
-use super::SerializerError;
-use crate::Number;
-
-pub struct ConvertToNumber;
-
-impl ser::Serializer for ConvertToNumber {
-    type Ok = Number;
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    fn serialize_bool(self, value: bool) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_char(self, _value: char) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "char" })
-    }
-
-    fn serialize_i8(self, value: i8) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(i64::from(value)))
-    }
-
-    fn serialize_i16(self, value: i16) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(i64::from(value)))
-    }
-
-    fn serialize_i32(self, value: i32) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(i64::from(value)))
-    }
-
-    fn serialize_i64(self, value: i64) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(value))
-    }
-
-    fn serialize_u8(self, value: u8) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_u16(self, value: u16) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_u32(self, value: u32) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_u64(self, value: u64) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(value))
-    }
-
-    fn serialize_f32(self, value: f32) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Float(OrderedFloat(f64::from(value))))
-    }
-
-    fn serialize_f64(self, value: f64) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Float(OrderedFloat(value)))
-    }
-
-    fn serialize_str(self, value: &str) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::from_str(value)?)
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnrankableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnrankableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "map" })
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "struct",
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "struct variant",
-        })
-    }
-}

meilisearch-core/src/serde/convert_to_string.rs

@@ -1,279 +0,0 @@
-use serde::ser;
-use serde::Serialize;
-
-use super::SerializerError;
-
-pub struct ConvertToString;
-
-impl ser::Serializer for ConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-    type SerializeSeq = SeqConvertToString;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = MapConvertToString;
-    type SerializeStruct = StructConvertToString;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    fn serialize_bool(self, value: bool) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_char(self, value: char) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i8(self, value: i8) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i16(self, value: i16) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i32(self, value: i32) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i64(self, value: i64) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u8(self, value: u8) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u16(self, value: u16) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u32(self, value: u32) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u64(self, value: u64) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_f32(self, value: f32) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_f64(self, value: f64) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_str(self, value: &str) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Ok(String::new())
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Ok(SeqConvertToString {
-            text: String::new(),
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        Ok(MapConvertToString {
-            text: String::new(),
-        })
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Ok(StructConvertToString {
-            text: String::new(),
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct MapConvertToString {
-    text: String,
-}
-
-impl ser::SerializeMap for MapConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = key.serialize(ConvertToString)?;
-        self.text.push_str(&text);
-        self.text.push_str(" ");
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.text.push_str(&text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.text)
-    }
-}
-
-pub struct StructConvertToString {
-    text: String,
-}
-
-impl ser::SerializeStruct for StructConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let value = value.serialize(ConvertToString)?;
-        self.text.push_str(key);
-        self.text.push_str(" ");
-        self.text.push_str(&value);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.text)
-    }
-}
-
-pub struct SeqConvertToString {
-    text: String,
-}
-
-impl ser::SerializeSeq for SeqConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-
-    fn serialize_element<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = key.serialize(ConvertToString)?;
-        self.text.push_str(&text);
-        self.text.push_str(" ");
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.text)
-    }
-}

meilisearch-core/src/serde/extract_document_id.rs

@@ -1,310 +0,0 @@
-use std::hash::{Hash, Hasher};
-
-use crate::DocumentId;
-use serde::{ser, Serialize};
-use serde_json::{Value, Number};
-use siphasher::sip::SipHasher;
-
-use super::{ConvertToString, SerializerError};
-
-pub fn extract_document_id<D>(
-    primary_key: &str,
-    document: &D,
-) -> Result<Option<DocumentId>, SerializerError>
-where
-    D: serde::Serialize,
-{
-    let serializer = ExtractDocumentId { primary_key };
-    document.serialize(serializer)
-}
-
-fn validate_number(value: &Number) -> Option<String> {
-    if value.is_f64() {
-        return None
-    }
-    Some(value.to_string())
-}
-
-fn validate_string(value: &str) -> Option<String> {
-    if value.chars().all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') {
-        Some(value.to_string())
-    } else {
-        None
-    }
-}
-
-pub fn value_to_string(value: &Value) -> Option<String> {
-    match value {
-        Value::Null => None,
-        Value::Bool(_) => None,
-        Value::Number(value) => validate_number(value),
-        Value::String(value) => validate_string(value),
-        Value::Array(_) => None,
-        Value::Object(_) => None,
-    }
-}
-
-pub fn compute_document_id<H: Hash>(t: H) -> DocumentId {
-    let mut s = SipHasher::new();
-    t.hash(&mut s);
-    let hash = s.finish();
-    DocumentId(hash)
-}
-
-struct ExtractDocumentId<'a> {
-    primary_key: &'a str,
-}
-
-impl<'a> ser::Serializer for ExtractDocumentId<'a> {
-    type Ok = Option<DocumentId>;
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = ExtractDocumentIdMapSerializer<'a>;
-    type SerializeStruct = ExtractDocumentIdStructSerializer<'a>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    forward_to_unserializable_type! {
-        bool => serialize_bool,
-        char => serialize_char,
-
-        i8  => serialize_i8,
-        i16 => serialize_i16,
-        i32 => serialize_i32,
-        i64 => serialize_i64,
-
-        u8  => serialize_u8,
-        u16 => serialize_u16,
-        u32 => serialize_u32,
-        u64 => serialize_u64,
-
-        f32 => serialize_f32,
-        f64 => serialize_f64,
-    }
-
-    fn serialize_str(self, _value: &str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "str" })
-    }
-
-    fn serialize_bytes(self, _value: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        let serializer = ExtractDocumentIdMapSerializer {
-            primary_key: self.primary_key,
-            document_id: None,
-            current_key_name: None,
-        };
-
-        Ok(serializer)
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        let serializer = ExtractDocumentIdStructSerializer {
-            primary_key: self.primary_key,
-            document_id: None,
-        };
-
-        Ok(serializer)
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct ExtractDocumentIdMapSerializer<'a> {
-    primary_key: &'a str,
-    document_id: Option<DocumentId>,
-    current_key_name: Option<String>,
-}
-
-impl<'a> ser::SerializeMap for ExtractDocumentIdMapSerializer<'a> {
-    type Ok = Option<DocumentId>;
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-        self.current_key_name = Some(key);
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        let key = self.current_key_name.take().unwrap();
-        self.serialize_entry(&key, value)
-    }
-
-    fn serialize_entry<K: ?Sized, V: ?Sized>(
-        &mut self,
-        key: &K,
-        value: &V,
-    ) -> Result<(), Self::Error>
-    where
-        K: Serialize,
-        V: Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-
-        if self.primary_key == key {
-            let value = serde_json::to_string(value).and_then(|s| serde_json::from_str(&s))?;
-            match value_to_string(&value).map(|s| compute_document_id(&s)) {
-                Some(document_id) => self.document_id = Some(document_id),
-                None => return Err(SerializerError::InvalidDocumentIdType),
-            }
-        }
-
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.document_id)
-    }
-}
-
-pub struct ExtractDocumentIdStructSerializer<'a> {
-    primary_key: &'a str,
-    document_id: Option<DocumentId>,
-}
-
-impl<'a> ser::SerializeStruct for ExtractDocumentIdStructSerializer<'a> {
-    type Ok = Option<DocumentId>;
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        if self.primary_key == key {
-            let value = serde_json::to_string(value).and_then(|s| serde_json::from_str(&s))?;
-            match value_to_string(&value).map(compute_document_id) {
-                Some(document_id) => self.document_id = Some(document_id),
-                None => return Err(SerializerError::InvalidDocumentIdType),
-            }
-        }
-
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.document_id)
-    }
-}

meilisearch-core/src/serde/indexer.rs

@@ -1,362 +0,0 @@
-use meilisearch_schema::IndexedPos;
-use serde::ser;
-use serde::Serialize;
-
-use super::{ConvertToString, SerializerError};
-use crate::raw_indexer::RawIndexer;
-use crate::DocumentId;
-
-pub struct Indexer<'a> {
-    pub pos: IndexedPos,
-    pub indexer: &'a mut RawIndexer,
-    pub document_id: DocumentId,
-}
-
-impl<'a> ser::Serializer for Indexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-    type SerializeSeq = SeqIndexer<'a>;
-    type SerializeTuple = TupleIndexer<'a>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = MapIndexer<'a>;
-    type SerializeStruct = StructIndexer<'a>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    fn serialize_bool(self, _value: bool) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_char(self, value: char) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i8(self, value: i8) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i16(self, value: i16) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i32(self, value: i32) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i64(self, value: i64) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u8(self, value: u8) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u16(self, value: u16) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u32(self, value: u32) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u64(self, value: u64) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_f32(self, value: f32) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_f64(self, value: f64) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_str(self, text: &str) -> Result<Self::Ok, Self::Error> {
-        let number_of_words = self
-            .indexer
-            .index_text(self.document_id, self.pos, text);
-        Ok(Some(number_of_words))
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_some<T: ?Sized>(self, value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        let number_of_words = self
-            .indexer
-            .index_text(self.document_id, self.pos, &text);
-        Ok(Some(number_of_words))
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        Err(SerializerError::UnindexableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        let indexer = SeqIndexer {
-            pos: self.pos,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        let indexer = TupleIndexer {
-            pos: self.pos,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        let indexer = MapIndexer {
-            pos: self.pos,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        let indexer = StructIndexer {
-            pos: self.pos,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct SeqIndexer<'a> {
-    pos: IndexedPos,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeSeq for SeqIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_element<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.pos, texts);
-        Ok(None)
-    }
-}
-
-pub struct MapIndexer<'a> {
-    pos: IndexedPos,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeMap for MapIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = key.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.pos, texts);
-        Ok(None)
-    }
-}
-
-pub struct StructIndexer<'a> {
-    pos: IndexedPos,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeStruct for StructIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let key_text = key.to_owned();
-        let value_text = value.serialize(ConvertToString)?;
-        self.texts.push(key_text);
-        self.texts.push(value_text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.pos, texts);
-        Ok(None)
-    }
-}
-
-pub struct TupleIndexer<'a> {
-    pos: IndexedPos,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeTuple for TupleIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_element<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.pos, texts);
-        Ok(None)
-    }
-}

meilisearch-core/src/serde/mod.rs

@@ -1,26 +1,6 @@
-macro_rules! forward_to_unserializable_type {
-    ($($ty:ident => $se_method:ident,)*) => {
-        $(
-            fn $se_method(self, _v: $ty) -> Result<Self::Ok, Self::Error> {
-                Err(SerializerError::UnserializableType { type_name: "$ty" })
-            }
-        )*
-    }
-}
-
-mod convert_to_number;
-mod convert_to_string;
 mod deserializer;
-mod extract_document_id;
-mod indexer;
-mod serializer;
 
-pub use self::convert_to_number::ConvertToNumber;
-pub use self::convert_to_string::ConvertToString;
 pub use self::deserializer::{Deserializer, DeserializerError};
-pub use self::extract_document_id::{compute_document_id, extract_document_id, value_to_string};
-pub use self::indexer::Indexer;
-pub use self::serializer::{serialize_value, serialize_value_with_id, Serializer};
 
 use std::{error::Error, fmt};
 
@@ -33,7 +13,7 @@ use crate::ParseNumberError;
 #[derive(Debug)]
 pub enum SerializerError {
     DocumentIdNotFound,
-    InvalidDocumentIdType,
+    InvalidDocumentIdFormat,
     Zlmdb(heed::Error),
     SerdeJson(SerdeJsonError),
     ParseNumber(ParseNumberError),
@@ -54,9 +34,9 @@ impl fmt::Display for SerializerError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
             SerializerError::DocumentIdNotFound => {
-                f.write_str("serialized document does not have an id according to the schema")
+                f.write_str("Primary key is missing.")
             }
-            SerializerError::InvalidDocumentIdType => {
+            SerializerError::InvalidDocumentIdFormat => {
                 f.write_str("a document primary key can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_).")
             }
             SerializerError::Zlmdb(e) => write!(f, "heed related error: {}", e),

meilisearch-core/src/serde/serializer.rs

@@ -1,361 +0,0 @@
-use meilisearch_schema::{Schema, FieldId};
-use serde::ser;
-
-use crate::database::MainT;
-use crate::raw_indexer::RawIndexer;
-use crate::store::{DocumentsFields, DocumentsFieldsCounts};
-use crate::{DocumentId, RankedMap};
-
-use super::{ConvertToNumber, ConvertToString, Indexer, SerializerError};
-
-pub struct Serializer<'a, 'b> {
-    pub txn: &'a mut heed::RwTxn<'b, MainT>,
-    pub schema: &'a mut Schema,
-    pub document_store: DocumentsFields,
-    pub document_fields_counts: DocumentsFieldsCounts,
-    pub indexer: &'a mut RawIndexer,
-    pub ranked_map: &'a mut RankedMap,
-    pub document_id: DocumentId,
-}
-
-impl<'a, 'b> ser::Serializer for Serializer<'a, 'b> {
-    type Ok = ();
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = MapSerializer<'a, 'b>;
-    type SerializeStruct = StructSerializer<'a, 'b>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    forward_to_unserializable_type! {
-        bool => serialize_bool,
-        char => serialize_char,
-
-        i8  => serialize_i8,
-        i16 => serialize_i16,
-        i32 => serialize_i32,
-        i64 => serialize_i64,
-
-        u8  => serialize_u8,
-        u16 => serialize_u16,
-        u32 => serialize_u32,
-        u64 => serialize_u64,
-
-        f32 => serialize_f32,
-        f64 => serialize_f64,
-    }
-
-    fn serialize_str(self, _v: &str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "str" })
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        Ok(MapSerializer {
-            txn: self.txn,
-            schema: self.schema,
-            document_id: self.document_id,
-            document_store: self.document_store,
-            document_fields_counts: self.document_fields_counts,
-            indexer: self.indexer,
-            ranked_map: self.ranked_map,
-            current_key_name: None,
-        })
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Ok(StructSerializer {
-            txn: self.txn,
-            schema: self.schema,
-            document_id: self.document_id,
-            document_store: self.document_store,
-            document_fields_counts: self.document_fields_counts,
-            indexer: self.indexer,
-            ranked_map: self.ranked_map,
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct MapSerializer<'a, 'b> {
-    txn: &'a mut heed::RwTxn<'b, MainT>,
-    schema: &'a mut Schema,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    document_fields_counts: DocumentsFieldsCounts,
-    indexer: &'a mut RawIndexer,
-    ranked_map: &'a mut RankedMap,
-    current_key_name: Option<String>,
-}
-
-impl<'a, 'b> ser::SerializeMap for MapSerializer<'a, 'b> {
-    type Ok = ();
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-        self.current_key_name = Some(key);
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let key = self.current_key_name.take().unwrap();
-        self.serialize_entry(&key, value)
-    }
-
-    fn serialize_entry<K: ?Sized, V: ?Sized>(
-        &mut self,
-        key: &K,
-        value: &V,
-    ) -> Result<(), Self::Error>
-    where
-        K: ser::Serialize,
-        V: ser::Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-        serialize_value(
-            self.txn,
-            key.as_str(),
-            self.schema,
-            self.document_id,
-            self.document_store,
-            self.document_fields_counts,
-            self.indexer,
-            self.ranked_map,
-            value,
-        )
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(())
-    }
-}
-
-pub struct StructSerializer<'a, 'b> {
-    txn: &'a mut heed::RwTxn<'b, MainT>,
-    schema: &'a mut Schema,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    document_fields_counts: DocumentsFieldsCounts,
-    indexer: &'a mut RawIndexer,
-    ranked_map: &'a mut RankedMap,
-}
-
-impl<'a, 'b> ser::SerializeStruct for StructSerializer<'a, 'b> {
-    type Ok = ();
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        serialize_value(
-            self.txn,
-            key,
-            self.schema,
-            self.document_id,
-            self.document_store,
-            self.document_fields_counts,
-            self.indexer,
-            self.ranked_map,
-            value,
-        )
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(())
-    }
-}
-
-pub fn serialize_value<'a, T: ?Sized>(
-    txn: &mut heed::RwTxn<MainT>,
-    attribute: &str,
-    schema: &'a mut Schema,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    documents_fields_counts: DocumentsFieldsCounts,
-    indexer: &mut RawIndexer,
-    ranked_map: &mut RankedMap,
-    value: &T,
-) -> Result<(), SerializerError>
-where
-    T: ser::Serialize,
-{
-    let field_id = schema.insert_and_index(&attribute)?;
-    serialize_value_with_id(
-        txn,
-        field_id,
-        schema,
-        document_id,
-        document_store,
-        documents_fields_counts,
-        indexer,
-        ranked_map,
-        value,
-    )
-}
-
-pub fn serialize_value_with_id<'a, T: ?Sized>(
-    txn: &mut heed::RwTxn<MainT>,
-    field_id: FieldId,
-    schema: &'a Schema,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    documents_fields_counts: DocumentsFieldsCounts,
-    indexer: &mut RawIndexer,
-    ranked_map: &mut RankedMap,
-    value: &T,
-) -> Result<(), SerializerError>
-where
-    T: ser::Serialize,
-{
-    let serialized = serde_json::to_vec(value)?;
-    document_store.put_document_field(txn, document_id, field_id, &serialized)?;
-
-    if let Some(indexed_pos) = schema.is_indexed(field_id) {
-        let indexer = Indexer {
-            pos: *indexed_pos,
-            indexer,
-            document_id,
-        };
-        if let Some(number_of_words) = value.serialize(indexer)? {
-            documents_fields_counts.put_document_field_count(
-                txn,
-                document_id,
-                *indexed_pos,
-                number_of_words as u16,
-            )?;
-        }
-    }
-
-    if schema.is_ranked(field_id) {
-        let number = value.serialize(ConvertToNumber).unwrap_or_default();
-        ranked_map.insert(document_id, field_id, number);
-    }
-
-    Ok(())
-}

meilisearch-core/src/settings.rs

@@ -10,8 +10,7 @@ use self::RankingRule::*;
 pub const DEFAULT_RANKING_RULES: [RankingRule; 6] = [Typo, Words, Proximity, Attribute, WordsPosition, Exactness];
 
 static RANKING_RULE_REGEX: Lazy<regex::Regex> = Lazy::new(|| {
-    let regex = regex::Regex::new(r"(asc|desc)\(([a-zA-Z0-9-_]*)\)").unwrap();
-    regex
+    regex::Regex::new(r"(asc|desc)\(([a-zA-Z0-9-_]*)\)").unwrap()
 });
 
 #[derive(Default, Clone, Serialize, Deserialize)]
@@ -30,7 +29,7 @@ pub struct Settings {
     #[serde(default, deserialize_with = "deserialize_some")]
     pub synonyms: Option<Option<BTreeMap<String, Vec<String>>>>,
     #[serde(default, deserialize_with = "deserialize_some")]
-    pub accept_new_fields: Option<Option<bool>>,
+    pub attributes_for_faceting: Option<Option<Vec<String>>>,
 }
 
 // Any value that is present is considered Some value, including null.
@@ -42,11 +41,11 @@ fn deserialize_some<'de, T, D>(deserializer: D) -> Result<Option<T>, D::Error>
 }
 
 impl Settings {
-    pub fn into_update(&self) -> Result<SettingsUpdate, RankingRuleConversionError> {
+    pub fn to_update(&self) -> Result<SettingsUpdate, RankingRuleConversionError> {
         let settings = self.clone();
 
         let ranking_rules = match settings.ranking_rules {
-            Some(Some(rules)) => UpdateState::Update(RankingRule::from_iter(rules.iter())?),
+            Some(Some(rules)) => UpdateState::Update(RankingRule::try_from_iter(rules.iter())?),
             Some(None) => UpdateState::Clear,
             None => UpdateState::Nothing,
         };
@@ -59,7 +58,7 @@ impl Settings {
             displayed_attributes: settings.displayed_attributes.into(),
             stop_words: settings.stop_words.into(),
             synonyms: settings.synonyms.into(),
-            accept_new_fields: settings.accept_new_fields.into(),
+            attributes_for_faceting: settings.attributes_for_faceting.into(),
         })
     }
 }
@@ -149,7 +148,7 @@ impl RankingRule {
         }
     }
 
-    pub fn from_iter(rules: impl IntoIterator<Item = impl AsRef<str>>) -> Result<Vec<RankingRule>, RankingRuleConversionError> {
+    pub fn try_from_iter(rules: impl IntoIterator<Item = impl AsRef<str>>) -> Result<Vec<RankingRule>, RankingRuleConversionError> {
         rules.into_iter()
             .map(|s| RankingRule::from_str(s.as_ref()))
             .collect()
@@ -165,7 +164,7 @@ pub struct SettingsUpdate {
     pub displayed_attributes: UpdateState<HashSet<String>>,
     pub stop_words: UpdateState<BTreeSet<String>>,
     pub synonyms: UpdateState<BTreeMap<String, Vec<String>>>,
-    pub accept_new_fields: UpdateState<bool>,
+    pub attributes_for_faceting: UpdateState<Vec<String>>,
 }
 
 impl Default for SettingsUpdate {
@@ -178,7 +177,7 @@ impl Default for SettingsUpdate {
             displayed_attributes: UpdateState::Nothing,
             stop_words: UpdateState::Nothing,
             synonyms: UpdateState::Nothing,
-            accept_new_fields: UpdateState::Nothing,
+            attributes_for_faceting: UpdateState::Nothing,
         }
     }
 }

meilisearch-core/src/store/cow_set.rs

@@ -0,0 +1,32 @@
+use std::borrow::Cow;
+
+use heed::{types::CowSlice, BytesEncode, BytesDecode};
+use sdset::{Set, SetBuf};
+use zerocopy::{AsBytes, FromBytes};
+
+pub struct CowSet<T>(std::marker::PhantomData<T>);
+
+impl<'a, T: 'a> BytesEncode<'a> for CowSet<T>
+where
+    T: AsBytes,
+{
+    type EItem = Set<T>;
+
+    fn bytes_encode(item: &'a Self::EItem) -> Option<Cow<[u8]>> {
+        CowSlice::bytes_encode(item.as_slice())
+    }
+}
+
+impl<'a, T: 'a> BytesDecode<'a> for CowSet<T>
+where
+    T: FromBytes + Copy,
+{
+    type DItem = Cow<'a, Set<T>>;
+
+    fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
+        match CowSlice::<T>::bytes_decode(bytes)? {
+            Cow::Owned(vec) => Some(Cow::Owned(SetBuf::new_unchecked(vec))),
+            Cow::Borrowed(slice) => Some(Cow::Borrowed(Set::new_unchecked(slice))),
+        }
+    }
+}

meilisearch-core/src/store/docs_words.rs

@@ -1,13 +1,15 @@
-use super::BEU64;
-use crate::database::MainT;
-use crate::DocumentId;
-use heed::types::{ByteSlice, OwnedType};
+use std::borrow::Cow;
+
 use heed::Result as ZResult;
-use std::sync::Arc;
+use heed::types::{ByteSlice, OwnedType};
+
+use crate::database::MainT;
+use crate::{DocumentId, FstSetCow};
+use super::BEU32;
 
 #[derive(Copy, Clone)]
 pub struct DocsWords {
-    pub(crate) docs_words: heed::Database<OwnedType<BEU64>, ByteSlice>,
+    pub(crate) docs_words: heed::Database<OwnedType<BEU32>, ByteSlice>,
 }
 
 impl DocsWords {
@@ -15,15 +17,15 @@ impl DocsWords {
         self,
         writer: &mut heed::RwTxn<MainT>,
         document_id: DocumentId,
-        words: &fst::Set,
+        words: &FstSetCow,
     ) -> ZResult<()> {
-        let document_id = BEU64::new(document_id.0);
+        let document_id = BEU32::new(document_id.0);
         let bytes = words.as_fst().as_bytes();
         self.docs_words.put(writer, &document_id, bytes)
     }
 
     pub fn del_doc_words(self, writer: &mut heed::RwTxn<MainT>, document_id: DocumentId) -> ZResult<bool> {
-        let document_id = BEU64::new(document_id.0);
+        let document_id = BEU32::new(document_id.0);
         self.docs_words.delete(writer, &document_id)
     }
 
@@ -31,20 +33,11 @@ impl DocsWords {
         self.docs_words.clear(writer)
     }
 
-    pub fn doc_words(
-        self,
-        reader: &heed::RoTxn<MainT>,
-        document_id: DocumentId,
-    ) -> ZResult<Option<fst::Set>> {
-        let document_id = BEU64::new(document_id.0);
+    pub fn doc_words(self, reader: &heed::RoTxn<MainT>, document_id: DocumentId) -> ZResult<FstSetCow> {
+        let document_id = BEU32::new(document_id.0);
         match self.docs_words.get(reader, &document_id)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
+            Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
+            None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()),
         }
     }
 }

meilisearch-core/src/store/documents_fields_counts.rs

@@ -4,6 +4,7 @@ use crate::DocumentId;
 use heed::types::OwnedType;
 use heed::Result as ZResult;
 use meilisearch_schema::IndexedPos;
+use crate::MResult;
 
 #[derive(Copy, Clone)]
 pub struct DocumentsFieldsCounts {
@@ -60,7 +61,7 @@ impl DocumentsFieldsCounts {
         Ok(DocumentFieldsCountsIter { iter })
     }
 
-    pub fn documents_ids<'txn>(self, reader: &'txn heed::RoTxn<MainT>) -> ZResult<DocumentsIdsIter<'txn>> {
+    pub fn documents_ids<'txn>(self, reader: &'txn heed::RoTxn<MainT>) -> MResult<DocumentsIdsIter<'txn>> {
         let iter = self.documents_fields_counts.iter(reader)?;
         Ok(DocumentsIdsIter {
             last_seen_id: None,
@@ -102,7 +103,7 @@ pub struct DocumentsIdsIter<'txn> {
 }
 
 impl Iterator for DocumentsIdsIter<'_> {
-    type Item = ZResult<DocumentId>;
+    type Item = MResult<DocumentId>;
 
     fn next(&mut self) -> Option<Self::Item> {
         for result in &mut self.iter {
@@ -114,7 +115,7 @@ impl Iterator for DocumentsIdsIter<'_> {
                         return Some(Ok(document_id));
                     }
                 }
-                Err(e) => return Some(Err(e)),
+                Err(e) => return Some(Err(e.into())),
             }
         }
         None

meilisearch-core/src/store/documents_ids.rs

@@ -0,0 +1,75 @@
+use std::borrow::Cow;
+
+use heed::{BytesDecode, BytesEncode};
+use sdset::Set;
+
+use crate::DocumentId;
+use super::cow_set::CowSet;
+
+pub struct DocumentsIds;
+
+impl BytesEncode<'_> for DocumentsIds {
+    type EItem = Set<DocumentId>;
+
+    fn bytes_encode(item: &Self::EItem) -> Option<Cow<[u8]>> {
+        CowSet::bytes_encode(item)
+    }
+}
+
+impl<'a> BytesDecode<'a> for DocumentsIds {
+    type DItem = Cow<'a, Set<DocumentId>>;
+
+    fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
+        CowSet::bytes_decode(bytes)
+    }
+}
+
+pub struct DiscoverIds<'a> {
+    ids_iter: std::slice::Iter<'a, DocumentId>,
+    left_id: Option<u32>,
+    right_id: Option<u32>,
+    available_range: std::ops::Range<u32>,
+}
+
+impl DiscoverIds<'_> {
+    pub fn new(ids: &Set<DocumentId>) -> DiscoverIds {
+        let mut ids_iter = ids.iter();
+        let right_id = ids_iter.next().map(|id| id.0);
+        let available_range = 0..right_id.unwrap_or(u32::max_value());
+        DiscoverIds { ids_iter, left_id: None, right_id, available_range }
+    }
+}
+
+impl Iterator for DiscoverIds<'_> {
+    type Item = DocumentId;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            match self.available_range.next() {
+                // The available range gives us a new id, we return it.
+                Some(id) => return Some(DocumentId(id)),
+                // The available range is exhausted, we need to find the next one.
+                None if self.available_range.end == u32::max_value() => return None,
+                None => loop {
+                    self.left_id = self.right_id.take();
+                    self.right_id = self.ids_iter.next().map(|id| id.0);
+                    match (self.left_id, self.right_id) {
+                        // We found a gap in the used ids, we can yield all ids
+                        // until the end of the gap
+                        (Some(l), Some(r)) => if l.saturating_add(1) != r {
+                            self.available_range = (l + 1)..r;
+                            break;
+                        },
+                        // The last used id has been reached, we can use all ids
+                        // until u32 MAX
+                        (Some(l), None) => {
+                            self.available_range = l.saturating_add(1)..u32::max_value();
+                            break;
+                        },
+                        _ => (),
+                    }
+                },
+            }
+        }
+    }
+}

meilisearch-core/src/store/facets.rs

@@ -0,0 +1,58 @@
+use std::borrow::Cow;
+use std::collections::HashMap;
+
+use heed::{RwTxn, RoTxn, Result as ZResult, RoRange};
+use sdset::{SetBuf, Set, SetOperation};
+
+use meilisearch_types::DocumentId;
+use meilisearch_schema::FieldId;
+
+use crate::database::MainT;
+use crate::facets::FacetKey;
+use super::cow_set::CowSet;
+
+/// contains facet info
+#[derive(Clone, Copy)]
+pub struct Facets {
+    pub(crate) facets: heed::Database<FacetKey, CowSet<DocumentId>>,
+}
+
+impl Facets {
+    // we use sdset::SetBuf to ensure the docids are sorted.
+    pub fn put_facet_document_ids(&self, writer: &mut RwTxn<MainT>, facet_key: FacetKey, doc_ids: &Set<DocumentId>) -> ZResult<()> {
+        self.facets.put(writer, &facet_key, doc_ids)
+    }
+
+    pub fn field_document_ids<'txn>(&self, reader: &'txn RoTxn<MainT>, field_id: FieldId) -> ZResult<RoRange<'txn, FacetKey, CowSet<DocumentId>>> {
+        self.facets.prefix_iter(reader, &FacetKey::new(field_id, String::new()))
+    }
+
+    pub fn facet_document_ids<'txn>(&self, reader: &'txn RoTxn<MainT>, facet_key: &FacetKey) -> ZResult<Option<Cow<'txn, Set<DocumentId>>>> {
+        self.facets.get(reader, &facet_key)
+    }
+
+    /// updates the facets  store, revmoving the documents from the facets provided in the
+    /// `facet_map` argument
+    pub fn remove(&self, writer: &mut RwTxn<MainT>, facet_map: HashMap<FacetKey, Vec<DocumentId>>) -> ZResult<()> {
+        for (key, document_ids) in facet_map {
+            if let Some(old) = self.facets.get(writer, &key)? {
+                let to_remove = SetBuf::from_dirty(document_ids);
+                let new = sdset::duo::OpBuilder::new(old.as_ref(), to_remove.as_set()).difference().into_set_buf();
+                self.facets.put(writer, &key, new.as_set())?;
+            }
+        }
+        Ok(())
+    }
+
+    pub fn add(&self, writer: &mut RwTxn<MainT>, facet_map: HashMap<FacetKey, Vec<DocumentId>>) -> ZResult<()> {
+        for (key, document_ids) in facet_map {
+            let set = SetBuf::from_dirty(document_ids);
+            self.put_facet_document_ids(writer, key, set.as_set())?;
+        }
+        Ok(())
+    }
+
+    pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
+        self.facets.clear(writer)
+    }
+}

meilisearch-core/src/store/main.rs

@@ -1,26 +1,33 @@
-use std::sync::Arc;
+use std::borrow::Cow;
 use std::collections::HashMap;
 
 use chrono::{DateTime, Utc};
-use heed::types::{ByteSlice, OwnedType, SerdeBincode, Str};
-use heed::Result as ZResult;
-use meilisearch_schema::Schema;
+use heed::types::{ByteSlice, OwnedType, SerdeBincode, Str, CowSlice};
+use meilisearch_schema::{FieldId, Schema};
+use meilisearch_types::DocumentId;
+use sdset::Set;
 
 use crate::database::MainT;
-use crate::RankedMap;
+use crate::{RankedMap, MResult};
 use crate::settings::RankingRule;
+use crate::{FstSetCow, FstMapCow};
+use super::{CowSet, DocumentsIds};
 
+const ATTRIBUTES_FOR_FACETING_KEY: &str = "attributes-for-faceting";
 const CREATED_AT_KEY: &str = "created-at";
-const RANKING_RULES_KEY: &str = "ranking-rules";
-const DISTINCT_ATTRIBUTE_KEY: &str = "distinct-attribute";
-const STOP_WORDS_KEY: &str = "stop-words";
-const SYNONYMS_KEY: &str = "synonyms";
 const CUSTOMS_KEY: &str = "customs";
-const FIELDS_FREQUENCY_KEY: &str = "fields-frequency";
+const DISTINCT_ATTRIBUTE_KEY: &str = "distinct-attribute";
+const EXTERNAL_DOCIDS_KEY: &str = "external-docids";
+const FIELDS_DISTRIBUTION_KEY: &str = "fields-distribution";
+const INTERNAL_DOCIDS_KEY: &str = "internal-docids";
 const NAME_KEY: &str = "name";
 const NUMBER_OF_DOCUMENTS_KEY: &str = "number-of-documents";
 const RANKED_MAP_KEY: &str = "ranked-map";
+const RANKING_RULES_KEY: &str = "ranking-rules";
 const SCHEMA_KEY: &str = "schema";
+const SORTED_DOCUMENT_IDS_CACHE_KEY: &str = "sorted-document-ids-cache";
+const STOP_WORDS_KEY: &str = "stop-words";
+const SYNONYMS_KEY: &str = "synonyms";
 const UPDATED_AT_KEY: &str = "updated-at";
 const WORDS_KEY: &str = "words";
 
@@ -34,122 +41,200 @@ pub struct Main {
 }
 
 impl Main {
-    pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
-        self.main.clear(writer)
+    pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> MResult<()> {
+        Ok(self.main.clear(writer)?)
     }
 
-    pub fn put_name(self, writer: &mut heed::RwTxn<MainT>, name: &str) -> ZResult<()> {
-        self.main.put::<_, Str, Str>(writer, NAME_KEY, name)
+    pub fn put_name(self, writer: &mut heed::RwTxn<MainT>, name: &str) -> MResult<()> {
+        Ok(self.main.put::<_, Str, Str>(writer, NAME_KEY, name)?)
     }
 
-    pub fn name(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<String>> {
+    pub fn name(self, reader: &heed::RoTxn<MainT>) -> MResult<Option<String>> {
         Ok(self
             .main
             .get::<_, Str, Str>(reader, NAME_KEY)?
             .map(|name| name.to_owned()))
     }
 
-    pub fn put_created_at(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
-        self.main
-            .put::<_, Str, SerdeDatetime>(writer, CREATED_AT_KEY, &Utc::now())
+    pub fn put_created_at(self, writer: &mut heed::RwTxn<MainT>) -> MResult<()> {
+        Ok(self.main.put::<_, Str, SerdeDatetime>(writer, CREATED_AT_KEY, &Utc::now())?)
     }
 
-    pub fn created_at(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<DateTime<Utc>>> {
-        self.main.get::<_, Str, SerdeDatetime>(reader, CREATED_AT_KEY)
+    pub fn created_at(self, reader: &heed::RoTxn<MainT>) -> MResult<Option<DateTime<Utc>>> {
+        Ok(self.main.get::<_, Str, SerdeDatetime>(reader, CREATED_AT_KEY)?)
     }
 
-    pub fn put_updated_at(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
-        self.main
-            .put::<_, Str, SerdeDatetime>(writer, UPDATED_AT_KEY, &Utc::now())
+    pub fn put_updated_at(self, writer: &mut heed::RwTxn<MainT>) -> MResult<()> {
+        Ok(self.main.put::<_, Str, SerdeDatetime>(writer, UPDATED_AT_KEY, &Utc::now())?)
     }
 
-    pub fn updated_at(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<DateTime<Utc>>> {
-        self.main.get::<_, Str, SerdeDatetime>(reader, UPDATED_AT_KEY)
+    pub fn updated_at(self, reader: &heed::RoTxn<MainT>) -> MResult<Option<DateTime<Utc>>> {
+        Ok(self.main.get::<_, Str, SerdeDatetime>(reader, UPDATED_AT_KEY)?)
     }
 
-    pub fn put_words_fst(self, writer: &mut heed::RwTxn<MainT>, fst: &fst::Set) -> ZResult<()> {
-        let bytes = fst.as_fst().as_bytes();
-        self.main.put::<_, Str, ByteSlice>(writer, WORDS_KEY, bytes)
+    pub fn put_internal_docids(self, writer: &mut heed::RwTxn<MainT>, ids: &sdset::Set<DocumentId>) -> MResult<()> {
+        Ok(self.main.put::<_, Str, DocumentsIds>(writer, INTERNAL_DOCIDS_KEY, ids)?)
     }
 
-    pub unsafe fn static_words_fst(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<fst::Set>> {
-        match self.main.get::<_, Str, ByteSlice>(reader, WORDS_KEY)? {
-            Some(bytes) => {
-                let bytes: &'static [u8] = std::mem::transmute(bytes);
-                let set = fst::Set::from_static_slice(bytes).unwrap();
-                Ok(Some(set))
-            }
-            None => Ok(None),
+    pub fn internal_docids<'txn>(self, reader: &'txn heed::RoTxn<MainT>) -> MResult<Cow<'txn, sdset::Set<DocumentId>>> {
+        match self.main.get::<_, Str, DocumentsIds>(reader, INTERNAL_DOCIDS_KEY)? {
+            Some(ids) => Ok(ids),
+            None => Ok(Cow::default()),
         }
     }
 
-    pub fn words_fst(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<fst::Set>> {
-        match self.main.get::<_, Str, ByteSlice>(reader, WORDS_KEY)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
+    pub fn merge_internal_docids(self, writer: &mut heed::RwTxn<MainT>, new_ids: &sdset::Set<DocumentId>) -> MResult<()> {
+        use sdset::SetOperation;
+
+        // We do an union of the old and new internal ids.
+        let internal_docids = self.internal_docids(writer)?;
+        let internal_docids = sdset::duo::Union::new(&internal_docids, new_ids).into_set_buf();
+        Ok(self.put_internal_docids(writer, &internal_docids)?)
+    }
+
+    pub fn remove_internal_docids(self, writer: &mut heed::RwTxn<MainT>, ids: &sdset::Set<DocumentId>) -> MResult<()> {
+        use sdset::SetOperation;
+
+        // We do a difference of the old and new internal ids.
+        let internal_docids = self.internal_docids(writer)?;
+        let internal_docids = sdset::duo::Difference::new(&internal_docids, ids).into_set_buf();
+        Ok(self.put_internal_docids(writer, &internal_docids)?)
+    }
+
+    pub fn put_external_docids<A>(self, writer: &mut heed::RwTxn<MainT>, ids: &fst::Map<A>) -> MResult<()>
+    where A: AsRef<[u8]>,
+    {
+        Ok(self.main.put::<_, Str, ByteSlice>(writer, EXTERNAL_DOCIDS_KEY, ids.as_fst().as_bytes())?)
+    }
+
+    pub fn merge_external_docids<A>(self, writer: &mut heed::RwTxn<MainT>, new_docids: &fst::Map<A>) -> MResult<()>
+    where A: AsRef<[u8]>,
+    {
+        use fst::{Streamer, IntoStreamer};
+
+        // Do an union of the old and the new set of external docids.
+        let external_docids = self.external_docids(writer)?;
+        let mut op = external_docids.op().add(new_docids.into_stream()).r#union();
+        let mut build = fst::MapBuilder::memory();
+        while let Some((docid, values)) = op.next() {
+            build.insert(docid, values[0].value).unwrap();
+        }
+        drop(op);
+
+        let external_docids = build.into_map();
+        Ok(self.put_external_docids(writer, &external_docids)?)
+    }
+
+    pub fn remove_external_docids<A>(self, writer: &mut heed::RwTxn<MainT>, ids: &fst::Map<A>) -> MResult<()>
+    where A: AsRef<[u8]>,
+    {
+        use fst::{Streamer, IntoStreamer};
+
+        // Do an union of the old and the new set of external docids.
+        let external_docids = self.external_docids(writer)?;
+        let mut op = external_docids.op().add(ids.into_stream()).difference();
+        let mut build = fst::MapBuilder::memory();
+        while let Some((docid, values)) = op.next() {
+            build.insert(docid, values[0].value).unwrap();
+        }
+        drop(op);
+
+        let external_docids = build.into_map();
+        self.put_external_docids(writer, &external_docids)
+    }
+
+    pub fn external_docids(self, reader: &heed::RoTxn<MainT>) -> MResult<FstMapCow> {
+        match self.main.get::<_, Str, ByteSlice>(reader, EXTERNAL_DOCIDS_KEY)? {
+            Some(bytes) => Ok(fst::Map::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
+            None => Ok(fst::Map::default().map_data(Cow::Owned).unwrap()),
         }
     }
 
-    pub fn put_schema(self, writer: &mut heed::RwTxn<MainT>, schema: &Schema) -> ZResult<()> {
-        self.main.put::<_, Str, SerdeBincode<Schema>>(writer, SCHEMA_KEY, schema)
+    pub fn external_to_internal_docid(self, reader: &heed::RoTxn<MainT>, external_docid: &str) -> MResult<Option<DocumentId>> {
+        let external_ids = self.external_docids(reader)?;
+        Ok(external_ids.get(external_docid).map(|id| DocumentId(id as u32)))
     }
 
-    pub fn schema(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<Schema>> {
-        self.main.get::<_, Str, SerdeBincode<Schema>>(reader, SCHEMA_KEY)
+    pub fn words_fst(self, reader: &heed::RoTxn<MainT>) -> MResult<FstSetCow> {
+        match self.main.get::<_, Str, ByteSlice>(reader, WORDS_KEY)? {
+            Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
+            None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()),
+        }
     }
 
-    pub fn delete_schema(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<bool> {
-        self.main.delete::<_, Str>(writer, SCHEMA_KEY)
+    pub fn put_words_fst<A: AsRef<[u8]>>(self, writer: &mut heed::RwTxn<MainT>, fst: &fst::Set<A>) -> MResult<()> {
+        Ok(self.main.put::<_, Str, ByteSlice>(writer, WORDS_KEY, fst.as_fst().as_bytes())?)
     }
 
-    pub fn put_ranked_map(self, writer: &mut heed::RwTxn<MainT>, ranked_map: &RankedMap) -> ZResult<()> {
-        self.main.put::<_, Str, SerdeBincode<RankedMap>>(writer, RANKED_MAP_KEY, &ranked_map)
+    pub fn put_sorted_document_ids_cache(self, writer: &mut heed::RwTxn<MainT>, documents_ids: &[DocumentId]) -> MResult<()> {
+        Ok(self.main.put::<_, Str, CowSlice<DocumentId>>(writer, SORTED_DOCUMENT_IDS_CACHE_KEY, documents_ids)?)
     }
 
-    pub fn ranked_map(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<RankedMap>> {
-        self.main.get::<_, Str, SerdeBincode<RankedMap>>(reader, RANKED_MAP_KEY)
+    pub fn sorted_document_ids_cache(self, reader: &heed::RoTxn<MainT>) -> MResult<Option<Cow<[DocumentId]>>> {
+        Ok(self.main.get::<_, Str, CowSlice<DocumentId>>(reader, SORTED_DOCUMENT_IDS_CACHE_KEY)?)
     }
 
-    pub fn put_synonyms_fst(self, writer: &mut heed::RwTxn<MainT>, fst: &fst::Set) -> ZResult<()> {
+    pub fn put_schema(self, writer: &mut heed::RwTxn<MainT>, schema: &Schema) -> MResult<()> {
+        Ok(self.main.put::<_, Str, SerdeBincode<Schema>>(writer, SCHEMA_KEY, schema)?)
+    }
+
+    pub fn schema(self, reader: &heed::RoTxn<MainT>) -> MResult<Option<Schema>> {
+        Ok(self.main.get::<_, Str, SerdeBincode<Schema>>(reader, SCHEMA_KEY)?)
+    }
+
+    pub fn delete_schema(self, writer: &mut heed::RwTxn<MainT>) -> MResult<bool> {
+        Ok(self.main.delete::<_, Str>(writer, SCHEMA_KEY)?)
+    }
+
+    pub fn put_ranked_map(self, writer: &mut heed::RwTxn<MainT>, ranked_map: &RankedMap) -> MResult<()> {
+        Ok(self.main.put::<_, Str, SerdeBincode<RankedMap>>(writer, RANKED_MAP_KEY, &ranked_map)?)
+    }
+
+    pub fn ranked_map(self, reader: &heed::RoTxn<MainT>) -> MResult<Option<RankedMap>> {
+        Ok(self.main.get::<_, Str, SerdeBincode<RankedMap>>(reader, RANKED_MAP_KEY)?)
+    }
+
+    pub fn put_synonyms_fst<A: AsRef<[u8]>>(self, writer: &mut heed::RwTxn<MainT>, fst: &fst::Set<A>) -> MResult<()> {
         let bytes = fst.as_fst().as_bytes();
-        self.main.put::<_, Str, ByteSlice>(writer, SYNONYMS_KEY, bytes)
+        Ok(self.main.put::<_, Str, ByteSlice>(writer, SYNONYMS_KEY, bytes)?)
     }
 
-    pub fn synonyms_fst(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<fst::Set>> {
+    pub(crate) fn synonyms_fst(self, reader: &heed::RoTxn<MainT>) -> MResult<FstSetCow> {
         match self.main.get::<_, Str, ByteSlice>(reader, SYNONYMS_KEY)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
+            Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
+            None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()),
         }
     }
 
-    pub fn put_stop_words_fst(self, writer: &mut heed::RwTxn<MainT>, fst: &fst::Set) -> ZResult<()> {
+    pub fn synonyms(self, reader: &heed::RoTxn<MainT>) -> MResult<Vec<String>> {
+        let synonyms = self
+            .synonyms_fst(&reader)?
+            .stream()
+            .into_strs()?;
+        Ok(synonyms)
+    }
+
+    pub fn put_stop_words_fst<A: AsRef<[u8]>>(self, writer: &mut heed::RwTxn<MainT>, fst: &fst::Set<A>) -> MResult<()> {
         let bytes = fst.as_fst().as_bytes();
-        self.main.put::<_, Str, ByteSlice>(writer, STOP_WORDS_KEY, bytes)
+        Ok(self.main.put::<_, Str, ByteSlice>(writer, STOP_WORDS_KEY, bytes)?)
     }
 
-    pub fn stop_words_fst(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<fst::Set>> {
+    pub(crate) fn stop_words_fst(self, reader: &heed::RoTxn<MainT>) -> MResult<FstSetCow> {
         match self.main.get::<_, Str, ByteSlice>(reader, STOP_WORDS_KEY)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
+            Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
+            None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()),
         }
     }
 
-    pub fn put_number_of_documents<F>(self, writer: &mut heed::RwTxn<MainT>, f: F) -> ZResult<u64>
+    pub fn stop_words(self, reader: &heed::RoTxn<MainT>) -> MResult<Vec<String>> {
+        let stop_word_list = self
+            .stop_words_fst(reader)?
+            .stream()
+            .into_strs()?;
+        Ok(stop_word_list)
+    }
+
+    pub fn put_number_of_documents<F>(self, writer: &mut heed::RwTxn<MainT>, f: F) -> MResult<u64>
     where
         F: Fn(u64) -> u64,
     {
@@ -159,68 +244,77 @@ impl Main {
         Ok(new)
     }
 
-    pub fn number_of_documents(self, reader: &heed::RoTxn<MainT>) -> ZResult<u64> {
+    pub fn number_of_documents(self, reader: &heed::RoTxn<MainT>) -> MResult<u64> {
         match self
             .main
-            .get::<_, Str, OwnedType<u64>>(reader, NUMBER_OF_DOCUMENTS_KEY)?
-        {
+            .get::<_, Str, OwnedType<u64>>(reader, NUMBER_OF_DOCUMENTS_KEY)? {
             Some(value) => Ok(value),
             None => Ok(0),
         }
     }
 
-    pub fn put_fields_frequency(
+    pub fn put_fields_distribution(
         self,
         writer: &mut heed::RwTxn<MainT>,
         fields_frequency: &FreqsMap,
-    ) -> ZResult<()> {
-        self.main
-            .put::<_, Str, SerdeFreqsMap>(writer, FIELDS_FREQUENCY_KEY, fields_frequency)
+    ) -> MResult<()> {
+        Ok(self.main.put::<_, Str, SerdeFreqsMap>(writer, FIELDS_DISTRIBUTION_KEY, fields_frequency)?)
     }
 
-    pub fn fields_frequency(&self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<FreqsMap>> {
+    pub fn fields_distribution(&self, reader: &heed::RoTxn<MainT>) -> MResult<Option<FreqsMap>> {
         match self
             .main
-            .get::<_, Str, SerdeFreqsMap>(reader, FIELDS_FREQUENCY_KEY)?
+            .get::<_, Str, SerdeFreqsMap>(reader, FIELDS_DISTRIBUTION_KEY)?
         {
             Some(freqs) => Ok(Some(freqs)),
             None => Ok(None),
         }
     }
 
-    pub fn ranking_rules(&self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<Vec<RankingRule>>> {
-        self.main.get::<_, Str, SerdeBincode<Vec<RankingRule>>>(reader, RANKING_RULES_KEY)
+    pub fn attributes_for_faceting<'txn>(&self, reader: &'txn heed::RoTxn<MainT>) -> MResult<Option<Cow<'txn, Set<FieldId>>>> {
+        Ok(self.main.get::<_, Str, CowSet<FieldId>>(reader, ATTRIBUTES_FOR_FACETING_KEY)?)
     }
 
-    pub fn put_ranking_rules(self, writer: &mut heed::RwTxn<MainT>, value: &[RankingRule]) -> ZResult<()> {
-        self.main.put::<_, Str, SerdeBincode<Vec<RankingRule>>>(writer, RANKING_RULES_KEY, &value.to_vec())
+    pub fn put_attributes_for_faceting(self, writer: &mut heed::RwTxn<MainT>, attributes: &Set<FieldId>) -> MResult<()> {
+        Ok(self.main.put::<_, Str, CowSet<FieldId>>(writer, ATTRIBUTES_FOR_FACETING_KEY, attributes)?)
     }
 
-    pub fn delete_ranking_rules(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<bool> {
-        self.main.delete::<_, Str>(writer, RANKING_RULES_KEY)
+    pub fn delete_attributes_for_faceting(self, writer: &mut heed::RwTxn<MainT>) -> MResult<bool> {
+        Ok(self.main.delete::<_, Str>(writer, ATTRIBUTES_FOR_FACETING_KEY)?)
     }
 
-    pub fn distinct_attribute(&self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<String>> {
-        if let Some(value) = self.main.get::<_, Str, Str>(reader, DISTINCT_ATTRIBUTE_KEY)? {
-            return Ok(Some(value.to_owned()))
+    pub fn ranking_rules(&self, reader: &heed::RoTxn<MainT>) -> MResult<Option<Vec<RankingRule>>> {
+        Ok(self.main.get::<_, Str, SerdeBincode<Vec<RankingRule>>>(reader, RANKING_RULES_KEY)?)
+    }
+
+    pub fn put_ranking_rules(self, writer: &mut heed::RwTxn<MainT>, value: &[RankingRule]) -> MResult<()> {
+        Ok(self.main.put::<_, Str, SerdeBincode<Vec<RankingRule>>>(writer, RANKING_RULES_KEY, &value.to_vec())?)
+    }
+
+    pub fn delete_ranking_rules(self, writer: &mut heed::RwTxn<MainT>) -> MResult<bool> {
+        Ok(self.main.delete::<_, Str>(writer, RANKING_RULES_KEY)?)
+    }
+
+    pub fn distinct_attribute(&self, reader: &heed::RoTxn<MainT>) -> MResult<Option<FieldId>> {
+        match self.main.get::<_, Str, OwnedType<u16>>(reader, DISTINCT_ATTRIBUTE_KEY)? {
+            Some(value) => Ok(Some(FieldId(value.to_owned()))),
+            None => Ok(None),
         }
-        return Ok(None)
     }
 
-    pub fn put_distinct_attribute(self, writer: &mut heed::RwTxn<MainT>, value: &str) -> ZResult<()> {
-        self.main.put::<_, Str, Str>(writer, DISTINCT_ATTRIBUTE_KEY, value)
+    pub fn put_distinct_attribute(self, writer: &mut heed::RwTxn<MainT>, value: FieldId) -> MResult<()> {
+        Ok(self.main.put::<_, Str, OwnedType<u16>>(writer, DISTINCT_ATTRIBUTE_KEY, &value.0)?)
     }
 
-    pub fn delete_distinct_attribute(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<bool> {
-        self.main.delete::<_, Str>(writer, DISTINCT_ATTRIBUTE_KEY)
+    pub fn delete_distinct_attribute(self, writer: &mut heed::RwTxn<MainT>) -> MResult<bool> {
+        Ok(self.main.delete::<_, Str>(writer, DISTINCT_ATTRIBUTE_KEY)?)
     }
 
-    pub fn put_customs(self, writer: &mut heed::RwTxn<MainT>, customs: &[u8]) -> ZResult<()> {
-        self.main
-            .put::<_, Str, ByteSlice>(writer, CUSTOMS_KEY, customs)
+    pub fn put_customs(self, writer: &mut heed::RwTxn<MainT>, customs: &[u8]) -> MResult<()> {
+        Ok(self.main.put::<_, Str, ByteSlice>(writer, CUSTOMS_KEY, customs)?)
     }
 
-    pub fn customs<'txn>(self, reader: &'txn heed::RoTxn<MainT>) -> ZResult<Option<&'txn [u8]>> {
-        self.main.get::<_, Str, ByteSlice>(reader, CUSTOMS_KEY)
+    pub fn customs<'txn>(self, reader: &'txn heed::RoTxn<MainT>) -> MResult<Option<&'txn [u8]>> {
+        Ok(self.main.get::<_, Str, ByteSlice>(reader, CUSTOMS_KEY)?)
     }
 }

meilisearch-core/src/store/mod.rs

@@ -1,23 +1,27 @@
+mod cow_set;
 mod docs_words;
-mod prefix_documents_cache;
-mod prefix_postings_lists_cache;
+mod documents_ids;
 mod documents_fields;
 mod documents_fields_counts;
+mod facets;
 mod main;
 mod postings_lists;
+mod prefix_documents_cache;
+mod prefix_postings_lists_cache;
 mod synonyms;
 mod updates;
 mod updates_results;
 
+pub use self::cow_set::CowSet;
 pub use self::docs_words::DocsWords;
-pub use self::prefix_documents_cache::PrefixDocumentsCache;
-pub use self::prefix_postings_lists_cache::PrefixPostingsListsCache;
 pub use self::documents_fields::{DocumentFieldsIter, DocumentsFields};
-pub use self::documents_fields_counts::{
-    DocumentFieldsCountsIter, DocumentsFieldsCounts, DocumentsIdsIter,
-};
+pub use self::documents_fields_counts::{DocumentFieldsCountsIter, DocumentsFieldsCounts, DocumentsIdsIter};
+pub use self::documents_ids::{DocumentsIds, DiscoverIds};
+pub use self::facets::Facets;
 pub use self::main::Main;
 pub use self::postings_lists::PostingsLists;
+pub use self::prefix_documents_cache::PrefixDocumentsCache;
+pub use self::prefix_postings_lists_cache::PrefixPostingsListsCache;
 pub use self::synonyms::Synonyms;
 pub use self::updates::Updates;
 pub use self::updates_results::UpdatesResults;
@@ -27,7 +31,6 @@ use std::collections::HashSet;
 use std::convert::TryInto;
 use std::{mem, ptr};
 
-use heed::Result as ZResult;
 use heed::{BytesEncode, BytesDecode};
 use meilisearch_schema::{IndexedPos, FieldId};
 use sdset::{Set, SetBuf};
@@ -41,20 +44,21 @@ use crate::serde::Deserializer;
 use crate::settings::SettingsUpdate;
 use crate::{query_builder::QueryBuilder, update, DocIndex, DocumentId, Error, MResult};
 
+type BEU32 = zerocopy::U32<byteorder::BigEndian>;
 type BEU64 = zerocopy::U64<byteorder::BigEndian>;
-type BEU16 = zerocopy::U16<byteorder::BigEndian>;
+pub type BEU16 = zerocopy::U16<byteorder::BigEndian>;
 
 #[derive(Debug, Copy, Clone, AsBytes, FromBytes)]
 #[repr(C)]
 pub struct DocumentFieldIndexedKey {
-    docid: BEU64,
+    docid: BEU32,
     indexed_pos: BEU16,
 }
 
 impl DocumentFieldIndexedKey {
     fn new(docid: DocumentId, indexed_pos: IndexedPos) -> DocumentFieldIndexedKey {
         DocumentFieldIndexedKey {
-            docid: BEU64::new(docid.0),
+            docid: BEU32::new(docid.0),
             indexed_pos: BEU16::new(indexed_pos.0),
         }
     }
@@ -63,14 +67,14 @@ impl DocumentFieldIndexedKey {
 #[derive(Debug, Copy, Clone, AsBytes, FromBytes)]
 #[repr(C)]
 pub struct DocumentFieldStoredKey {
-    docid: BEU64,
+    docid: BEU32,
     field_id: BEU16,
 }
 
 impl DocumentFieldStoredKey {
     fn new(docid: DocumentId, field_id: FieldId) -> DocumentFieldStoredKey {
         DocumentFieldStoredKey {
-            docid: BEU64::new(docid.0),
+            docid: BEU32::new(docid.0),
             field_id: BEU16::new(field_id.0),
         }
     }
@@ -94,7 +98,7 @@ impl<'a> BytesEncode<'a> for PostingsCodec {
 
         let mut buffer = Vec::with_capacity(u64_size + docids_size + matches_size);
 
-        let docids_len = item.docids.len();
+        let docids_len = item.docids.len() as u64;
         buffer.extend_from_slice(&docids_len.to_be_bytes());
         buffer.extend_from_slice(item.docids.as_bytes());
         buffer.extend_from_slice(item.matches.as_bytes());
@@ -197,12 +201,17 @@ fn updates_results_name(name: &str) -> String {
     format!("store-{}-updates-results", name)
 }
 
+fn facets_name(name: &str) -> String {
+    format!("store-{}-facets", name)
+}
+
 #[derive(Clone)]
 pub struct Index {
     pub main: Main,
     pub postings_lists: PostingsLists,
     pub documents_fields: DocumentsFields,
     pub documents_fields_counts: DocumentsFieldsCounts,
+    pub facets: Facets,
     pub synonyms: Synonyms,
     pub docs_words: DocsWords,
     pub prefix_documents_cache: PrefixDocumentsCache,
@@ -269,14 +278,14 @@ impl Index {
         }
     }
 
-    pub fn customs_update(&self, writer: &mut heed::RwTxn<UpdateT>, customs: Vec<u8>) -> ZResult<u64> {
+    pub fn customs_update(&self, writer: &mut heed::RwTxn<UpdateT>, customs: Vec<u8>) -> MResult<u64> {
         let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        update::push_customs_update(writer, self.updates, self.updates_results, customs)
+        Ok(update::push_customs_update(writer, self.updates, self.updates_results, customs)?)
     }
 
-    pub fn settings_update(&self, writer: &mut heed::RwTxn<UpdateT>, update: SettingsUpdate) -> ZResult<u64> {
+    pub fn settings_update(&self, writer: &mut heed::RwTxn<UpdateT>, update: SettingsUpdate) -> MResult<u64> {
         let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        update::push_settings_update(writer, self.updates, self.updates_results, update)
+        Ok(update::push_settings_update(writer, self.updates, self.updates_results, update)?)
     }
 
     pub fn documents_addition<D>(&self) -> update::DocumentsAddition<D> {
@@ -352,29 +361,14 @@ impl Index {
     }
 
     pub fn query_builder(&self) -> QueryBuilder {
-        QueryBuilder::new(
-            self.main,
-            self.postings_lists,
-            self.documents_fields_counts,
-            self.synonyms,
-            self.prefix_documents_cache,
-            self.prefix_postings_lists_cache,
-        )
+        QueryBuilder::new(self)
     }
 
-    pub fn query_builder_with_criteria<'c, 'f, 'd>(
-        &self,
+    pub fn query_builder_with_criteria<'c, 'f, 'd, 'i>(
+        &'i self,
         criteria: Criteria<'c>,
-    ) -> QueryBuilder<'c, 'f, 'd> {
-        QueryBuilder::with_criteria(
-            self.main,
-            self.postings_lists,
-            self.documents_fields_counts,
-            self.synonyms,
-            self.prefix_documents_cache,
-            self.prefix_postings_lists_cache,
-            criteria,
-        )
+    ) -> QueryBuilder<'c, 'f, 'd, 'i> {
+        QueryBuilder::with_criteria(self, criteria)
     }
 }
 
@@ -395,12 +389,14 @@ pub fn create(
     let prefix_postings_lists_cache_name = prefix_postings_lists_cache_name(name);
     let updates_name = updates_name(name);
     let updates_results_name = updates_results_name(name);
+    let facets_name = facets_name(name);
 
     // open all the stores
     let main = env.create_poly_database(Some(&main_name))?;
     let postings_lists = env.create_database(Some(&postings_lists_name))?;
     let documents_fields = env.create_database(Some(&documents_fields_name))?;
     let documents_fields_counts = env.create_database(Some(&documents_fields_counts_name))?;
+    let facets = env.create_database(Some(&facets_name))?;
     let synonyms = env.create_database(Some(&synonyms_name))?;
     let docs_words = env.create_database(Some(&docs_words_name))?;
     let prefix_documents_cache = env.create_database(Some(&prefix_documents_cache_name))?;
@@ -417,6 +413,8 @@ pub fn create(
         docs_words: DocsWords { docs_words },
         prefix_postings_lists_cache: PrefixPostingsListsCache { prefix_postings_lists_cache },
         prefix_documents_cache: PrefixDocumentsCache { prefix_documents_cache },
+        facets: Facets { facets },
+
         updates: Updates { updates },
         updates_results: UpdatesResults { updates_results },
         updates_notifier,
@@ -437,6 +435,7 @@ pub fn open(
     let synonyms_name = synonyms_name(name);
     let docs_words_name = docs_words_name(name);
     let prefix_documents_cache_name = prefix_documents_cache_name(name);
+    let facets_name = facets_name(name);
     let prefix_postings_lists_cache_name = prefix_postings_lists_cache_name(name);
     let updates_name = updates_name(name);
     let updates_results_name = updates_results_name(name);
@@ -470,6 +469,10 @@ pub fn open(
         Some(prefix_documents_cache) => prefix_documents_cache,
         None => return Ok(None),
     };
+    let facets = match env.open_database(Some(&facets_name))? {
+        Some(facets) => facets,
+        None => return Ok(None),
+    };
     let prefix_postings_lists_cache = match env.open_database(Some(&prefix_postings_lists_cache_name))? {
         Some(prefix_postings_lists_cache) => prefix_postings_lists_cache,
         None => return Ok(None),
@@ -491,6 +494,7 @@ pub fn open(
         synonyms: Synonyms { synonyms },
         docs_words: DocsWords { docs_words },
         prefix_documents_cache: PrefixDocumentsCache { prefix_documents_cache },
+        facets: Facets { facets },
         prefix_postings_lists_cache: PrefixPostingsListsCache { prefix_postings_lists_cache },
         updates: Updates { updates },
         updates_results: UpdatesResults { updates_results },

meilisearch-core/src/store/prefix_documents_cache.rs

@@ -4,7 +4,7 @@ use heed::types::{OwnedType, CowSlice};
 use heed::Result as ZResult;
 use zerocopy::{AsBytes, FromBytes};
 
-use super::BEU64;
+use super::{BEU64, BEU32};
 use crate::{DocumentId, Highlight};
 use crate::database::MainT;
 
@@ -13,15 +13,15 @@ use crate::database::MainT;
 pub struct PrefixKey {
     prefix: [u8; 4],
     index: BEU64,
-    docid: BEU64,
+    docid: BEU32,
 }
 
 impl PrefixKey {
-    pub fn new(prefix: [u8; 4], index: u64, docid: u64) -> PrefixKey {
+    pub fn new(prefix: [u8; 4], index: u64, docid: u32) -> PrefixKey {
         PrefixKey {
             prefix,
             index: BEU64::new(index),
-            docid: BEU64::new(docid),
+            docid: BEU32::new(docid),
         }
     }
 }
@@ -54,7 +54,7 @@ impl PrefixDocumentsCache {
         prefix: [u8; 4],
     ) -> ZResult<PrefixDocumentsIter<'txn>> {
         let start = PrefixKey::new(prefix, 0, 0);
-        let end = PrefixKey::new(prefix, u64::max_value(), u64::max_value());
+        let end = PrefixKey::new(prefix, u64::max_value(), u32::max_value());
         let iter = self.prefix_documents_cache.range(reader, &(start..=end))?;
         Ok(PrefixDocumentsIter { iter })
     }

meilisearch-core/src/store/synonyms.rs

@@ -1,7 +1,10 @@
-use heed::types::ByteSlice;
-use crate::database::MainT;
+use std::borrow::Cow;
+
 use heed::Result as ZResult;
-use std::sync::Arc;
+use heed::types::ByteSlice;
+
+use crate::database::MainT;
+use crate::{FstSetCow, MResult};
 
 #[derive(Copy, Clone)]
 pub struct Synonyms {
@@ -9,12 +12,9 @@ pub struct Synonyms {
 }
 
 impl Synonyms {
-    pub fn put_synonyms(
-        self,
-        writer: &mut heed::RwTxn<MainT>,
-        word: &[u8],
-        synonyms: &fst::Set,
-    ) -> ZResult<()> {
+    pub fn put_synonyms<A>(self, writer: &mut heed::RwTxn<MainT>, word: &[u8], synonyms: &fst::Set<A>) -> ZResult<()>
+    where A: AsRef<[u8]>,
+    {
         let bytes = synonyms.as_fst().as_bytes();
         self.synonyms.put(writer, word, bytes)
     }
@@ -27,15 +27,18 @@ impl Synonyms {
         self.synonyms.clear(writer)
     }
 
-    pub fn synonyms(self, reader: &heed::RoTxn<MainT>, word: &[u8]) -> ZResult<Option<fst::Set>> {
+    pub(crate) fn synonyms_fst<'txn>(self, reader: &'txn heed::RoTxn<MainT>, word: &[u8]) -> ZResult<FstSetCow<'txn>> {
         match self.synonyms.get(reader, word)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
+            Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
+            None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()),
         }
     }
+
+    pub fn synonyms(self, reader: &heed::RoTxn<MainT>, word: &[u8]) -> MResult<Vec<String>> {
+        let synonyms = self
+            .synonyms_fst(&reader, word)?
+            .stream()
+            .into_strs()?;
+        Ok(synonyms)
+    }
 }

meilisearch-core/src/update/clear_all.rs

@@ -7,6 +7,8 @@ pub fn apply_clear_all(
     index: &store::Index,
 ) -> MResult<()> {
     index.main.put_words_fst(writer, &fst::Set::default())?;
+    index.main.put_external_docids(writer, &fst::Map::default())?;
+    index.main.put_internal_docids(writer, &sdset::SetBuf::default())?;
     index.main.put_ranked_map(writer, &RankedMap::default())?;
     index.main.put_number_of_documents(writer, |_| 0)?;
     index.documents_fields.clear(writer)?;

meilisearch-core/src/update/customs_update.rs

@@ -1,14 +1,13 @@
-use heed::Result as ZResult;
 
 use crate::database::{MainT, UpdateT};
-use crate::store;
+use crate::{store, MResult};
 use crate::update::{next_update_id, Update};
 
 pub fn apply_customs_update(
     writer: &mut heed::RwTxn<MainT>,
     main_store: store::Main,
     customs: &[u8],
-) -> ZResult<()> {
+) -> MResult<()> {
     main_store.put_customs(writer, customs)
 }
 
@@ -17,7 +16,7 @@ pub fn push_customs_update(
     updates_store: store::Updates,
     updates_results_store: store::UpdatesResults,
     customs: Vec<u8>,
-) -> ZResult<u64> {
+) -> MResult<u64> {
     let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
 
     let update = Update::customs(customs);

meilisearch-core/src/update/documents_addition.rs

@@ -1,15 +1,21 @@
-use std::collections::HashMap;
+use std::borrow::Cow;
+use std::collections::{HashMap, BTreeMap};
 
 use fst::{set::OpBuilder, SetBuilder};
 use indexmap::IndexMap;
+use meilisearch_schema::{Schema, FieldId};
+use meilisearch_types::DocumentId;
 use sdset::{duo::Union, SetOperation};
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;
+use serde_json::Value;
 
 use crate::database::{MainT, UpdateT};
 use crate::database::{UpdateEvent, UpdateEventsEmitter};
+use crate::facets;
 use crate::raw_indexer::RawIndexer;
-use crate::serde::{extract_document_id, serialize_value_with_id, Deserializer, Serializer};
-use crate::store;
+use crate::serde::Deserializer;
+use crate::store::{self, DocumentsFields, DocumentsFieldsCounts, DiscoverIds};
+use crate::update::helpers::{index_value, value_to_number, extract_document_id};
 use crate::update::{apply_documents_deletion, compute_short_prefixes, next_update_id, Update};
 use crate::{Error, MResult, RankedMap};
 
@@ -103,33 +109,109 @@ pub fn push_documents_addition<D: serde::Serialize>(
     Ok(last_update_id)
 }
 
-pub fn apply_documents_addition<'a, 'b>(
+#[allow(clippy::too_many_arguments)]
+fn index_document<A>(
+    writer: &mut heed::RwTxn<MainT>,
+    documents_fields: DocumentsFields,
+    documents_fields_counts: DocumentsFieldsCounts,
+    ranked_map: &mut RankedMap,
+    indexer: &mut RawIndexer<A>,
+    schema: &Schema,
+    field_id: FieldId,
+    document_id: DocumentId,
+    value: &Value,
+) -> MResult<()>
+where A: AsRef<[u8]>,
+{
+    let serialized = serde_json::to_vec(value)?;
+    documents_fields.put_document_field(writer, document_id, field_id, &serialized)?;
+
+    if let Some(indexed_pos) = schema.is_indexed(field_id) {
+        let number_of_words = index_value(indexer, document_id, *indexed_pos, value);
+        if let Some(number_of_words) = number_of_words {
+            documents_fields_counts.put_document_field_count(
+                writer,
+                document_id,
+                *indexed_pos,
+                number_of_words as u16,
+            )?;
+        }
+    }
+
+    if schema.is_ranked(field_id) {
+        let number = value_to_number(value).unwrap_or_default();
+        ranked_map.insert(document_id, field_id, number);
+    }
+
+    Ok(())
+}
+
+pub fn apply_addition<'a, 'b>(
     writer: &'a mut heed::RwTxn<'b, MainT>,
     index: &store::Index,
-    addition: Vec<IndexMap<String, serde_json::Value>>,
-) -> MResult<()> {
-    let mut documents_additions = HashMap::new();
-
+    new_documents: Vec<IndexMap<String, Value>>,
+    partial: bool
+) -> MResult<()>
+{
     let mut schema = match index.main.schema(writer)? {
         Some(schema) => schema,
         None => return Err(Error::SchemaMissing),
     };
 
+    // Retrieve the documents ids related structures
+    let external_docids = index.main.external_docids(writer)?;
+    let internal_docids = index.main.internal_docids(writer)?;
+    let mut available_ids = DiscoverIds::new(&internal_docids);
+
     let primary_key = schema.primary_key().ok_or(Error::MissingPrimaryKey)?;
 
     // 1. store documents ids for future deletion
-    for document in addition {
-        let document_id = match extract_document_id(&primary_key, &document)? {
-            Some(id) => id,
-            None => return Err(Error::MissingDocumentId),
+    let mut documents_additions = HashMap::new();
+    let mut new_external_docids = BTreeMap::new();
+    let mut new_internal_docids = Vec::with_capacity(new_documents.len());
+
+    for mut document in new_documents {
+        let external_docids_get = |docid: &str| {
+            match (external_docids.get(docid), new_external_docids.get(docid)) {
+                (_, Some(&id))
+                | (Some(id), _) => Some(id as u32),
+                (None, None) => None,
+            }
         };
 
-        documents_additions.insert(document_id, document);
+        let (internal_docid, external_docid) =
+            extract_document_id(
+                &primary_key,
+                &document,
+                &external_docids_get,
+                &mut available_ids,
+            )?;
+
+        new_external_docids.insert(external_docid, internal_docid.0 as u64);
+        new_internal_docids.push(internal_docid);
+
+        if partial {
+            let mut deserializer = Deserializer {
+                document_id: internal_docid,
+                reader: writer,
+                documents_fields: index.documents_fields,
+                schema: &schema,
+                fields: None,
+            };
+
+            let old_document = Option::<HashMap<String, Value>>::deserialize(&mut deserializer)?;
+            if let Some(old_document) = old_document {
+                for (key, value) in old_document {
+                    document.entry(key).or_insert(value);
+                }
+            }
+        }
+        documents_additions.insert(internal_docid, document);
     }
 
-    // 2. remove the documents posting lists
+    // 2. remove the documents postings lists
     let number_of_inserted_documents = documents_additions.len();
-    let documents_ids = documents_additions.iter().map(|(id, _)| *id).collect();
+    let documents_ids = new_external_docids.iter().map(|(id, _)| id.clone()).collect();
     apply_documents_deletion(writer, index, documents_ids)?;
 
     let mut ranked_map = match index.main.ranked_map(writer)? {
@@ -137,26 +219,28 @@ pub fn apply_documents_addition<'a, 'b>(
         None => RankedMap::default(),
     };
 
-    let stop_words = match index.main.stop_words_fst(writer)? {
-        Some(stop_words) => stop_words,
-        None => fst::Set::default(),
-    };
+    let stop_words = index.main.stop_words_fst(writer)?.map_data(Cow::into_owned)?;
+
 
-    // 3. index the documents fields in the stores
     let mut indexer = RawIndexer::new(stop_words);
 
-    for (document_id, document) in documents_additions {
-        let serializer = Serializer {
-            txn: writer,
-            schema: &mut schema,
-            document_store: index.documents_fields,
-            document_fields_counts: index.documents_fields_counts,
-            indexer: &mut indexer,
-            ranked_map: &mut ranked_map,
-            document_id,
-        };
-
-        document.serialize(serializer)?;
+    // For each document in this update
+    for (document_id, document) in &documents_additions {
+        // For each key-value pair in the document.
+        for (attribute, value) in document {
+            let field_id = schema.insert_and_index(&attribute)?;
+            index_document(
+                writer,
+                index.documents_fields,
+                index.documents_fields_counts,
+                &mut ranked_map,
+                &mut indexer,
+                &schema,
+                field_id,
+                *document_id,
+                &value,
+            )?;
+        }
     }
 
     write_documents_addition_index(
@@ -169,93 +253,39 @@ pub fn apply_documents_addition<'a, 'b>(
 
     index.main.put_schema(writer, &schema)?;
 
+    let new_external_docids = fst::Map::from_iter(new_external_docids.iter().map(|(ext, id)| (ext, *id as u64)))?;
+    let new_internal_docids = sdset::SetBuf::from_dirty(new_internal_docids);
+    index.main.merge_external_docids(writer, &new_external_docids)?;
+    index.main.merge_internal_docids(writer, &new_internal_docids)?;
+
+    // recompute all facet attributes after document update.
+    if let Some(attributes_for_facetting) = index.main.attributes_for_faceting(writer)? {
+        let docids = index.main.internal_docids(writer)?;
+        let facet_map = facets::facet_map_from_docids(writer, index, &docids, attributes_for_facetting.as_ref())?;
+        index.facets.add(writer, facet_map)?;
+    }
+
+    // update is finished; update sorted document id cache with new state
+    let mut document_ids = index.main.internal_docids(writer)?.to_vec();
+    super::cache_document_ids_sorted(writer, &ranked_map, index, &mut document_ids)?;
+
     Ok(())
 }
 
 pub fn apply_documents_partial_addition<'a, 'b>(
     writer: &'a mut heed::RwTxn<'b, MainT>,
     index: &store::Index,
-    addition: Vec<IndexMap<String, serde_json::Value>>,
+    new_documents: Vec<IndexMap<String, Value>>,
 ) -> MResult<()> {
-    let mut documents_additions = HashMap::new();
+    apply_addition(writer, index, new_documents, true)
+}
 
-    let mut schema = match index.main.schema(writer)? {
-        Some(schema) => schema,
-        None => return Err(Error::SchemaMissing),
-    };
-
-    let primary_key = schema.primary_key().ok_or(Error::MissingPrimaryKey)?;
-
-    // 1. store documents ids for future deletion
-    for mut document in addition {
-        let document_id = match extract_document_id(&primary_key, &document)? {
-            Some(id) => id,
-            None => return Err(Error::MissingDocumentId),
-        };
-
-        let mut deserializer = Deserializer {
-            document_id,
-            reader: writer,
-            documents_fields: index.documents_fields,
-            schema: &schema,
-            fields: None,
-        };
-
-        // retrieve the old document and
-        // update the new one with missing keys found in the old one
-        let result = Option::<HashMap<String, serde_json::Value>>::deserialize(&mut deserializer)?;
-        if let Some(old_document) = result {
-            for (key, value) in old_document {
-                document.entry(key).or_insert(value);
-            }
-        }
-
-        documents_additions.insert(document_id, document);
-    }
-
-    // 2. remove the documents posting lists
-    let number_of_inserted_documents = documents_additions.len();
-    let documents_ids = documents_additions.iter().map(|(id, _)| *id).collect();
-    apply_documents_deletion(writer, index, documents_ids)?;
-
-    let mut ranked_map = match index.main.ranked_map(writer)? {
-        Some(ranked_map) => ranked_map,
-        None => RankedMap::default(),
-    };
-
-    let stop_words = match index.main.stop_words_fst(writer)? {
-        Some(stop_words) => stop_words,
-        None => fst::Set::default(),
-    };
-
-    // 3. index the documents fields in the stores
-    let mut indexer = RawIndexer::new(stop_words);
-
-    for (document_id, document) in documents_additions {
-        let serializer = Serializer {
-            txn: writer,
-            schema: &mut schema,
-            document_store: index.documents_fields,
-            document_fields_counts: index.documents_fields_counts,
-            indexer: &mut indexer,
-            ranked_map: &mut ranked_map,
-            document_id,
-        };
-
-        document.serialize(serializer)?;
-    }
-
-    write_documents_addition_index(
-        writer,
-        index,
-        &ranked_map,
-        number_of_inserted_documents,
-        indexer,
-    )?;
-
-    index.main.put_schema(writer, &schema)?;
-
-    Ok(())
+pub fn apply_documents_addition<'a, 'b>(
+    writer: &'a mut heed::RwTxn<'b, MainT>,
+    index: &store::Index,
+    new_documents: Vec<IndexMap<String, Value>>,
+) -> MResult<()> {
+    apply_addition(writer, index, new_documents, false)
 }
 
 pub fn reindex_all_documents(writer: &mut heed::RwTxn<MainT>, index: &store::Index) -> MResult<()> {
@@ -277,36 +307,43 @@ pub fn reindex_all_documents(writer: &mut heed::RwTxn<MainT>, index: &store::Ind
     index.main.put_words_fst(writer, &fst::Set::default())?;
     index.main.put_ranked_map(writer, &ranked_map)?;
     index.main.put_number_of_documents(writer, |_| 0)?;
+    index.facets.clear(writer)?;
     index.postings_lists.clear(writer)?;
     index.docs_words.clear(writer)?;
 
-    let stop_words = match index.main.stop_words_fst(writer)? {
-        Some(stop_words) => stop_words,
-        None => fst::Set::default(),
-    };
+    let stop_words = index.main
+        .stop_words_fst(writer)?
+        .map_data(Cow::into_owned)
+        .unwrap();
 
     let number_of_inserted_documents = documents_ids_to_reindex.len();
     let mut indexer = RawIndexer::new(stop_words);
     let mut ram_store = HashMap::new();
 
-    for document_id in documents_ids_to_reindex {
-        for result in index.documents_fields.document_fields(writer, document_id)? {
+    if let Some(ref attributes_for_facetting) = index.main.attributes_for_faceting(writer)? {
+        let facet_map = facets::facet_map_from_docids(writer, &index, &documents_ids_to_reindex, &attributes_for_facetting)?;
+        index.facets.add(writer, facet_map)?;
+    }
+    // ^-- https://github.com/meilisearch/MeiliSearch/pull/631#issuecomment-626624470 --v
+    for document_id in &documents_ids_to_reindex {
+        for result in index.documents_fields.document_fields(writer, *document_id)? {
             let (field_id, bytes) = result?;
-            let value: serde_json::Value = serde_json::from_slice(bytes)?;
+            let value: Value = serde_json::from_slice(bytes)?;
             ram_store.insert((document_id, field_id), value);
         }
 
-        for ((docid, field_id), value) in ram_store.drain() {
-            serialize_value_with_id(
+        // For each key-value pair in the document.
+        for ((document_id, field_id), value) in ram_store.drain() {
+            index_document(
                 writer,
-                field_id,
-                &schema,
-                docid,
                 index.documents_fields,
                 index.documents_fields_counts,
-                &mut indexer,
                 &mut ranked_map,
-                &value
+                &mut indexer,
+                &schema,
+                field_id,
+                *document_id,
+                &value,
             )?;
         }
     }
@@ -322,16 +359,29 @@ pub fn reindex_all_documents(writer: &mut heed::RwTxn<MainT>, index: &store::Ind
 
     index.main.put_schema(writer, &schema)?;
 
+    // recompute all facet attributes after document update.
+    if let Some(attributes_for_facetting) = index.main.attributes_for_faceting(writer)? {
+        let docids = index.main.internal_docids(writer)?;
+        let facet_map = facets::facet_map_from_docids(writer, index, &docids, attributes_for_facetting.as_ref())?;
+        index.facets.add(writer, facet_map)?;
+    }
+
+    // update is finished; update sorted document id cache with new state
+    let mut document_ids = index.main.internal_docids(writer)?.to_vec();
+    super::cache_document_ids_sorted(writer, &ranked_map, index, &mut document_ids)?;
+
     Ok(())
 }
 
-pub fn write_documents_addition_index(
+pub fn write_documents_addition_index<A>(
     writer: &mut heed::RwTxn<MainT>,
     index: &store::Index,
     ranked_map: &RankedMap,
     number_of_inserted_documents: usize,
-    indexer: RawIndexer,
-) -> MResult<()> {
+    indexer: RawIndexer<A>,
+) -> MResult<()>
+where A: AsRef<[u8]>,
+{
     let indexed = indexer.build();
     let mut delta_words_builder = SetBuilder::memory();
 
@@ -350,33 +400,27 @@ pub fn write_documents_addition_index(
         index.docs_words.put_doc_words(writer, id, &words)?;
     }
 
-    let delta_words = delta_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
+    let delta_words = delta_words_builder.into_set();
 
-    let words = match index.main.words_fst(writer)? {
-        Some(words) => {
-            let op = OpBuilder::new()
-                .add(words.stream())
-                .add(delta_words.stream())
-                .r#union();
+    let words_fst = index.main.words_fst(writer)?;
+    let words = if !words_fst.is_empty() {
+        let op = OpBuilder::new()
+            .add(words_fst.stream())
+            .add(delta_words.stream())
+            .r#union();
 
-            let mut words_builder = SetBuilder::memory();
-            words_builder.extend_stream(op).unwrap();
-            words_builder
-                .into_inner()
-                .and_then(fst::Set::from_bytes)
-                .unwrap()
-        }
-        None => delta_words,
+        let mut words_builder = SetBuilder::memory();
+        words_builder.extend_stream(op).unwrap();
+        words_builder.into_set()
+    } else {
+        delta_words
     };
 
     index.main.put_words_fst(writer, &words)?;
     index.main.put_ranked_map(writer, ranked_map)?;
     index.main.put_number_of_documents(writer, |old| old + number_of_inserted_documents as u64)?;
 
-    compute_short_prefixes(writer, index)?;
+    compute_short_prefixes(writer, &words, index)?;
 
     Ok(())
 }

meilisearch-core/src/update/documents_deletion.rs

@@ -1,21 +1,20 @@
 use std::collections::{BTreeSet, HashMap, HashSet};
 
 use fst::{SetBuilder, Streamer};
-use meilisearch_schema::Schema;
 use sdset::{duo::DifferenceByKey, SetBuf, SetOperation};
 
 use crate::database::{MainT, UpdateT};
 use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::serde::extract_document_id;
+use crate::facets;
 use crate::store;
 use crate::update::{next_update_id, compute_short_prefixes, Update};
-use crate::{DocumentId, Error, MResult, RankedMap};
+use crate::{DocumentId, Error, MResult, RankedMap, MainWriter, Index};
 
 pub struct DocumentsDeletion {
     updates_store: store::Updates,
     updates_results_store: store::UpdatesResults,
     updates_notifier: UpdateEventsEmitter,
-    documents: Vec<DocumentId>,
+    external_docids: Vec<String>,
 }
 
 impl DocumentsDeletion {
@@ -28,27 +27,12 @@ impl DocumentsDeletion {
             updates_store,
             updates_results_store,
             updates_notifier,
-            documents: Vec::new(),
+            external_docids: Vec::new(),
         }
     }
 
-    pub fn delete_document_by_id(&mut self, document_id: DocumentId) {
-        self.documents.push(document_id);
-    }
-
-    pub fn delete_document<D>(&mut self, schema: &Schema, document: D) -> MResult<()>
-    where
-        D: serde::Serialize,
-    {
-        let primary_key = schema.primary_key().ok_or(Error::MissingPrimaryKey)?;
-        let document_id = match extract_document_id(&primary_key, &document)? {
-            Some(id) => id,
-            None => return Err(Error::MissingDocumentId),
-        };
-
-        self.delete_document_by_id(document_id);
-
-        Ok(())
+    pub fn delete_document_by_external_docid(&mut self, document_id: String) {
+        self.external_docids.push(document_id);
     }
 
     pub fn finalize(self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64> {
@@ -57,15 +41,15 @@ impl DocumentsDeletion {
             writer,
             self.updates_store,
             self.updates_results_store,
-            self.documents,
+            self.external_docids,
         )?;
         Ok(update_id)
     }
 }
 
-impl Extend<DocumentId> for DocumentsDeletion {
-    fn extend<T: IntoIterator<Item = DocumentId>>(&mut self, iter: T) {
-        self.documents.extend(iter)
+impl Extend<String> for DocumentsDeletion {
+    fn extend<T: IntoIterator<Item=String>>(&mut self, iter: T) {
+        self.external_docids.extend(iter)
     }
 }
 
@@ -73,11 +57,11 @@ pub fn push_documents_deletion(
     writer: &mut heed::RwTxn<UpdateT>,
     updates_store: store::Updates,
     updates_results_store: store::UpdatesResults,
-    deletion: Vec<DocumentId>,
+    external_docids: Vec<String>,
 ) -> MResult<u64> {
     let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
 
-    let update = Update::documents_deletion(deletion);
+    let update = Update::documents_deletion(external_docids);
     updates_store.put_update(writer, last_update_id, &update)?;
 
     Ok(last_update_id)
@@ -86,9 +70,23 @@ pub fn push_documents_deletion(
 pub fn apply_documents_deletion(
     writer: &mut heed::RwTxn<MainT>,
     index: &store::Index,
-    deletion: Vec<DocumentId>,
-) -> MResult<()> {
-    let idset = SetBuf::from_dirty(deletion);
+    external_docids: Vec<String>,
+) -> MResult<()>
+{
+    let (external_docids, internal_docids) = {
+        let new_external_docids = SetBuf::from_dirty(external_docids);
+        let mut internal_docids = Vec::new();
+
+        let old_external_docids = index.main.external_docids(writer)?;
+        for external_docid in new_external_docids.as_slice() {
+            if let Some(id) = old_external_docids.get(external_docid) {
+                internal_docids.push(DocumentId(id as u32));
+            }
+        }
+
+        let new_external_docids = fst::Map::from_iter(new_external_docids.into_iter().map(|k| (k, 0))).unwrap();
+        (new_external_docids, SetBuf::from_dirty(internal_docids))
+    };
 
     let schema = match index.main.schema(writer)? {
         Some(schema) => schema,
@@ -100,17 +98,24 @@ pub fn apply_documents_deletion(
         None => RankedMap::default(),
     };
 
+    // facet filters deletion
+    if let Some(attributes_for_facetting) = index.main.attributes_for_faceting(writer)? {
+        let facet_map = facets::facet_map_from_docids(writer, &index, &internal_docids, &attributes_for_facetting)?;
+        index.facets.remove(writer, facet_map)?;
+    }
+
     // collect the ranked attributes according to the schema
     let ranked_fields = schema.ranked();
 
     let mut words_document_ids = HashMap::new();
-    for id in idset {
+    for id in internal_docids.iter().cloned() {
         // remove all the ranked attributes from the ranked_map
         for ranked_attr in ranked_fields {
             ranked_map.remove(id, *ranked_attr);
         }
 
-        if let Some(words) = index.docs_words.doc_words(writer, id)? {
+        let words = index.docs_words.doc_words(writer, id)?;
+        if !words.is_empty() {
             let mut stream = words.stream();
             while let Some(word) = stream.next() {
                 let word = word.to_vec();
@@ -148,33 +153,55 @@ pub fn apply_documents_deletion(
     }
 
     let deleted_documents_len = deleted_documents.len() as u64;
-    for id in deleted_documents {
-        index.docs_words.del_doc_words(writer, id)?;
+    for id in &deleted_documents {
+        index.docs_words.del_doc_words(writer, *id)?;
     }
 
     let removed_words = fst::Set::from_iter(removed_words).unwrap();
-    let words = match index.main.words_fst(writer)? {
-        Some(words_set) => {
-            let op = fst::set::OpBuilder::new()
-                .add(words_set.stream())
-                .add(removed_words.stream())
-                .difference();
+    let words = {
+        let words_set = index.main.words_fst(writer)?;
+        let op = fst::set::OpBuilder::new()
+            .add(words_set.stream())
+            .add(removed_words.stream())
+            .difference();
 
-            let mut words_builder = SetBuilder::memory();
-            words_builder.extend_stream(op).unwrap();
-            words_builder
-                .into_inner()
-                .and_then(fst::Set::from_bytes)
-                .unwrap()
-        }
-        None => fst::Set::default(),
+        let mut words_builder = SetBuilder::memory();
+        words_builder.extend_stream(op).unwrap();
+        words_builder.into_set()
     };
 
     index.main.put_words_fst(writer, &words)?;
     index.main.put_ranked_map(writer, &ranked_map)?;
     index.main.put_number_of_documents(writer, |old| old - deleted_documents_len)?;
 
-    compute_short_prefixes(writer, index)?;
+    // We apply the changes to the user and internal ids
+    index.main.remove_external_docids(writer, &external_docids)?;
+    index.main.remove_internal_docids(writer, &internal_docids)?;
+
+    compute_short_prefixes(writer, &words, index)?;
+
+    // update is finished; update sorted document id cache with new state
+    document_cache_remove_deleted(writer, index, &ranked_map, &deleted_documents)?;
 
     Ok(())
 }
+
+/// rebuilds the document id cache by either removing deleted documents from the existing cache,
+/// and generating a new one from docs in store
+fn document_cache_remove_deleted(writer: &mut MainWriter, index: &Index, ranked_map: &RankedMap, documents_to_delete: &HashSet<DocumentId>) -> MResult<()> {
+    let new_cache = match index.main.sorted_document_ids_cache(writer)? {
+        // only keep documents that are not in the list of deleted documents. Order is preserved,
+        // no need to resort
+        Some(old_cache) => {
+            old_cache.iter().filter(|docid| !documents_to_delete.contains(docid)).cloned().collect::<Vec<_>>()
+        }
+        // couldn't find cached documents, try building a new cache from documents in store
+        None => {
+            let mut document_ids = index.main.internal_docids(writer)?.to_vec();
+            super::cache_document_ids_sorted(writer, ranked_map, index, &mut document_ids)?;
+            document_ids
+        }
+    };
+    index.main.put_sorted_document_ids_cache(writer, &new_cache)?;
+    Ok(())
+}

meilisearch-core/src/update/helpers.rs

@@ -0,0 +1,143 @@
+use std::fmt::Write as _;
+
+use indexmap::IndexMap;
+use meilisearch_schema::IndexedPos;
+use meilisearch_types::DocumentId;
+use ordered_float::OrderedFloat;
+use serde_json::Value;
+
+use crate::Number;
+use crate::raw_indexer::RawIndexer;
+use crate::serde::SerializerError;
+use crate::store::DiscoverIds;
+
+/// Returns the number of words indexed or `None` if the type is unindexable.
+pub fn index_value<A>(
+    indexer: &mut RawIndexer<A>,
+    document_id: DocumentId,
+    indexed_pos: IndexedPos,
+    value: &Value,
+) -> Option<usize>
+where A: AsRef<[u8]>,
+{
+    match value {
+        Value::Null => None,
+        Value::Bool(boolean) => {
+            let text = boolean.to_string();
+            let number_of_words = indexer.index_text(document_id, indexed_pos, &text);
+            Some(number_of_words)
+        },
+        Value::Number(number) => {
+            let text = number.to_string();
+            Some(indexer.index_text(document_id, indexed_pos, &text))
+        },
+        Value::String(string) => {
+            Some(indexer.index_text(document_id, indexed_pos, &string))
+        },
+        Value::Array(_) => {
+            let text = value_to_string(value);
+            Some(indexer.index_text(document_id, indexed_pos, &text))
+        },
+        Value::Object(_) => {
+            let text = value_to_string(value);
+            Some(indexer.index_text(document_id, indexed_pos, &text))
+        },
+    }
+}
+
+/// Transforms the JSON Value type into a String.
+pub fn value_to_string(value: &Value) -> String {
+    fn internal_value_to_string(string: &mut String, value: &Value) {
+        match value {
+            Value::Null => (),
+            Value::Bool(boolean) => { let _ = write!(string, "{}", &boolean); },
+            Value::Number(number) => { let _ = write!(string, "{}", &number); },
+            Value::String(text) => string.push_str(&text),
+            Value::Array(array) => {
+                for value in array {
+                    internal_value_to_string(string, value);
+                    let _ = string.write_str(". ");
+                }
+            },
+            Value::Object(object) => {
+                for (key, value) in object {
+                    string.push_str(key);
+                    let _ = string.write_str(". ");
+                    internal_value_to_string(string, value);
+                    let _ = string.write_str(". ");
+                }
+            },
+        }
+    }
+
+    let mut string = String::new();
+    internal_value_to_string(&mut string, value);
+    string
+}
+
+/// Transforms the JSON Value type into a Number.
+pub fn value_to_number(value: &Value) -> Option<Number> {
+    use std::str::FromStr;
+
+    match value {
+        Value::Null => None,
+        Value::Bool(boolean) => Some(Number::Unsigned(*boolean as u64)),
+        Value::Number(number) => {
+            match (number.as_i64(), number.as_u64(), number.as_f64()) {
+                (Some(n), _, _) => Some(Number::Signed(n)),
+                (_, Some(n), _) => Some(Number::Unsigned(n)),
+                (_, _, Some(n)) => Some(Number::Float(OrderedFloat(n))),
+                (None, None, None) => None,
+            }
+        },
+        Value::String(string) => Number::from_str(string).ok(),
+        Value::Array(_array) => None,
+        Value::Object(_object) => None,
+    }
+}
+
+/// Validates a string representation to be a correct document id and returns
+/// the corresponding id or generate a new one, this is the way we produce documents ids.
+pub fn discover_document_id<F>(
+    docid: &str,
+    external_docids_get: F,
+    available_docids: &mut DiscoverIds<'_>,
+) -> Result<DocumentId, SerializerError>
+where
+    F: FnOnce(&str) -> Option<u32>
+{
+    if docid.chars().all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') {
+        match external_docids_get(docid) {
+            Some(id) => Ok(DocumentId(id)),
+            None => {
+                let internal_id = available_docids.next().expect("no more ids available");
+                Ok(internal_id)
+            },
+        }
+    } else {
+        Err(SerializerError::InvalidDocumentIdFormat)
+    }
+}
+
+/// Extracts and validates the document id of a document.
+pub fn extract_document_id<F>(
+    primary_key: &str,
+    document: &IndexMap<String, Value>,
+    external_docids_get: F,
+    available_docids: &mut DiscoverIds<'_>,
+) -> Result<(DocumentId, String), SerializerError>
+where
+    F: FnOnce(&str) -> Option<u32>
+{
+    match document.get(primary_key) {
+        Some(value) => {
+            let docid = match value {
+                Value::Number(number) => number.to_string(),
+                Value::String(string) => string.clone(),
+                _ => return Err(SerializerError::InvalidDocumentIdFormat),
+            };
+            discover_document_id(&docid, external_docids_get, available_docids).map(|id| (id, docid))
+        }
+        None => Err(SerializerError::DocumentIdNotFound),
+    }
+}

meilisearch-core/src/update/mod.rs

@@ -3,13 +3,13 @@ mod customs_update;
 mod documents_addition;
 mod documents_deletion;
 mod settings_update;
+mod helpers;
 
 pub use self::clear_all::{apply_clear_all, push_clear_all};
 pub use self::customs_update::{apply_customs_update, push_customs_update};
-pub use self::documents_addition::{
-    apply_documents_addition, apply_documents_partial_addition, DocumentsAddition,
-};
+pub use self::documents_addition::{apply_documents_addition, apply_documents_partial_addition, DocumentsAddition};
 pub use self::documents_deletion::{apply_documents_deletion, DocumentsDeletion};
+pub use self::helpers::{index_value, value_to_string, value_to_number, discover_document_id, extract_document_id};
 pub use self::settings_update::{apply_settings_update, push_settings_update};
 
 use std::cmp;
@@ -22,8 +22,12 @@ use indexmap::IndexMap;
 use log::debug;
 use sdset::Set;
 use serde::{Deserialize, Serialize};
+use serde_json::Value;
 
-use crate::{store, DocumentId, MResult};
+use meilisearch_error::ErrorCode;
+use meilisearch_types::DocumentId;
+
+use crate::{store, MResult, RankedMap};
 use crate::database::{MainT, UpdateT};
 use crate::settings::SettingsUpdate;
 
@@ -48,21 +52,21 @@ impl Update {
         }
     }
 
-    fn documents_addition(data: Vec<IndexMap<String, serde_json::Value>>) -> Update {
+    fn documents_addition(documents: Vec<IndexMap<String, Value>>) -> Update {
         Update {
-            data: UpdateData::DocumentsAddition(data),
+            data: UpdateData::DocumentsAddition(documents),
             enqueued_at: Utc::now(),
         }
     }
 
-    fn documents_partial(data: Vec<IndexMap<String, serde_json::Value>>) -> Update {
+    fn documents_partial(documents: Vec<IndexMap<String, Value>>) -> Update {
         Update {
-            data: UpdateData::DocumentsPartial(data),
+            data: UpdateData::DocumentsPartial(documents),
             enqueued_at: Utc::now(),
         }
     }
 
-    fn documents_deletion(data: Vec<DocumentId>) -> Update {
+    fn documents_deletion(data: Vec<String>) -> Update {
         Update {
             data: UpdateData::DocumentsDeletion(data),
             enqueued_at: Utc::now(),
@@ -71,7 +75,7 @@ impl Update {
 
     fn settings(data: SettingsUpdate) -> Update {
         Update {
-            data: UpdateData::Settings(data),
+            data: UpdateData::Settings(Box::new(data)),
             enqueued_at: Utc::now(),
         }
     }
@@ -81,10 +85,10 @@ impl Update {
 pub enum UpdateData {
     ClearAll,
     Customs(Vec<u8>),
-    DocumentsAddition(Vec<IndexMap<String, serde_json::Value>>),
-    DocumentsPartial(Vec<IndexMap<String, serde_json::Value>>),
-    DocumentsDeletion(Vec<DocumentId>),
-    Settings(SettingsUpdate)
+    DocumentsAddition(Vec<IndexMap<String, Value>>),
+    DocumentsPartial(Vec<IndexMap<String, Value>>),
+    DocumentsDeletion(Vec<String>),
+    Settings(Box<SettingsUpdate>)
 }
 
 impl UpdateData {
@@ -116,7 +120,7 @@ pub enum UpdateType {
     DocumentsAddition { number: usize },
     DocumentsPartial { number: usize },
     DocumentsDeletion { number: usize },
-    Settings { settings: SettingsUpdate },
+    Settings { settings: Box<SettingsUpdate> },
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -127,6 +131,12 @@ pub struct ProcessedUpdateResult {
     pub update_type: UpdateType,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub error: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error_type: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error_code: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error_link: Option<String>,
     pub duration: f64, // in seconds
     pub enqueued_at: DateTime<Utc>,
     pub processed_at: DateTime<Utc>,
@@ -272,7 +282,7 @@ pub fn update_task<'a, 'b>(
             let result = apply_settings_update(
                 writer,
                 index,
-                settings,
+                *settings,
             );
 
             (update_type, result, start.elapsed())
@@ -287,7 +297,10 @@ pub fn update_task<'a, 'b>(
     let status = ProcessedUpdateResult {
         update_id,
         update_type,
-        error: result.map_err(|e| e.to_string()).err(),
+        error: result.as_ref().map_err(|e| e.to_string()).err(),
+        error_code: result.as_ref().map_err(|e| e.error_name()).err(),
+        error_type: result.as_ref().map_err(|e| e.error_type()).err(),
+        error_link: result.as_ref().map_err(|e| e.error_url()).err(),
         duration: duration.as_secs_f64(),
         enqueued_at,
         processed_at: Utc::now(),
@@ -296,13 +309,13 @@ pub fn update_task<'a, 'b>(
     Ok(status)
 }
 
-fn compute_short_prefixes(writer: &mut heed::RwTxn<MainT>, index: &store::Index) -> MResult<()> {
-    // retrieve the words fst to compute all those prefixes
-    let words_fst = match index.main.words_fst(writer)? {
-        Some(fst) => fst,
-        None => return Ok(()),
-    };
-
+fn compute_short_prefixes<A>(
+    writer: &mut heed::RwTxn<MainT>,
+    words_fst: &fst::Set<A>,
+    index: &store::Index,
+) -> MResult<()>
+where A: AsRef<[u8]>,
+{
     // clear the prefixes
     let pplc_store = index.prefix_postings_lists_cache;
     pplc_store.clear(writer)?;
@@ -359,3 +372,13 @@ fn compute_short_prefixes(writer: &mut heed::RwTxn<MainT>, index: &store::Index)
 
     Ok(())
 }
+
+fn cache_document_ids_sorted(
+    writer: &mut heed::RwTxn<MainT>,
+    ranked_map: &RankedMap,
+    index: &store::Index,
+    document_ids: &mut [DocumentId],
+) -> MResult<()> {
+    crate::bucket_sort::placeholder_document_sort(document_ids, index, writer, ranked_map)?;
+    index.main.put_sorted_document_ids_cache(writer, &document_ids)
+}

meilisearch-core/src/update/settings_update.rs

@@ -46,12 +46,6 @@ pub fn apply_settings_update(
         UpdateState::Update(v) => {
             let ranked_field: Vec<&str> = v.iter().filter_map(RankingRule::field).collect();
             schema.update_ranked(&ranked_field)?;
-            for name in ranked_field {
-                if schema.accept_new_fields() {
-                    schema.set_indexed(name.as_ref())?;
-                    schema.set_displayed(name.as_ref())?;
-                }
-            }
             index.main.put_ranking_rules(writer, &v)?;
             must_reindex = true;
         },
@@ -65,7 +59,8 @@ pub fn apply_settings_update(
 
     match settings.distinct_attribute {
         UpdateState::Update(v) => {
-            index.main.put_distinct_attribute(writer, &v)?;
+            let field_id = schema.insert(&v)?;
+            index.main.put_distinct_attribute(writer, field_id)?;
         },
         UpdateState::Clear => {
             index.main.delete_distinct_attribute(writer)?;
@@ -73,19 +68,13 @@ pub fn apply_settings_update(
         UpdateState::Nothing => (),
     }
 
-    match settings.accept_new_fields {
-        UpdateState::Update(v) => {
-            schema.set_accept_new_fields(v);
-        },
-        UpdateState::Clear => {
-            schema.set_accept_new_fields(true);
-        },
-        UpdateState::Nothing => (),
-    }
-
     match settings.searchable_attributes.clone() {
         UpdateState::Update(v) => {
-            schema.update_indexed(v)?;
+            if v.iter().any(|e| e == "*") || v.is_empty() {
+                schema.set_all_fields_as_indexed();
+            } else {
+                schema.update_indexed(v)?;
+            }
             must_reindex = true;
         },
         UpdateState::Clear => {
@@ -95,13 +84,31 @@ pub fn apply_settings_update(
         UpdateState::Nothing => (),
     }
     match settings.displayed_attributes.clone() {
-        UpdateState::Update(v) => schema.update_displayed(v)?,
+        UpdateState::Update(v) => {
+            if v.contains("*") || v.is_empty() {
+                schema.set_all_fields_as_displayed();
+            } else {
+                schema.update_displayed(v)?
+            }
+        },
         UpdateState::Clear => {
             schema.set_all_fields_as_displayed();
         },
         UpdateState::Nothing => (),
     }
 
+    match settings.attributes_for_faceting {
+        UpdateState::Update(attrs) => {
+            apply_attributes_for_faceting_update(writer, index, &mut schema, &attrs)?;
+            must_reindex = true;
+        },
+        UpdateState::Clear => {
+            index.main.delete_attributes_for_faceting(writer)?;
+            index.facets.clear(writer)?;
+        },
+        UpdateState::Nothing => (),
+    }
+
     index.main.put_schema(writer, &schema)?;
 
     match settings.stop_words {
@@ -131,6 +138,21 @@ pub fn apply_settings_update(
     Ok(())
 }
 
+fn apply_attributes_for_faceting_update(
+    writer: &mut heed::RwTxn<MainT>,
+    index: &store::Index,
+    schema: &mut Schema,
+    attributes: &[String]
+    ) -> MResult<()> {
+    let mut attribute_ids = Vec::new();
+    for name in attributes {
+        attribute_ids.push(schema.insert(name)?);
+    }
+    let attributes_for_faceting = SetBuf::from_dirty(attribute_ids);
+    index.main.put_attributes_for_faceting(writer, &attributes_for_faceting)?;
+    Ok(())
+}
+
 pub fn apply_stop_words_update(
     writer: &mut heed::RwTxn<MainT>,
     index: &store::Index,
@@ -141,7 +163,6 @@ pub fn apply_stop_words_update(
 
     let old_stop_words: BTreeSet<String> = index.main
         .stop_words_fst(writer)?
-        .unwrap_or_default()
         .stream()
         .into_strs()?
         .into_iter()
@@ -159,7 +180,8 @@ pub fn apply_stop_words_update(
         apply_stop_words_deletion(writer, index, deletion)?;
     }
 
-    if let Some(words_fst) = index.main.words_fst(writer)? {
+    let words_fst = index.main.words_fst(writer)?;
+    if !words_fst.is_empty() {
         let stop_words = fst::Set::from_iter(stop_words)?;
         let op = OpBuilder::new()
             .add(&words_fst)
@@ -168,7 +190,7 @@ pub fn apply_stop_words_update(
 
         let mut builder = fst::SetBuilder::memory();
         builder.extend_stream(op)?;
-        let words_fst = builder.into_inner().and_then(fst::Set::from_bytes)?;
+        let words_fst = builder.into_set();
 
         index.main.put_words_fst(writer, &words_fst)?;
         index.main.put_stop_words_fst(writer, &stop_words)?;
@@ -195,28 +217,25 @@ fn apply_stop_words_addition(
     }
 
     // create the new delta stop words fst
-    let delta_stop_words = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)?;
+    let delta_stop_words = stop_words_builder.into_set();
 
     // we also need to remove all the stop words from the main fst
-    if let Some(word_fst) = main_store.words_fst(writer)? {
+    let words_fst = main_store.words_fst(writer)?;
+    if !words_fst.is_empty() {
         let op = OpBuilder::new()
-            .add(&word_fst)
+            .add(&words_fst)
             .add(&delta_stop_words)
             .difference();
 
         let mut word_fst_builder = SetBuilder::memory();
         word_fst_builder.extend_stream(op)?;
-        let word_fst = word_fst_builder
-            .into_inner()
-            .and_then(fst::Set::from_bytes)?;
+        let word_fst = word_fst_builder.into_set();
 
         main_store.put_words_fst(writer, &word_fst)?;
     }
 
     // now we add all of these stop words from the main store
-    let stop_words_fst = main_store.stop_words_fst(writer)?.unwrap_or_default();
+    let stop_words_fst = main_store.stop_words_fst(writer)?;
 
     let op = OpBuilder::new()
         .add(&stop_words_fst)
@@ -225,9 +244,7 @@ fn apply_stop_words_addition(
 
     let mut stop_words_builder = SetBuilder::memory();
     stop_words_builder.extend_stream(op)?;
-    let stop_words_fst = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)?;
+    let stop_words_fst = stop_words_builder.into_set();
 
     main_store.put_stop_words_fst(writer, &stop_words_fst)?;
 
@@ -247,12 +264,10 @@ fn apply_stop_words_deletion(
     }
 
     // create the new delta stop words fst
-    let delta_stop_words = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)?;
+    let delta_stop_words = stop_words_builder.into_set();
 
     // now we delete all of these stop words from the main store
-    let stop_words_fst = index.main.stop_words_fst(writer)?.unwrap_or_default();
+    let stop_words_fst = index.main.stop_words_fst(writer)?;
 
     let op = OpBuilder::new()
         .add(&stop_words_fst)
@@ -261,7 +276,7 @@ fn apply_stop_words_deletion(
 
     let mut stop_words_builder = SetBuilder::memory();
     stop_words_builder.extend_stream(op)?;
-    let stop_words_fst = stop_words_builder.into_inner().and_then(fst::Set::from_bytes)?;
+    let stop_words_fst = stop_words_builder.into_set();
 
     Ok(index.main.put_stop_words_fst(writer, &stop_words_fst)?)
 }
@@ -284,16 +299,13 @@ pub fn apply_synonyms_update(
             let alternatives = SetBuf::from_dirty(alternatives);
             let mut alternatives_builder = SetBuilder::memory();
             alternatives_builder.extend_iter(alternatives)?;
-            let bytes = alternatives_builder.into_inner()?;
-            fst::Set::from_bytes(bytes)?
+            alternatives_builder.into_set()
         };
 
         synonyms_store.put_synonyms(writer, word.as_bytes(), &alternatives)?;
     }
 
-    let synonyms_set = synonyms_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)?;
+    let synonyms_set = synonyms_builder.into_set();
 
     main_store.put_synonyms_fst(writer, &synonyms_set)?;
 

meilisearch-error/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "meilisearch-error"
+version = "0.13.0"
+authors = ["marin <postma.marin@protonmail.com>"]
+edition = "2018"
+
+[dependencies]
+actix-http = "1.0.1"

meilisearch-error/src/lib.rs

@@ -0,0 +1,180 @@
+use std::fmt;
+
+use actix_http::http::StatusCode;
+
+pub trait ErrorCode: std::error::Error {
+    fn error_code(&self) -> Code;
+
+    /// returns the HTTP status code ascociated with the error
+    fn http_status(&self) -> StatusCode {
+        self.error_code().http()
+    }
+
+    /// returns the doc url ascociated with the error
+    fn error_url(&self) -> String {
+        self.error_code().url()
+    }
+
+    /// returns error name, used as error code
+    fn error_name(&self) -> String {
+        self.error_code().name()
+    }
+
+    /// return the error type
+    fn error_type(&self) -> String {
+        self.error_code().type_()
+    }
+}
+
+#[allow(clippy::enum_variant_names)]
+enum ErrorType {
+    InternalError,
+    InvalidRequestError,
+    AuthenticationError,
+}
+
+impl fmt::Display for ErrorType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use ErrorType::*;
+
+        match self {
+            InternalError => write!(f, "internal_error"),
+            InvalidRequestError => write!(f, "invalid_request_error"),
+            AuthenticationError => write!(f, "authentication_error"),
+        }
+    }
+}
+
+pub enum Code {
+    // index related error
+    CreateIndex,
+    IndexAlreadyExists,
+    IndexNotFound,
+    InvalidIndexUid,
+    OpenIndex,
+
+    // invalid state error
+    InvalidState,
+    MissingPrimaryKey,
+    PrimaryKeyAlreadyPresent,
+
+    MaxFieldsLimitExceeded,
+    MissingDocumentId,
+
+    Facet,
+    Filter,
+
+    BadParameter,
+    BadRequest,
+    DocumentNotFound,
+    Internal,
+    InvalidToken,
+    Maintenance,
+    MissingAuthorizationHeader,
+    NotFound,
+    PayloadTooLarge,
+    RetrieveDocument,
+    SearchDocuments,
+    UnsupportedMediaType,
+}
+
+impl Code {
+
+    /// ascociate a `Code` variant to the actual ErrCode
+    fn err_code(&self) -> ErrCode {
+        use Code::*;
+
+        match self {
+            // index related errors
+            // create index is thrown on internal error while creating an index.
+            CreateIndex => ErrCode::internal("index_creation_failed", StatusCode::BAD_REQUEST),
+            IndexAlreadyExists => ErrCode::invalid("index_already_exists", StatusCode::BAD_REQUEST),
+            // thrown when requesting an unexisting index
+            IndexNotFound => ErrCode::invalid("index_not_found", StatusCode::NOT_FOUND),
+            InvalidIndexUid => ErrCode::invalid("invalid_index_uid", StatusCode::BAD_REQUEST),
+            OpenIndex => ErrCode::internal("index_not_accessible", StatusCode::INTERNAL_SERVER_ERROR),
+
+            // invalid state error
+            InvalidState => ErrCode::internal("invalid_state", StatusCode::INTERNAL_SERVER_ERROR),
+            // thrown when no primary key has been set
+            MissingPrimaryKey => ErrCode::invalid("missing_primary_key", StatusCode::BAD_REQUEST),
+            // error thrown when trying to set an already existing primary key
+            PrimaryKeyAlreadyPresent => ErrCode::invalid("primary_key_already_present", StatusCode::BAD_REQUEST),
+
+            // invalid document
+            MaxFieldsLimitExceeded => ErrCode::invalid("max_fields_limit_exceeded", StatusCode::BAD_REQUEST),
+            MissingDocumentId => ErrCode::invalid("missing_document_id", StatusCode::BAD_REQUEST),
+
+            // error related to facets
+            Facet => ErrCode::invalid("invalid_facet", StatusCode::BAD_REQUEST),
+            // error related to filters
+            Filter => ErrCode::invalid("invalid_filter", StatusCode::BAD_REQUEST),
+
+            BadParameter => ErrCode::invalid("bad_parameter", StatusCode::BAD_REQUEST),
+            BadRequest => ErrCode::invalid("bad_request", StatusCode::BAD_REQUEST),
+            DocumentNotFound => ErrCode::invalid("document_not_found", StatusCode::NOT_FOUND),
+            Internal => ErrCode::internal("internal", StatusCode::INTERNAL_SERVER_ERROR),
+            InvalidToken => ErrCode::authentication("invalid_token", StatusCode::FORBIDDEN),
+            Maintenance =>  ErrCode::internal("maintenance", StatusCode::SERVICE_UNAVAILABLE),
+            MissingAuthorizationHeader => ErrCode::authentication("missing_authorization_header", StatusCode::UNAUTHORIZED),
+            NotFound => ErrCode::invalid("not_found", StatusCode::NOT_FOUND),
+            PayloadTooLarge => ErrCode::invalid("payload_too_large", StatusCode::PAYLOAD_TOO_LARGE),
+            RetrieveDocument => ErrCode::internal("unretrievable_document", StatusCode::BAD_REQUEST),
+            SearchDocuments => ErrCode::internal("search_error", StatusCode::BAD_REQUEST),
+            UnsupportedMediaType => ErrCode::invalid("unsupported_media_type", StatusCode::UNSUPPORTED_MEDIA_TYPE),
+        }
+    }
+
+    /// return the HTTP status code ascociated with the `Code`
+    fn http(&self) -> StatusCode {
+        self.err_code().status_code
+    }
+
+    /// return error name, used as error code
+    fn name(&self) -> String {
+        self.err_code().error_name.to_string()
+    }
+
+    /// return the error type
+    fn type_(&self) -> String {
+        self.err_code().error_type.to_string()
+    }
+
+    /// return the doc url ascociated with the error
+    fn url(&self) -> String {
+        format!("https://docs.meilisearch.com/errors#{}", self.name())
+    }
+}
+
+/// Internal structure providing a convenient way to create error codes
+struct ErrCode {
+    status_code: StatusCode,
+    error_type: ErrorType,
+    error_name: &'static str,
+}
+
+impl ErrCode {
+    fn authentication(error_name: &'static str, status_code: StatusCode) -> ErrCode {
+        ErrCode {
+            status_code,
+            error_name,
+            error_type: ErrorType::AuthenticationError,
+        }
+    }
+
+    fn internal(error_name: &'static str, status_code: StatusCode) -> ErrCode {
+        ErrCode {
+            status_code,
+            error_name,
+            error_type: ErrorType::InternalError,
+        }
+    }
+
+    fn invalid(error_name: &'static str, status_code: StatusCode) -> ErrCode {
+        ErrCode {
+            status_code,
+            error_name,
+            error_type: ErrorType::InvalidRequestError,
+        }
+    }
+}

meilisearch-http/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "meilisearch-http"
 description = "MeiliSearch HTTP server"
-version = "0.10.0"
+version = "0.13.0"
 license = "MIT"
 authors = [
     "Quentin de Quelen <quentin@dequelen.me>",
@@ -13,41 +13,64 @@ edition = "2018"
 name = "meilisearch"
 path = "src/main.rs"
 
+[features]
+default = ["sentry"]
+
 [dependencies]
-async-std = { version = "1.5.0", features = ["attributes"] }
+actix-cors = "0.2.0"
+actix-http = "1"
+actix-rt = "1"
+actix-service = "1.0.5"
+actix-web = { version = "2.0.0", features = ["rustls"] }
+actix-web-macros = "0.1.0"
+bytes = "0.5.4"
 chrono = { version = "0.4.11", features = ["serde"] }
 crossbeam-channel = "0.4.2"
 env_logger = "0.7.1"
 futures = "0.3.4"
-heed = "0.7.0"
 http = "0.1.19"
 indexmap = { version = "1.3.2", features = ["serde-1"] }
 log = "0.4.8"
 main_error = "0.1.0"
-meilisearch-core = { path = "../meilisearch-core", version = "0.10.0" }
-meilisearch-schema = { path = "../meilisearch-schema", version = "0.10.0" }
-meilisearch-tokenizer = {path = "../meilisearch-tokenizer", version = "0.10.0"}
+meilisearch-core = { path = "../meilisearch-core", version = "0.13.0" }
+meilisearch-error = { path = "../meilisearch-error", version = "0.13.0" }
+meilisearch-schema = { path = "../meilisearch-schema", version = "0.13.0" }
+meilisearch-tokenizer = {path = "../meilisearch-tokenizer", version = "0.13.0"}
 mime = "0.3.16"
-pretty-bytes = "0.2.2"
 rand = "0.7.3"
-rayon = "1.3.0"
+regex = "1.3.6"
+rustls = "0.16.0"
 serde = { version = "1.0.105", features = ["derive"] }
 serde_json = { version = "1.0.50", features = ["preserve_order"] }
 serde_qs = "0.5.2"
 sha2 = "0.8.1"
 siphasher = "0.3.2"
+slice-group-by = "0.2.6"
 structopt = "0.3.12"
-sysinfo = "0.12.0"
-tide = "0.6.0"
+tokio = { version = "0.2.18", features = ["macros"] }
 ureq = { version = "0.12.0", features = ["tls"], default-features = false }
 walkdir = "2.3.1"
 whoami = "0.8.1"
 
+[dependencies.sentry]
+version = "0.18.1"
+default-features = false
+features = [
+    "with_client_implementation",
+    "with_panic",
+    "with_failure",
+    "with_device_info",
+    "with_rust_info",
+    "with_reqwest_transport",
+    "with_rustls",
+    "with_env_logger"
+]
+optional = true
+
 [dev-dependencies]
-http-service = "0.4.0"
-http-service-mock = "0.4.0"
+serde_url_params = "0.2.0"
 tempdir = "0.3.7"
-once_cell = "1.3.1"
+tokio = { version = "0.2.18", features = ["macros", "time"] }
 
 [dev-dependencies.assert-json-diff]
 git = "https://github.com/qdequele/assert-json-diff"

meilisearch-http/public/interface.html

@@ -79,6 +79,7 @@
         box-sizing: border-box;
         padding-left: 10px;
         color: rgba(0,0,0,.9);
+        overflow-wrap: break-word;
       }
     </style>
   </head>
@@ -93,6 +94,17 @@
           <h2 class="subtitle">
             This dashboard will help you check the search results with ease.
           </h2>
+
+          <div class="field">
+            <!-- API Key -->
+            <div class="field">
+              <div class="control">
+                <input id="apiKey" class="input is-small" type="password" placeholder="API key (optional)">
+                <div class="help">At least a private API key is required for the dashboard to access the indexes list.</div>
+              </div>
+            </div>
+          </div>
+
         </div>
       </div>
     </section>
@@ -157,13 +169,33 @@
         return str;
     }
 
-    function httpGet(theUrl) {
+    function httpGet(theUrl, apiKey) {
         var xmlHttp = new XMLHttpRequest();
         xmlHttp.open("GET", theUrl, false); // false for synchronous request
+        if (apiKey) {
+          xmlHttp.setRequestHeader("x-Meili-API-Key", apiKey);
+        }
         xmlHttp.send(null);
         return xmlHttp.responseText;
     }
 
+    function refreshIndexList() {
+        // TODO we must not block here
+        let result = JSON.parse(httpGet(`${baseUrl}/indexes`, localStorage.getItem('apiKey')));
+
+        if (!Array.isArray(result)) { return }
+
+        let select = document.getElementById("index");
+        select.innerHTML = '';
+
+        for (index of result) {
+            const option = document.createElement('option');
+            option.value = index.uid;
+            option.innerHTML = index.name;
+            select.appendChild(option);
+        }
+    }
+
     let lastRequest = undefined;
 
     function triggerSearch() {
@@ -177,6 +209,11 @@
         lastRequest = new XMLHttpRequest();
 
         lastRequest.open("GET", theUrl, true);
+
+        if (localStorage.getItem('apiKey')) {
+          lastRequest.setRequestHeader("x-Meili-API-Key", localStorage.getItem('apiKey'));
+        }
+
         lastRequest.onload = function (e) {
             if (lastRequest.readyState === 4 && lastRequest.status === 200) {
                 let sanitizedResponseText = sanitizeHTMLEntities(lastRequest.responseText);
@@ -249,18 +286,18 @@
         lastRequest.send(null);
     }
 
-    let baseUrl = window.location.origin;
-    // TODO we must not block here
-    let result = JSON.parse(httpGet(`${baseUrl}/indexes`));
-
-    let select = document.getElementById("index");
-    for (index of result) {
-        const option = document.createElement('option');
-        option.value = index.uid;
-        option.innerHTML = index.name;
-        select.appendChild(option);
+    if (!apiKey.value) {
+        apiKey.value = localStorage.getItem('apiKey');
     }
 
+    apiKey.addEventListener('input', function(e) {
+        localStorage.setItem('apiKey', apiKey.value);
+        refreshIndexList();
+    }, false);
+
+    let baseUrl = window.location.origin;
+    refreshIndexList();
+
     search.oninput = triggerSearch;
     select.onchange = triggerSearch;
 

meilisearch-http/src/analytics.rs

@@ -1,20 +1,72 @@
 use std::hash::{Hash, Hasher};
-use std::thread;
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use std::{error, thread};
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
 
 use log::error;
 use serde::Serialize;
 use serde_qs as qs;
 use siphasher::sip::SipHasher;
+use walkdir::WalkDir;
+
+use crate::Data;
+use crate::Opt;
 
 const AMPLITUDE_API_KEY: &str = "f7fba398780e06d8fe6666a9be7e3d47";
 
+#[derive(Debug, Serialize)]
+struct EventProperties {
+    database_size: u64,
+    last_update_timestamp: Option<i64>, //timestamp
+    number_of_documents: Vec<u64>,
+}
+
+impl EventProperties {
+    fn from(data: Data) -> Result<EventProperties, Box<dyn error::Error>> {
+        let mut index_list = Vec::new();
+
+        let reader = data.db.main_read_txn()?;
+
+        for index_uid in data.db.indexes_uids() {
+            if let Some(index) = data.db.open_index(&index_uid) {
+                let number_of_documents = index.main.number_of_documents(&reader)?;
+                index_list.push(number_of_documents);
+            }
+        }
+
+        let database_size = WalkDir::new(&data.db_path)
+            .into_iter()
+            .filter_map(|entry| entry.ok())
+            .filter_map(|entry| entry.metadata().ok())
+            .filter(|metadata| metadata.is_file())
+            .fold(0, |acc, m| acc + m.len());
+
+        let last_update_timestamp = data.db.last_update(&reader)?.map(|u| u.timestamp());
+
+        Ok(EventProperties {
+            database_size,
+            last_update_timestamp,
+            number_of_documents: index_list,
+        })
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct UserProperties<'a> {
+    env: &'a str,
+    start_since_days: u64,
+    user_email: Option<String>,
+    server_provider: Option<String>,
+}
+
 #[derive(Debug, Serialize)]
 struct Event<'a> {
     user_id: &'a str,
     event_type: &'a str,
     device_id: &'a str,
     time: u64,
+    app_version: &'a str,
+    user_properties: UserProperties<'a>,
+    event_properties: Option<EventProperties>,
 }
 
 #[derive(Debug, Serialize)]
@@ -23,7 +75,7 @@ struct AmplitudeRequest<'a> {
     event: &'a str,
 }
 
-pub fn analytics_sender() {
+pub fn analytics_sender(data: Data, opt: Opt) {
     let username = whoami::username();
     let hostname = whoami::hostname();
     let platform = whoami::platform();
@@ -36,6 +88,7 @@ pub fn analytics_sender() {
 
     let uid = format!("{:X}", hash);
     let platform = platform.to_string();
+    let first_start = Instant::now();
 
     loop {
         let n = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
@@ -43,12 +96,27 @@ pub fn analytics_sender() {
         let device_id = &platform;
         let time = n.as_secs();
         let event_type = "runtime_tick";
+        let elapsed_since_start = first_start.elapsed().as_secs() / 86_400; // One day
+        let event_properties = EventProperties::from(data.clone()).ok();
+        let app_version = env!("CARGO_PKG_VERSION").to_string();
+        let app_version = app_version.as_str();
+        let user_email = std::env::var("MEILI_USER_EMAIL").ok();
+        let server_provider = std::env::var("MEILI_SERVER_PROVIDER").ok();
+        let user_properties = UserProperties {
+            env: &opt.env,
+            start_since_days: elapsed_since_start,
+            user_email,
+            server_provider,
+        };
 
         let event = Event {
             user_id,
             event_type,
             device_id,
             time,
+            app_version,
+            user_properties,
+            event_properties
         };
         let event = serde_json::to_string(&event).unwrap();
 
@@ -64,6 +132,6 @@ pub fn analytics_sender() {
             error!("Unsuccessful call to Amplitude: {}", body);
         }
 
-        thread::sleep(Duration::from_secs(86_400)) // one day
+        thread::sleep(Duration::from_secs(3600)) // one hour
     }
 }

meilisearch-http/src/data.rs

@@ -1,20 +1,12 @@
-use std::collections::HashMap;
+use std::error::Error;
 use std::ops::Deref;
 use std::sync::Arc;
 
-use chrono::{DateTime, Utc};
-use heed::types::{SerdeBincode, Str};
-use log::error;
-use meilisearch_core::{Database, Error as MError, MResult, MainT, UpdateT};
+use meilisearch_core::{Database, DatabaseOptions};
 use sha2::Digest;
-use sysinfo::Pid;
 
+use crate::index_update_callback;
 use crate::option::Opt;
-use crate::routes::index::index_update_callback;
-
-const LAST_UPDATE_KEY: &str = "last-update";
-
-type SerdeDatetime = SerdeBincode<DateTime<Utc>>;
 
 #[derive(Clone)]
 pub struct Data {
@@ -34,10 +26,11 @@ pub struct DataInner {
     pub db: Arc<Database>,
     pub db_path: String,
     pub api_keys: ApiKeys,
-    pub server_pid: Pid,
+    pub server_pid: u32,
+    pub http_payload_size_limit: usize,
 }
 
-#[derive(Default, Clone)]
+#[derive(Clone)]
 pub struct ApiKeys {
     pub public: Option<String>,
     pub private: Option<String>,
@@ -61,81 +54,22 @@ impl ApiKeys {
     }
 }
 
-impl DataInner {
-    pub fn is_indexing(&self, reader: &heed::RoTxn<UpdateT>, index: &str) -> MResult<Option<bool>> {
-        match self.db.open_index(&index) {
-            Some(index) => index.current_update_id(&reader).map(|u| Some(u.is_some())),
-            None => Ok(None),
-        }
-    }
-
-    pub fn last_update(&self, reader: &heed::RoTxn<MainT>) -> MResult<Option<DateTime<Utc>>> {
-        match self
-            .db
-            .common_store()
-            .get::<_, Str, SerdeDatetime>(reader, LAST_UPDATE_KEY)?
-        {
-            Some(datetime) => Ok(Some(datetime)),
-            None => Ok(None),
-        }
-    }
-
-    pub fn set_last_update(&self, writer: &mut heed::RwTxn<MainT>) -> MResult<()> {
-        self.db
-            .common_store()
-            .put::<_, Str, SerdeDatetime>(writer, LAST_UPDATE_KEY, &Utc::now())
-            .map_err(Into::into)
-    }
-
-    pub fn compute_stats(&self, writer: &mut heed::RwTxn<MainT>, index_uid: &str) -> MResult<()> {
-        let index = match self.db.open_index(&index_uid) {
-            Some(index) => index,
-            None => {
-                error!("Impossible to retrieve index {}", index_uid);
-                return Ok(());
-            }
-        };
-
-        let schema = match index.main.schema(&writer)? {
-            Some(schema) => schema,
-            None => return Ok(()),
-        };
-
-        let all_documents_fields = index
-            .documents_fields_counts
-            .all_documents_fields_counts(&writer)?;
-
-        // count fields frequencies
-        let mut fields_frequency = HashMap::<_, usize>::new();
-        for result in all_documents_fields {
-            let (_, attr, _) = result?;
-            if let Some(field_id) = schema.indexed_pos_to_field_id(attr) {
-                *fields_frequency.entry(field_id).or_default() += 1;
-            }
-        }
-
-        // convert attributes to their names
-        let frequency: HashMap<_, _> = fields_frequency
-            .into_iter()
-            .filter_map(|(a, c)| schema.name(a).map(|name| (name.to_string(), c)))
-            .collect();
-
-        index
-            .main
-            .put_fields_frequency(writer, &frequency)
-            .map_err(MError::Zlmdb)
-    }
-}
-
 impl Data {
-    pub fn new(opt: Opt) -> Data {
+    pub fn new(opt: Opt) -> Result<Data, Box<dyn Error>> {
         let db_path = opt.db_path.clone();
-        let server_pid = sysinfo::get_current_pid().unwrap();
+        let server_pid = std::process::id();
 
-        let db = Arc::new(Database::open_or_create(opt.db_path).unwrap());
+        let db_opt = DatabaseOptions {
+            main_map_size: opt.main_map_size,
+            update_map_size: opt.update_map_size,
+        };
+
+        let http_payload_size_limit = opt.http_payload_size_limit;
+
+        let db = Arc::new(Database::open_or_create(opt.db_path, db_opt)?);
 
         let mut api_keys = ApiKeys {
-            master: opt.master_key.clone(),
+            master: opt.master_key,
             private: None,
             public: None,
         };
@@ -147,6 +81,7 @@ impl Data {
             db_path,
             api_keys,
             server_pid,
+            http_payload_size_limit,
         };
 
         let data = Data {
@@ -158,6 +93,6 @@ impl Data {
             index_update_callback(&index_uid, &callback_context, status);
         }));
 
-        data
+        Ok(data)
     }
 }

meilisearch-http/src/error.rs

@@ -1,191 +1,267 @@
-use std::fmt::Display;
+use std::error;
+use std::fmt;
 
-use http::status::StatusCode;
-use log::{error, warn};
-use meilisearch_core::{FstError, HeedError};
-use serde::{Deserialize, Serialize};
-use tide::IntoResponse;
-use tide::Response;
+use actix_http::ResponseBuilder;
+use actix_web as aweb;
+use actix_web::error::{JsonPayloadError, QueryPayloadError};
+use actix_web::http::StatusCode;
+use serde_json::json;
 
-use crate::helpers::meilisearch::Error as SearchError;
+use meilisearch_error::{ErrorCode, Code};
 
-pub type SResult<T> = Result<T, ResponseError>;
+#[derive(Debug)]
+pub struct ResponseError {
+    inner: Box<dyn ErrorCode>,
+}
 
-pub enum ResponseError {
-    Internal(String),
-    BadRequest(String),
-    InvalidToken(String),
-    NotFound(String),
-    IndexNotFound(String),
-    DocumentNotFound(String),
-    MissingHeader(String),
-    FilterParsing(String),
+impl error::Error for ResponseError {}
+
+impl ErrorCode for ResponseError {
+    fn error_code(&self) -> Code {
+        self.inner.error_code()
+    }
+}
+
+impl fmt::Display for ResponseError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.inner.fmt(f)
+    }
+}
+
+impl From<Error> for ResponseError {
+    fn from(error: Error) -> ResponseError {
+        ResponseError { inner: Box::new(error) }
+    }
+}
+
+#[derive(Debug)]
+pub enum Error {
     BadParameter(String, String),
-    OpenIndex(String),
+    BadRequest(String),
     CreateIndex(String),
+    DocumentNotFound(String),
+    IndexNotFound(String),
+    Internal(String),
     InvalidIndexUid,
+    InvalidToken(String),
     Maintenance,
+    MissingAuthorizationHeader,
+    NotFound(String),
+    OpenIndex(String),
+    RetrieveDocument(u32, String),
+    SearchDocuments(String),
+    PayloadTooLarge,
+    UnsupportedMediaType,
 }
 
-impl ResponseError {
-    pub fn internal(message: impl Display) -> ResponseError {
-        ResponseError::Internal(message.to_string())
-    }
+impl error::Error for Error {}
 
-    pub fn bad_request(message: impl Display) -> ResponseError {
-        ResponseError::BadRequest(message.to_string())
-    }
-
-    pub fn invalid_token(message: impl Display) -> ResponseError {
-        ResponseError::InvalidToken(message.to_string())
-    }
-
-    pub fn not_found(message: impl Display) -> ResponseError {
-        ResponseError::NotFound(message.to_string())
-    }
-
-    pub fn index_not_found(message: impl Display) -> ResponseError {
-        ResponseError::IndexNotFound(message.to_string())
-    }
-
-    pub fn document_not_found(message: impl Display) -> ResponseError {
-        ResponseError::DocumentNotFound(message.to_string())
-    }
-
-    pub fn missing_header(message: impl Display) -> ResponseError {
-        ResponseError::MissingHeader(message.to_string())
-    }
-
-    pub fn bad_parameter(name: impl Display, message: impl Display) -> ResponseError {
-        ResponseError::BadParameter(name.to_string(), message.to_string())
-    }
-
-    pub fn open_index(message: impl Display) -> ResponseError {
-        ResponseError::OpenIndex(message.to_string())
-    }
-
-    pub fn create_index(message: impl Display) -> ResponseError {
-        ResponseError::CreateIndex(message.to_string())
-    }
-}
-
-impl IntoResponse for ResponseError {
-    fn into_response(self) -> Response {
+impl ErrorCode for Error {
+    fn error_code(&self) -> Code {
+        use Error::*;
         match self {
-            ResponseError::Internal(err) => {
-                error!("internal server error: {}", err);
-                error("Internal server error".to_string(),
-                    StatusCode::INTERNAL_SERVER_ERROR,
-                )
-            }
-            ResponseError::FilterParsing(err) => {
-                warn!("error paring filter: {}", err);
-                error(format!("parsing error: {}", err),
-                StatusCode::BAD_REQUEST)
-            }
-            ResponseError::BadRequest(err) => {
-                warn!("bad request: {}", err);
-                error(err, StatusCode::BAD_REQUEST)
-            }
-            ResponseError::InvalidToken(err) => {
-                error(format!("Invalid API key: {}", err), StatusCode::FORBIDDEN)
-            }
-            ResponseError::NotFound(err) => error(err, StatusCode::NOT_FOUND),
-            ResponseError::IndexNotFound(index) => {
-                error(format!("Index {} not found", index), StatusCode::NOT_FOUND)
-            }
-            ResponseError::DocumentNotFound(id) => error(
-                format!("Document with id {} not found", id),
-                StatusCode::NOT_FOUND,
-            ),
-            ResponseError::MissingHeader(header) => error(
-                format!("Header {} is missing", header),
-                StatusCode::UNAUTHORIZED,
-            ),
-            ResponseError::BadParameter(param, e) => error(
-                format!("Url parameter {} error: {}", param, e),
-                StatusCode::BAD_REQUEST,
-            ),
-            ResponseError::CreateIndex(err) => error(
-                format!("Impossible to create index; {}", err),
-                StatusCode::BAD_REQUEST,
-            ),
-            ResponseError::OpenIndex(err) => error(
-                format!("Impossible to open index; {}", err),
-                StatusCode::BAD_REQUEST,
-            ),
-            ResponseError::InvalidIndexUid => error(
-                "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_).".to_string(),
-                StatusCode::BAD_REQUEST,
-            ),
-            ResponseError::Maintenance => error(
-                String::from("Server is in maintenance, please try again later"),
-                StatusCode::SERVICE_UNAVAILABLE,
-            ),
+            BadParameter(_, _) => Code::BadParameter,
+            BadRequest(_) => Code::BadRequest,
+            CreateIndex(_) => Code::CreateIndex,
+            DocumentNotFound(_) => Code::DocumentNotFound,
+            IndexNotFound(_) => Code::IndexNotFound,
+            Internal(_) => Code::Internal,
+            InvalidIndexUid => Code::InvalidIndexUid,
+            InvalidToken(_) => Code::InvalidToken,
+            Maintenance => Code::Maintenance,
+            MissingAuthorizationHeader => Code::MissingAuthorizationHeader,
+            NotFound(_) => Code::NotFound,
+            OpenIndex(_) => Code::OpenIndex,
+            RetrieveDocument(_, _) => Code::RetrieveDocument,
+            SearchDocuments(_) => Code::SearchDocuments,
+            PayloadTooLarge => Code::PayloadTooLarge,
+            UnsupportedMediaType => Code::UnsupportedMediaType,
         }
     }
 }
 
-#[derive(Serialize, Deserialize)]
-struct ErrorMessage {
-    message: String,
+#[derive(Debug)]
+pub enum FacetCountError {
+    AttributeNotSet(String),
+    SyntaxError(String),
+    UnexpectedToken { found: String, expected: &'static [&'static str] },
+    NoFacetSet,
 }
 
-fn error(message: String, status: StatusCode) -> Response {
-    let message = ErrorMessage { message };
-    tide::Response::new(status.as_u16())
-        .body_json(&message)
-        .unwrap()
+impl error::Error for FacetCountError {}
+
+impl ErrorCode for FacetCountError {
+    fn error_code(&self) -> Code {
+        Code::BadRequest
+    }
 }
 
-impl From<serde_json::Error> for ResponseError {
-    fn from(err: serde_json::Error) -> ResponseError {
-        ResponseError::internal(err)
+impl FacetCountError {
+    pub fn unexpected_token(found: impl ToString, expected: &'static [&'static str]) -> FacetCountError {
+        let found = found.to_string();
+        FacetCountError::UnexpectedToken { expected, found }
+    }
+}
+
+impl From<serde_json::error::Error> for FacetCountError {
+    fn from(other: serde_json::error::Error) -> FacetCountError {
+        FacetCountError::SyntaxError(other.to_string())
+    }
+}
+
+impl fmt::Display for FacetCountError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use FacetCountError::*;
+
+        match self {
+            AttributeNotSet(attr) => write!(f, "attribute {} is not set as facet", attr),
+            SyntaxError(msg) => write!(f, "syntax error: {}", msg),
+            UnexpectedToken { expected, found } => write!(f, "unexpected {} found, expected {:?}", found, expected),
+            NoFacetSet => write!(f, "can't perform facet count, as no facet is set"),
+        }
+    }
+}
+
+impl Error {
+    pub fn internal(err: impl fmt::Display) -> Error {
+        Error::Internal(err.to_string())
+    }
+
+    pub fn bad_request(err: impl fmt::Display) -> Error {
+        Error::BadRequest(err.to_string())
+    }
+
+    pub fn missing_authorization_header() -> Error {
+        Error::MissingAuthorizationHeader
+    }
+
+    pub fn invalid_token(err: impl fmt::Display) -> Error {
+        Error::InvalidToken(err.to_string())
+    }
+
+    pub fn not_found(err: impl fmt::Display) -> Error {
+        Error::NotFound(err.to_string())
+    }
+
+    pub fn index_not_found(err: impl fmt::Display) -> Error {
+        Error::IndexNotFound(err.to_string())
+    }
+
+    pub fn document_not_found(err: impl fmt::Display) -> Error {
+        Error::DocumentNotFound(err.to_string())
+    }
+
+    pub fn bad_parameter(param: impl fmt::Display, err: impl fmt::Display) -> Error {
+        Error::BadParameter(param.to_string(), err.to_string())
+    }
+
+    pub fn open_index(err: impl fmt::Display) -> Error {
+        Error::OpenIndex(err.to_string())
+    }
+
+    pub fn create_index(err: impl fmt::Display) -> Error {
+        Error::CreateIndex(err.to_string())
+    }
+
+    pub fn invalid_index_uid() -> Error {
+        Error::InvalidIndexUid
+    }
+
+    pub fn maintenance() -> Error {
+        Error::Maintenance
+    }
+
+    pub fn retrieve_document(doc_id: u32, err: impl fmt::Display) -> Error {
+        Error::RetrieveDocument(doc_id, err.to_string())
+    }
+
+    pub fn search_documents(err: impl fmt::Display) -> Error {
+        Error::SearchDocuments(err.to_string())
+    }
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::BadParameter(param, err) => write!(f, "Url parameter {} error: {}", param, err),
+            Self::BadRequest(err) => f.write_str(err),
+            Self::CreateIndex(err) => write!(f, "Impossible to create index; {}", err),
+            Self::DocumentNotFound(document_id) => write!(f, "Document with id {} not found", document_id),
+            Self::IndexNotFound(index_uid) => write!(f, "Index {} not found", index_uid),
+            Self::Internal(err) => f.write_str(err),
+            Self::InvalidIndexUid => f.write_str("Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."),
+            Self::InvalidToken(err) => write!(f, "Invalid API key: {}", err),
+            Self::Maintenance => f.write_str("Server is in maintenance, please try again later"),
+            Self::MissingAuthorizationHeader => f.write_str("You must have an authorization token"),
+            Self::NotFound(err) => write!(f, "{} not found", err),
+            Self::OpenIndex(err) => write!(f, "Impossible to open index; {}", err),
+            Self::RetrieveDocument(id, err) => write!(f, "impossible to retrieve the document with id: {}; {}", id, err),
+            Self::SearchDocuments(err) => write!(f, "impossible to search documents; {}", err),
+            Self::PayloadTooLarge => f.write_str("Payload to large"),
+            Self::UnsupportedMediaType => f.write_str("Unsupported media type"),
+        }
+    }
+}
+
+impl aweb::error::ResponseError for ResponseError {
+    fn error_response(&self) -> aweb::HttpResponse {
+        ResponseBuilder::new(self.status_code()).json(json!({
+            "message": self.to_string(),
+            "errorCode": self.error_name(),
+            "errorType": self.error_type(),
+            "errorLink": self.error_url(),
+        }))
+    }
+
+    fn status_code(&self) -> StatusCode {
+        self.http_status()
     }
 }
 
 impl From<meilisearch_core::Error> for ResponseError {
     fn from(err: meilisearch_core::Error) -> ResponseError {
-        ResponseError::internal(err)
+        ResponseError { inner: Box::new(err) }
     }
 }
 
-impl From<HeedError> for ResponseError {
-    fn from(err: HeedError) -> ResponseError {
-        ResponseError::internal(err)
+impl From<meilisearch_schema::Error> for ResponseError {
+    fn from(err: meilisearch_schema::Error) -> ResponseError {
+        ResponseError { inner: Box::new(err) }
     }
 }
 
-impl From<FstError> for ResponseError {
-    fn from(err: FstError) -> ResponseError {
-        ResponseError::internal(err)
+impl From<actix_http::Error> for Error {
+    fn from(err: actix_http::Error) -> Error {
+        Error::Internal(err.to_string())
     }
 }
 
-impl From<SearchError> for ResponseError {
-    fn from(err: SearchError) -> ResponseError {
+impl From<FacetCountError> for ResponseError {
+    fn from(err: FacetCountError) -> ResponseError {
+        ResponseError { inner: Box::new(err) }
+    }
+}
+
+impl From<JsonPayloadError> for Error {
+    fn from(err: JsonPayloadError) -> Error {
         match err {
-            SearchError::FilterParsing(s) => ResponseError::FilterParsing(s),
-            _ => ResponseError::internal(err),
+            JsonPayloadError::Deserialize(err) => Error::BadRequest(format!("Invalid JSON: {}", err)),
+            JsonPayloadError::Overflow => Error::PayloadTooLarge,
+            JsonPayloadError::ContentType => Error::UnsupportedMediaType,
+            JsonPayloadError::Payload(err) => Error::BadRequest(format!("Problem while decoding the request: {}", err)),
         }
     }
 }
 
-impl From<meilisearch_core::settings::RankingRuleConversionError> for ResponseError {
-    fn from(err: meilisearch_core::settings::RankingRuleConversionError) -> ResponseError {
-        ResponseError::internal(err)
-    }
-}
-
-pub trait IntoInternalError<T> {
-    fn into_internal_error(self) -> SResult<T>;
-}
-
-impl<T> IntoInternalError<T> for Option<T> {
-    fn into_internal_error(self) -> SResult<T> {
-        match self {
-            Some(value) => Ok(value),
-            None => Err(ResponseError::internal("Heed cannot find requested value")),
+impl From<QueryPayloadError> for Error {
+    fn from(err: QueryPayloadError) -> Error {
+        match err {
+            QueryPayloadError::Deserialize(err) => Error::BadRequest(format!("Invalid query parameters: {}", err)),
         }
     }
 }
+
+pub fn payload_error_handler<E: Into<Error>>(err: E) -> ResponseError {
+    let error: Error = err.into();
+    error.into()
+}

meilisearch-http/src/helpers/authentication.rs

@@ -0,0 +1,103 @@
+use std::cell::RefCell;
+use std::pin::Pin;
+use std::rc::Rc;
+use std::task::{Context, Poll};
+
+use actix_service::{Service, Transform};
+use actix_web::{dev::ServiceRequest, dev::ServiceResponse};
+use futures::future::{err, ok, Future, Ready};
+
+use crate::error::{Error, ResponseError};
+use crate::Data;
+
+#[derive(Clone)]
+pub enum Authentication {
+    Public,
+    Private,
+    Admin,
+}
+
+impl<S: 'static, B> Transform<S> for Authentication
+where
+    S: Service<Request = ServiceRequest, Response = ServiceResponse<B>, Error = actix_web::Error>,
+    S::Future: 'static,
+    B: 'static,
+{
+    type Request = ServiceRequest;
+    type Response = ServiceResponse<B>;
+    type Error = actix_web::Error;
+    type InitError = ();
+    type Transform = LoggingMiddleware<S>;
+    type Future = Ready<Result<Self::Transform, Self::InitError>>;
+
+    fn new_transform(&self, service: S) -> Self::Future {
+        ok(LoggingMiddleware {
+            acl: self.clone(),
+            service: Rc::new(RefCell::new(service)),
+        })
+    }
+}
+
+pub struct LoggingMiddleware<S> {
+    acl: Authentication,
+    service: Rc<RefCell<S>>,
+}
+
+#[allow(clippy::type_complexity)]
+impl<S, B> Service for LoggingMiddleware<S>
+where
+    S: Service<Request = ServiceRequest, Response = ServiceResponse<B>, Error = actix_web::Error> + 'static,
+    S::Future: 'static,
+    B: 'static,
+{
+    type Request = ServiceRequest;
+    type Response = ServiceResponse<B>;
+    type Error = actix_web::Error;
+    type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>>>>;
+
+    fn poll_ready(&mut self, cx: &mut Context) -> Poll<Result<(), Self::Error>> {
+        self.service.poll_ready(cx)
+    }
+
+    fn call(&mut self, req: ServiceRequest) -> Self::Future {
+        let mut svc = self.service.clone();
+        // This unwrap is left because this error should never appear. If that's the case, then
+        // it means that actix-web has an issue or someone changes the type `Data`.
+        let data = req.app_data::<Data>().unwrap();
+
+        if data.api_keys.master.is_none() {
+            return Box::pin(svc.call(req));
+        }
+
+        let auth_header = match req.headers().get("X-Meili-API-Key") {
+            Some(auth) => match auth.to_str() {
+                Ok(auth) => auth,
+                Err(_) => return Box::pin(err(ResponseError::from(Error::MissingAuthorizationHeader).into())),
+            },
+            None => {
+                return Box::pin(err(ResponseError::from(Error::MissingAuthorizationHeader).into()));
+            }
+        };
+
+        let authenticated = match self.acl {
+            Authentication::Admin => data.api_keys.master.as_deref() == Some(auth_header),
+            Authentication::Private => {
+                data.api_keys.master.as_deref() == Some(auth_header)
+                    || data.api_keys.private.as_deref() == Some(auth_header)
+            }
+            Authentication::Public => {
+                data.api_keys.master.as_deref() == Some(auth_header)
+                    || data.api_keys.private.as_deref() == Some(auth_header)
+                    || data.api_keys.public.as_deref() == Some(auth_header)
+            }
+        };
+
+        if authenticated {
+            Box::pin(svc.call(req))
+        } else {
+            Box::pin(err(
+                ResponseError::from(Error::InvalidToken(auth_header.to_string())).into()
+            ))
+        }
+    }
+}

meilisearch-http/src/helpers/meilisearch.rs

@@ -1,98 +1,30 @@
 use std::cmp::Ordering;
 use std::collections::{HashMap, HashSet};
-use std::convert::From;
-use std::error;
-use std::fmt;
 use std::hash::{Hash, Hasher};
-use std::time::{Duration, Instant};
+use std::time::Instant;
 
 use indexmap::IndexMap;
 use log::error;
-use meilisearch_core::Filter;
+use meilisearch_core::{Filter, MainReader};
+use meilisearch_core::facets::FacetFilter;
 use meilisearch_core::criterion::*;
 use meilisearch_core::settings::RankingRule;
-use meilisearch_core::{Highlight, Index, MainT, RankedMap};
+use meilisearch_core::{Highlight, Index, RankedMap};
 use meilisearch_schema::{FieldId, Schema};
 use meilisearch_tokenizer::is_cjk;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use siphasher::sip::SipHasher;
+use slice_group_by::GroupBy;
 
-#[derive(Debug)]
-pub enum Error {
-    SearchDocuments(String),
-    RetrieveDocument(u64, String),
-    DocumentNotFound(u64),
-    CropFieldWrongType(String),
-    FilterParsing(String),
-    AttributeNotFoundOnDocument(String),
-    AttributeNotFoundOnSchema(String),
-    MissingFilterValue,
-    UnknownFilteredAttribute,
-    Internal(String),
-}
-
-impl error::Error for Error {}
-
-impl fmt::Display for Error {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        use Error::*;
-
-        match self {
-            SearchDocuments(err) => write!(f, "impossible to search documents; {}", err),
-            RetrieveDocument(id, err) => write!(
-                f,
-                "impossible to retrieve the document with id: {}; {}",
-                id, err
-            ),
-            DocumentNotFound(id) => write!(f, "document {} not found", id),
-            CropFieldWrongType(field) => {
-                write!(f, "the field {} cannot be cropped it's not a string", field)
-            }
-            AttributeNotFoundOnDocument(field) => {
-                write!(f, "field {} is not found on document", field)
-            }
-            AttributeNotFoundOnSchema(field) => write!(f, "field {} is not found on schema", field),
-            MissingFilterValue => f.write_str("a filter doesn't have a value to compare it with"),
-            UnknownFilteredAttribute => {
-                f.write_str("a filter is specifying an unknown schema attribute")
-            }
-            Internal(err) => write!(f, "internal error; {}", err),
-            FilterParsing(err) => write!(f, "filter parsing error: {}", err),
-        }
-    }
-}
-
-impl From<meilisearch_core::Error> for Error {
-    fn from(error: meilisearch_core::Error) -> Self {
-        use meilisearch_core::pest_error::LineColLocation::*;
-        match error {
-            meilisearch_core::Error::FilterParseError(e) => {
-                let (line, column) = match e.line_col {
-                    Span((line, _), (column, _)) => (line, column),
-                    Pos((line, column)) => (line, column),
-                };
-                let message = format!("parsing error on line {} at column {}: {}", line, column, e.variant.message());
-
-                Error::FilterParsing(message) 
-            },
-            _ => Error::Internal(error.to_string()),
-        }
-    }
-}
-
-impl From<heed::Error> for Error {
-    fn from(error: heed::Error) -> Self {
-        Error::Internal(error.to_string())
-    }
-}
+use crate::error::{Error, ResponseError};
 
 pub trait IndexSearchExt {
-    fn new_search(&self, query: String) -> SearchBuilder;
+    fn new_search(&self, query: Option<String>) -> SearchBuilder;
 }
 
 impl IndexSearchExt for Index {
-    fn new_search(&self, query: String) -> SearchBuilder {
+    fn new_search(&self, query: Option<String>) -> SearchBuilder {
         SearchBuilder {
             index: self,
             query,
@@ -102,23 +34,25 @@ impl IndexSearchExt for Index {
             attributes_to_retrieve: None,
             attributes_to_highlight: None,
             filters: None,
-            timeout: Duration::from_millis(30),
             matches: false,
+            facet_filters: None,
+            facets: None,
         }
     }
 }
 
 pub struct SearchBuilder<'a> {
     index: &'a Index,
-    query: String,
+    query: Option<String>,
     offset: usize,
     limit: usize,
     attributes_to_crop: Option<HashMap<String, usize>>,
     attributes_to_retrieve: Option<HashSet<String>>,
     attributes_to_highlight: Option<HashSet<String>>,
     filters: Option<String>,
-    timeout: Duration,
     matches: bool,
+    facet_filters: Option<FacetFilter>,
+    facets: Option<Vec<(FieldId, String)>>
 }
 
 impl<'a> SearchBuilder<'a> {
@@ -153,13 +87,13 @@ impl<'a> SearchBuilder<'a> {
         self
     }
 
-    pub fn filters(&mut self, value: String) -> &SearchBuilder {
-        self.filters = Some(value);
+    pub fn add_facet_filters(&mut self, filters: FacetFilter) -> &SearchBuilder {
+        self.facet_filters = Some(filters);
         self
     }
 
-    pub fn timeout(&mut self, value: Duration) -> &SearchBuilder {
-        self.timeout = value;
+    pub fn filters(&mut self, value: String) -> &SearchBuilder {
+        self.filters = Some(value);
         self
     }
 
@@ -168,17 +102,19 @@ impl<'a> SearchBuilder<'a> {
         self
     }
 
-    pub fn search(&self, reader: &heed::RoTxn<MainT>) -> Result<SearchResult, Error> {
-        let schema = self.index.main.schema(reader);
-        let schema = schema.map_err(|e| Error::Internal(e.to_string()))?;
-        let schema = match schema {
-            Some(schema) => schema,
-            None => return Err(Error::Internal(String::from("missing schema"))),
-        };
+    pub fn add_facets(&mut self, facets: Vec<(FieldId, String)>) -> &SearchBuilder {
+        self.facets = Some(facets);
+        self
+    }
 
-        let ranked_map = self.index.main.ranked_map(reader);
-        let ranked_map = ranked_map.map_err(|e| Error::Internal(e.to_string()))?;
-        let ranked_map = ranked_map.unwrap_or_default();
+    pub fn search(self, reader: &MainReader) -> Result<SearchResult, ResponseError> {
+        let schema = self
+            .index
+            .main
+            .schema(reader)?
+            .ok_or(Error::internal("missing schema"))?;
+
+        let ranked_map = self.index.main.ranked_map(reader)?.unwrap_or_default();
 
         // Change criteria
         let mut query_builder = match self.get_criteria(reader, &ranked_map, &schema)? {
@@ -188,8 +124,8 @@ impl<'a> SearchBuilder<'a> {
 
         if let Some(filter_expression) = &self.filters {
             let filter = Filter::parse(filter_expression, &schema)?;
+            let index = &self.index;
             query_builder.with_filter(move |id| {
-                let index = &self.index;
                 let reader = &reader;
                 let filter = &filter;
                 match filter.test(reader, index, id) {
@@ -202,27 +138,26 @@ impl<'a> SearchBuilder<'a> {
             });
         }
 
-        query_builder.with_fetch_timeout(self.timeout);
-
         if let Some(field) = self.index.main.distinct_attribute(reader)? {
-            if let Some(field_id) = schema.id(&field) {
-                query_builder.with_distinct(1, move |id| {
-                    match self.index.document_attribute_bytes(reader, id, field_id) {
-                        Ok(Some(bytes)) => {
-                            let mut s = SipHasher::new();
-                            bytes.hash(&mut s);
-                            Some(s.finish())
-                        }
-                        _ => None,
+            let index = &self.index;
+            query_builder.with_distinct(1, move |id| {
+                match index.document_attribute_bytes(reader, id, field) {
+                    Ok(Some(bytes)) => {
+                        let mut s = SipHasher::new();
+                        bytes.hash(&mut s);
+                        Some(s.finish())
                     }
-                });
-            }
+                    _ => None,
+                }
+            });
         }
 
+        query_builder.set_facet_filter(self.facet_filters);
+        query_builder.set_facets(self.facets);
+
         let start = Instant::now();
-        let result =
-            query_builder.query(reader, &self.query, self.offset..(self.offset + self.limit));
-        let (docs, nb_hits) = result.map_err(|e| Error::SearchDocuments(e.to_string()))?;
+        let result = query_builder.query(reader, self.query.as_deref(), self.offset..(self.offset + self.limit));
+        let search_result = result.map_err(Error::search_documents)?;
         let time_ms = start.elapsed().as_millis() as usize;
 
         let mut all_attributes: HashSet<&str> = HashSet::new();
@@ -253,12 +188,14 @@ impl<'a> SearchBuilder<'a> {
         }
 
         let mut hits = Vec::with_capacity(self.limit);
-        for doc in docs {
+        for doc in search_result.documents {
             let mut document: IndexMap<String, Value> = self
                 .index
                 .document(reader, Some(&all_attributes), doc.id)
-                .map_err(|e| Error::RetrieveDocument(doc.id.0, e.to_string()))?
-                .ok_or(Error::DocumentNotFound(doc.id.0))?;
+                .map_err(|e| Error::retrieve_document(doc.id.0, e))?
+                .ok_or(Error::internal(
+                    "Impossible to retrieve the document; Corrupted data",
+                ))?;
 
             let mut formatted = document.iter()
                 .filter(|(key, _)| all_formatted.contains(key.as_str()))
@@ -275,7 +212,7 @@ impl<'a> SearchBuilder<'a> {
             // Transform to readable matches
             if let Some(attributes_to_highlight) = &self.attributes_to_highlight {
                 let matches = calculate_matches(
-                    matches.clone(),
+                    &matches,
                     self.attributes_to_highlight.clone(),
                     &schema,
                 );
@@ -283,7 +220,7 @@ impl<'a> SearchBuilder<'a> {
             }
 
             let matches_info = if self.matches {
-                Some(calculate_matches(matches, self.attributes_to_retrieve.clone(), &schema))
+                Some(calculate_matches(&matches, self.attributes_to_retrieve.clone(), &schema))
             } else {
                 None
             };
@@ -305,10 +242,12 @@ impl<'a> SearchBuilder<'a> {
             hits,
             offset: self.offset,
             limit: self.limit,
-            nb_hits,
-            exhaustive_nb_hits: false,
+            nb_hits: search_result.nb_hits,
+            exhaustive_nb_hits: search_result.exhaustive_nb_hit,
             processing_time_ms: time_ms,
-            query: self.query.to_string(),
+            query: self.query.unwrap_or_default(),
+            facets_distribution: search_result.facets,
+            exhaustive_facets_count: search_result.exhaustive_facets_count,
         };
 
         Ok(results)
@@ -316,10 +255,10 @@ impl<'a> SearchBuilder<'a> {
 
     pub fn get_criteria(
         &self,
-        reader: &heed::RoTxn<MainT>,
+        reader: &MainReader,
         ranked_map: &'a RankedMap,
         schema: &Schema,
-    ) -> Result<Option<Criteria<'a>>, Error> {
+    ) -> Result<Option<Criteria<'a>>, ResponseError> {
         let ranking_rules = self.index.main.ranking_rules(reader)?;
 
         if let Some(ranking_rules) = ranking_rules {
@@ -393,6 +332,10 @@ pub struct SearchResult {
     pub exhaustive_nb_hits: bool,
     pub processing_time_ms: usize,
     pub query: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub facets_distribution: Option<HashMap<String, HashMap<String, usize>>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub exhaustive_facets_count: Option<bool>,
 }
 
 /// returns the start index and the length on the crop.
@@ -411,10 +354,14 @@ fn aligned_crop(text: &str, match_index: usize, context: usize) -> (usize, usize
         return (match_index, 1 + text.chars().skip(match_index).take_while(is_word_component).count());
     }
     let start = match match_index.saturating_sub(context) {
-        n if n == 0 => n,
-        n => word_end_index(n)
+        0 => 0,
+        n => {
+            let word_end_index = word_end_index(n);
+            // skip whitespaces if any
+            word_end_index + text.chars().skip(word_end_index).take_while(char::is_ascii_whitespace).count()
+        }
     };
-    let end = word_end_index(start + 2 * context);
+    let end = word_end_index(match_index + context);
 
     (start, end - start)
 }
@@ -429,15 +376,21 @@ fn crop_text(
     let char_index = matches.peek().map(|m| m.char_index as usize).unwrap_or(0);
     let (start, count) = aligned_crop(text, char_index, context);
 
-    //TODO do something about the double allocation
-    let text = text.chars().skip(start).take(count).collect::<String>().trim().to_string();
+    // TODO do something about double allocation
+    let text = text
+        .chars()
+        .skip(start)
+        .take(count)
+        .collect::<String>()
+        .trim()
+        .to_string();
 
     // update matches index to match the new cropped text
     let matches = matches
-        .take_while(|m| (m.char_index as usize) + (m.char_length as usize) <= start + (context * 2))
-        .map(|match_| Highlight {
-            char_index: match_.char_index - start as u16,
-            ..match_
+        .take_while(|m| (m.char_index as usize) + (m.char_length as usize) <= start + count)
+        .map(|m| Highlight {
+            char_index: m.char_index - start as u16,
+            ..m
         })
         .collect();
 
@@ -476,14 +429,14 @@ fn crop_document(
 }
 
 fn calculate_matches(
-    matches: Vec<Highlight>,
+    matches: &[Highlight],
     attributes_to_retrieve: Option<HashSet<String>>,
     schema: &Schema,
 ) -> MatchesInfos {
     let mut matches_result: HashMap<String, Vec<MatchPosition>> = HashMap::new();
     for m in matches.iter() {
         if let Some(attribute) = schema.name(FieldId::new(m.attribute)) {
-            if let Some(attributes_to_retrieve) = attributes_to_retrieve.clone() {
+            if let Some(ref attributes_to_retrieve) = attributes_to_retrieve {
                 if !attributes_to_retrieve.contains(attribute) {
                     continue;
                 }
@@ -526,19 +479,23 @@ fn calculate_highlights(
                 let value: Vec<_> = value.chars().collect();
                 let mut highlighted_value = String::new();
                 let mut index = 0;
-                for m in matches {
-                    if m.start >= index {
-                        let before = value.get(index..m.start);
-                        let highlighted = value.get(m.start..(m.start + m.length));
-                        if let (Some(before), Some(highlighted)) = (before, highlighted) {
-                            highlighted_value.extend(before);
-                            highlighted_value.push_str("<em>");
-                            highlighted_value.extend(highlighted);
-                            highlighted_value.push_str("</em>");
-                            index = m.start + m.length;
-                        } else {
-                            error!("value: {:?}; index: {:?}, match: {:?}", value, index, m);
-                        }
+
+                let longest_matches = matches
+                    .linear_group_by_key(|m| m.start)
+                    .map(|group| group.last().unwrap())
+                    .filter(move |m| m.start >= index);
+
+                for m in longest_matches {
+                    let before = value.get(index..m.start);
+                    let highlighted = value.get(m.start..(m.start + m.length));
+                    if let (Some(before), Some(highlighted)) = (before, highlighted) {
+                        highlighted_value.extend(before);
+                        highlighted_value.push_str("<em>");
+                        highlighted_value.extend(highlighted);
+                        highlighted_value.push_str("</em>");
+                        index = m.start + m.length;
+                    } else {
+                        error!("value: {:?}; index: {:?}, match: {:?}", value, index, m);
                     }
                 }
                 highlighted_value.extend(value[index..].iter());
@@ -546,7 +503,6 @@ fn calculate_highlights(
             };
         }
     }
-
     highlight_result
 }
 
@@ -578,13 +534,41 @@ mod tests {
         // mixed charset
         let (start, length) = aligned_crop(&text, 5, 3);
         let cropped =  text.chars().skip(start).take(length).collect::<String>().trim().to_string();
-        assert_eq!("isのス", cropped);
+        assert_eq!("isの", cropped);
 
         // split regular word / CJK word, no space
         let (start, length) = aligned_crop(&text, 7, 1);
         let cropped =  text.chars().skip(start).take(length).collect::<String>().trim().to_string();
-        assert_eq!("のス", cropped);
+        assert_eq!("の", cropped);
+    }
 
+    #[test]
+    fn calculate_matches() {
+        let mut matches = Vec::new();
+        matches.push(Highlight { attribute: 0, char_index: 0, char_length: 3});
+        matches.push(Highlight { attribute: 0, char_index: 0, char_length: 2});
+
+        let mut attributes_to_retrieve: HashSet<String> = HashSet::new();
+        attributes_to_retrieve.insert("title".to_string());
+
+        let schema = Schema::with_primary_key("title");
+
+        let matches_result = super::calculate_matches(&matches, Some(attributes_to_retrieve), &schema);
+
+        let mut matches_result_expected: HashMap<String, Vec<MatchPosition>> = HashMap::new();
+
+        let mut positions = Vec::new();
+        positions.push(MatchPosition {
+            start: 0,
+            length: 2,
+        });
+        positions.push(MatchPosition {
+            start: 0,
+            length: 3,
+        });
+        matches_result_expected.insert("title".to_string(), positions);
+
+        assert_eq!(matches_result, matches_result_expected);
     }
 
     #[test]
@@ -625,4 +609,38 @@ mod tests {
 
         assert_eq!(result, result_expected);
     }
+
+    #[test]
+    fn highlight_longest_match() {
+        let data = r#"{
+            "title": "Ice"
+        }"#;
+
+        let document: IndexMap<String, Value> = serde_json::from_str(data).unwrap();
+        let mut attributes_to_highlight = HashSet::new();
+        attributes_to_highlight.insert("title".to_string());
+
+        let mut matches = HashMap::new();
+
+        let mut m = Vec::new();
+        m.push(MatchPosition {
+            start: 0,
+            length: 2,
+        });
+        m.push(MatchPosition {
+            start: 0,
+            length: 3,
+        });
+        matches.insert("title".to_string(), m);
+
+        let result = super::calculate_highlights(&document, &matches, &attributes_to_highlight);
+
+        let mut result_expected = IndexMap::new();
+        result_expected.insert(
+            "title".to_string(),
+            Value::String("<em>Ice</em>".to_string()),
+        );
+
+        assert_eq!(result, result_expected);
+    }
 }

meilisearch-http/src/helpers/mod.rs

@@ -1,2 +1,6 @@
+pub mod authentication;
 pub mod meilisearch;
-pub mod tide;
+pub mod normalize_path;
+
+pub use authentication::Authentication;
+pub use normalize_path::NormalizePath;

meilisearch-http/src/helpers/normalize_path.rs

@@ -0,0 +1,86 @@
+/// From https://docs.rs/actix-web/3.0.0-alpha.2/src/actix_web/middleware/normalize.rs.html#34
+use actix_http::Error;
+use actix_service::{Service, Transform};
+use actix_web::{
+    dev::ServiceRequest,
+    dev::ServiceResponse,
+    http::uri::{PathAndQuery, Uri},
+};
+use futures::future::{ok, Ready};
+use regex::Regex;
+use std::task::{Context, Poll};
+pub struct NormalizePath;
+
+impl<S, B> Transform<S> for NormalizePath
+where
+    S: Service<Request = ServiceRequest, Response = ServiceResponse<B>, Error = Error>,
+    S::Future: 'static,
+{
+    type Request = ServiceRequest;
+    type Response = ServiceResponse<B>;
+    type Error = Error;
+    type InitError = ();
+    type Transform = NormalizePathNormalization<S>;
+    type Future = Ready<Result<Self::Transform, Self::InitError>>;
+
+    fn new_transform(&self, service: S) -> Self::Future {
+        ok(NormalizePathNormalization {
+            service,
+            merge_slash: Regex::new("//+").unwrap(),
+        })
+    }
+}
+
+pub struct NormalizePathNormalization<S> {
+    service: S,
+    merge_slash: Regex,
+}
+
+impl<S, B> Service for NormalizePathNormalization<S>
+where
+    S: Service<Request = ServiceRequest, Response = ServiceResponse<B>, Error = Error>,
+    S::Future: 'static,
+{
+    type Request = ServiceRequest;
+    type Response = ServiceResponse<B>;
+    type Error = Error;
+    type Future = S::Future;
+
+    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        self.service.poll_ready(cx)
+    }
+
+    fn call(&mut self, mut req: ServiceRequest) -> Self::Future {
+        let head = req.head_mut();
+
+        // always add trailing slash, might be an extra one
+        let path = head.uri.path().to_string() + "/";
+
+        if self.merge_slash.find(&path).is_some() {
+            // normalize multiple /'s to one /
+            let path = self.merge_slash.replace_all(&path, "/");
+
+            let path = if path.len() > 1 {
+                path.trim_end_matches('/')
+            } else {
+                &path
+            };
+
+            let mut parts = head.uri.clone().into_parts();
+            let pq = parts.path_and_query.as_ref().unwrap();
+
+            let path = if let Some(q) = pq.query() {
+                bytes::Bytes::from(format!("{}?{}", path, q))
+            } else {
+                bytes::Bytes::copy_from_slice(path.as_bytes())
+            };
+            parts.path_and_query = Some(PathAndQuery::from_maybe_shared(path).unwrap());
+
+            let uri = Uri::from_parts(parts).unwrap();
+            req.match_info_mut().get_mut().update(&uri);
+            req.head_mut().uri = uri;
+        }
+
+        self.service.call(req)
+    }
+}

meilisearch-http/src/helpers/tide.rs

@@ -1,83 +0,0 @@
-use crate::error::{ResponseError, SResult};
-use crate::Data;
-use meilisearch_core::Index;
-use tide::Request;
-
-pub enum ACL {
-    Admin,
-    Private,
-    Public,
-}
-
-pub trait RequestExt {
-    fn is_allowed(&self, acl: ACL) -> SResult<()>;
-    fn url_param(&self, name: &str) -> SResult<String>;
-    fn index(&self) -> SResult<Index>;
-    fn document_id(&self) -> SResult<String>;
-}
-
-impl RequestExt for Request<Data> {
-    fn is_allowed(&self, acl: ACL) -> SResult<()> {
-        let user_api_key = self.header("X-Meili-API-Key");
-
-        if self.state().api_keys.master.is_none() {
-            return Ok(())
-        }
-
-        match acl {
-            ACL::Admin => {
-                if user_api_key == self.state().api_keys.master.as_deref() {
-                    return Ok(());
-                }
-            }
-            ACL::Private => {
-                if user_api_key == self.state().api_keys.master.as_deref() {
-                    return Ok(());
-                }
-                if user_api_key == self.state().api_keys.private.as_deref() {
-                    return Ok(());
-                }
-            }
-            ACL::Public => {
-                if user_api_key == self.state().api_keys.master.as_deref() {
-                    return Ok(());
-                }
-                if user_api_key == self.state().api_keys.private.as_deref() {
-                    return Ok(());
-                }
-                if user_api_key == self.state().api_keys.public.as_deref() {
-                    return Ok(());
-                }
-            }
-        }
-
-        Err(ResponseError::InvalidToken(
-            user_api_key.unwrap_or("Need a token").to_owned(),
-        ))
-    }
-
-    fn url_param(&self, name: &str) -> SResult<String> {
-        let param = self
-            .param::<String>(name)
-            .map_err(|e| ResponseError::bad_parameter(name, e))?;
-        Ok(param)
-    }
-
-    fn index(&self) -> SResult<Index> {
-        let index_uid = self.url_param("index")?;
-        let index = self
-            .state()
-            .db
-            .open_index(&index_uid)
-            .ok_or(ResponseError::index_not_found(index_uid))?;
-        Ok(index)
-    }
-
-    fn document_id(&self) -> SResult<String> {
-        let name = self
-            .param::<String>("document_id")
-            .map_err(|_| ResponseError::bad_parameter("documentId", "primaryKey"))?;
-
-        Ok(name)
-    }
-}

meilisearch-http/src/lib.rs

@@ -6,5 +6,81 @@ pub mod helpers;
 pub mod models;
 pub mod option;
 pub mod routes;
+pub mod analytics;
 
+use actix_http::Error;
+use actix_service::ServiceFactory;
+use actix_web::{dev, web, App};
+use chrono::Utc;
+use log::error;
+
+use meilisearch_core::ProcessedUpdateResult;
+
+pub use option::Opt;
 pub use self::data::Data;
+use self::error::{payload_error_handler, ResponseError};
+
+pub fn create_app(
+    data: &Data,
+) -> App<
+    impl ServiceFactory<
+        Config = (),
+        Request = dev::ServiceRequest,
+        Response = dev::ServiceResponse<actix_http::body::Body>,
+        Error = Error,
+        InitError = (),
+    >,
+    actix_http::body::Body,
+> {
+    App::new()
+        .app_data(web::Data::new(data.clone()))
+        .app_data(
+            web::JsonConfig::default()
+                .limit(data.http_payload_size_limit)
+                .content_type(|_mime| true) // Accept all mime types
+                .error_handler(|err, _req| payload_error_handler(err).into()),
+        )
+        .app_data(
+            web::QueryConfig::default()
+            .error_handler(|err, _req| payload_error_handler(err).into())
+        )
+        .service(routes::load_html)
+        .service(routes::load_css)
+        .configure(routes::document::services)
+        .configure(routes::index::services)
+        .configure(routes::search::services)
+        .configure(routes::setting::services)
+        .configure(routes::stop_words::services)
+        .configure(routes::synonym::services)
+        .configure(routes::health::services)
+        .configure(routes::stats::services)
+        .configure(routes::key::services)
+}
+
+pub fn index_update_callback(index_uid: &str, data: &Data, status: ProcessedUpdateResult) {
+    if status.error.is_some() {
+        return;
+    }
+
+    if let Some(index) = data.db.open_index(&index_uid) {
+        let db = &data.db;
+        let res = db.main_write::<_, _, ResponseError>(|mut writer| {
+            if let Err(e) = data.db.compute_stats(&mut writer, &index_uid) {
+                error!("Impossible to compute stats; {}", e)
+            }
+
+            if let Err(e) = data.db.set_last_update(&mut writer, &Utc::now()) {
+                error!("Impossible to update last_update; {}", e)
+            }
+
+            if let Err(e) = index.main.put_updated_at(&mut writer) {
+                error!("Impossible to update updated_at; {}", e)
+            }
+            Ok(())
+        });
+        match res {
+            Ok(_) => (),
+            Err(e) => error!("{}", e),
+        }
+    }
+}

meilisearch-http/src/main.rs

@@ -1,16 +1,11 @@
 use std::{env, thread};
 
-use async_std::task;
-use log::info;
+use actix_cors::Cors;
+use actix_web::{middleware, HttpServer};
 use main_error::MainError;
+use meilisearch_http::helpers::NormalizePath;
+use meilisearch_http::{create_app, index_update_callback, Data, Opt};
 use structopt::StructOpt;
-use tide::middleware::{Cors, RequestLogger, Origin};
-use http::header::HeaderValue;
-
-use meilisearch_http::data::Data;
-use meilisearch_http::option::Opt;
-use meilisearch_http::routes;
-use meilisearch_http::routes::index::index_update_callback;
 
 mod analytics;
 
@@ -18,9 +13,23 @@ mod analytics;
 #[global_allocator]
 static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
 
-pub fn main() -> Result<(), MainError> {
+#[actix_rt::main]
+async fn main() -> Result<(), MainError> {
     let opt = Opt::from_args();
 
+    #[cfg(all(not(debug_assertions), feature = "sentry"))]
+    let _sentry = sentry::init((
+        if !opt.no_sentry {
+            Some(opt.sentry_dsn.clone())
+        } else {
+            None
+        },
+        sentry::ClientOptions {
+            release: sentry::release_name!(),
+            ..Default::default()
+        },
+    ));
+
     match opt.env.as_ref() {
         "production" => {
             if opt.master_key.is_none() {
@@ -29,7 +38,12 @@ pub fn main() -> Result<(), MainError> {
                         .into(),
                 );
             }
-            env_logger::init();
+
+            #[cfg(all(not(debug_assertions), feature = "sentry"))]
+            if !opt.no_sentry && _sentry.is_enabled() {
+                sentry::integrations::panic::register_panic_handler(); // TODO: This shouldn't be needed when upgrading to sentry 0.19.0. These integrations are turned on by default when using `sentry::init`.
+                sentry::integrations::env_logger::init(None, Default::default());
+            }
         }
         "development" => {
             env_logger::from_env(env_logger::Env::default().default_filter_or("info")).init();
@@ -37,11 +51,13 @@ pub fn main() -> Result<(), MainError> {
         _ => unreachable!(),
     }
 
-    if !opt.no_analytics {
-        thread::spawn(analytics::analytics_sender);
-    }
+    let data = Data::new(opt.clone())?;
 
-    let data = Data::new(opt.clone());
+    if !opt.no_analytics {
+        let analytics_data = data.clone();
+        let analytics_opt = opt.clone();
+        thread::spawn(move || analytics::analytics_sender(analytics_data, analytics_opt));
+    }
 
     let data_cloned = data.clone();
     data.db.set_update_callback(Box::new(move |name, status| {
@@ -50,17 +66,29 @@ pub fn main() -> Result<(), MainError> {
 
     print_launch_resume(&opt, &data);
 
-    let mut app = tide::with_state(data);
+    let http_server = HttpServer::new(move || {
+        create_app(&data)
+            .wrap(
+                Cors::new()
+                    .send_wildcard()
+                    .allowed_headers(vec!["content-type", "x-meili-api-key"])
+                    .max_age(86_400) // 24h
+                    .finish(),
+            )
+            .wrap(middleware::Logger::default())
+            .wrap(middleware::Compress::default())
+            .wrap(NormalizePath)
+    });
 
-    app.middleware(Cors::new()
-        .allow_methods(HeaderValue::from_static("GET, POST, PUT, DELETE, OPTIONS"))
-        .allow_headers(HeaderValue::from_static("X-Meili-API-Key"))
-        .allow_origin(Origin::from("*")));
-    app.middleware(RequestLogger::new());
+    if let Some(config) = opt.get_ssl_config()? {
+        http_server
+            .bind_rustls(opt.http_addr, config)?
+            .run()
+            .await?;
+    } else {
+        http_server.bind(opt.http_addr)?.run().await?;
+    }
 
-    routes::load_routes(&mut app);
-
-    task::block_on(app.listen(opt.http_addr))?;
     Ok(())
 }
 
@@ -76,37 +104,52 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) {
 888       888  "Y8888  888 888 888  "Y8888P"   "Y8888  "Y888888 888     "Y8888P 888  888
 "#;
 
-    println!("{}", ascii_name);
+    eprintln!("{}", ascii_name);
 
-    info!("Database path: {:?}", opt.db_path);
-    info!("Start server on: {:?}", opt.http_addr);
-    info!("Environment: {:?}", opt.env);
-    info!("Commit SHA: {:?}", env!("VERGEN_SHA").to_string());
-    info!(
-        "Build date: {:?}",
+    eprintln!("Database path:\t\t{:?}", opt.db_path);
+    eprintln!("Server listening on:\t{:?}", opt.http_addr);
+    eprintln!("Environment:\t\t{:?}", opt.env);
+    eprintln!("Commit SHA:\t\t{:?}", env!("VERGEN_SHA").to_string());
+    eprintln!(
+        "Build date:\t\t{:?}",
         env!("VERGEN_BUILD_TIMESTAMP").to_string()
     );
-    info!(
-        "Package version: {:?}",
+    eprintln!(
+        "Package version:\t{:?}",
         env!("CARGO_PKG_VERSION").to_string()
     );
 
-    if let Some(master_key) = &data.api_keys.master {
-        info!("Master Key: {:?}", master_key);
-
-        if let Some(private_key) = &data.api_keys.private {
-            info!("Private Key: {:?}", private_key);
+    #[cfg(all(not(debug_assertions), feature = "sentry"))]
+    eprintln!(
+        "Sentry DSN:\t\t{:?}",
+        if !opt.no_sentry {
+            &opt.sentry_dsn
+        } else {
+            "Disabled"
         }
+    );
 
-        if let Some(public_key) = &data.api_keys.public {
-            info!("Public Key: {:?}", public_key);
+    eprintln!(
+        "Amplitude Analytics:\t{:?}",
+        if !opt.no_analytics {
+            "Enabled"
+        } else {
+            "Disabled"
         }
+    );
+
+    eprintln!();
+
+    if data.api_keys.master.is_some() {
+        eprintln!("A Master Key has been set. Requests to MeiliSearch won't be authorized unless you provide an authentication key.");
     } else {
-        info!("No master key found; The server will have no securities.\
-            If you need some protection in development mode, please export a key. export MEILI_MASTER_KEY=xxx");
+        eprintln!("No master key found; The server will accept unidentified requests. \
+            If you need some protection in development mode, please export a key: export MEILI_MASTER_KEY=xxx");
     }
 
-    info!("If you need extra information; Please refer to the documentation: http://docs.meilisearch.com");
-    info!("If you want to support us or help us; Please consult our Github repo: http://github.com/meilisearch/meilisearch");
-    info!("If you want to contact us; Please chat with us on http://meilisearch.com or by email to bonjour@meilisearch.com");
+    eprintln!();
+    eprintln!("Documentation:\t\thttps://docs.meilisearch.com");
+    eprintln!("Source code:\t\thttps://github.com/meilisearch/meilisearch");
+    eprintln!("Contact:\t\thttps://docs.meilisearch.com/resources/contact.html or bonjour@meilisearch.com");
+    eprintln!();
 }

meilisearch-http/src/option.rs

@@ -1,8 +1,18 @@
+use std::io::{BufReader, Read};
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::{error, fs};
+
+use rustls::internal::pemfile::{certs, pkcs8_private_keys, rsa_private_keys};
+use rustls::{
+    AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth,
+    RootCertStore,
+};
 use structopt::StructOpt;
 
 const POSSIBLE_ENV: [&str; 2] = ["development", "production"];
 
-#[derive(Debug, Clone, StructOpt)]
+#[derive(Debug, Default, Clone, StructOpt)]
 pub struct Opt {
     /// The destination where the database must be created.
     #[structopt(long, env = "MEILI_DB_PATH", default_value = "./data.ms")]
@@ -16,7 +26,18 @@ pub struct Opt {
     #[structopt(long, env = "MEILI_MASTER_KEY")]
     pub master_key: Option<String>,
 
-    /// This environment variable must be set to `production` if your are running in production.
+    /// The Sentry DSN to use for error reporting. This defaults to the MeiliSearch Sentry project.
+    /// You can disable sentry all together using the `--no-sentry` flag or `MEILI_NO_SENTRY` environment variable.
+    #[cfg(all(not(debug_assertions), feature = "sentry"))]
+    #[structopt(long, env = "SENTRY_DSN", default_value = "https://5ddfa22b95f241198be2271aaf028653@sentry.io/3060337")]
+    pub sentry_dsn: String,
+
+    /// Disable Sentry error reporting.
+    #[cfg(all(not(debug_assertions), feature = "sentry"))]
+    #[structopt(long, env = "MEILI_NO_SENTRY")]
+    pub no_sentry: bool,
+
+    /// This environment variable must be set to `production` if you are running in production.
     /// If the server is running in development mode more logs will be displayed,
     /// and the master key can be avoided which implies that there is no security on the updates routes.
     /// This is useful to debug when integrating the engine with another service.
@@ -26,4 +47,137 @@ pub struct Opt {
     /// Do not send analytics to Meili.
     #[structopt(long, env = "MEILI_NO_ANALYTICS")]
     pub no_analytics: bool,
+
+    /// The maximum size, in bytes, of the main lmdb database directory
+    #[structopt(long, env = "MEILI_MAIN_MAP_SIZE", default_value = "107374182400")] // 100GB
+    pub main_map_size: usize,
+
+    /// The maximum size, in bytes, of the update lmdb database directory
+    #[structopt(long, env = "MEILI_UPDATE_MAP_SIZE", default_value = "107374182400")] // 100GB
+    pub update_map_size: usize,
+
+    /// The maximum size, in bytes, of accepted JSON payloads
+    #[structopt(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "10485760")] // 10MB
+    pub http_payload_size_limit: usize,
+
+    /// Read server certificates from CERTFILE.
+    /// This should contain PEM-format certificates
+    /// in the right order (the first certificate should
+    /// certify KEYFILE, the last should be a root CA).
+    #[structopt(long, env = "MEILI_SSL_CERT_PATH", parse(from_os_str))]
+    pub ssl_cert_path: Option<PathBuf>,
+
+    /// Read private key from KEYFILE.  This should be a RSA
+    /// private key or PKCS8-encoded private key, in PEM format.
+    #[structopt(long, env = "MEILI_SSL_KEY_PATH", parse(from_os_str))]
+    pub ssl_key_path: Option<PathBuf>,
+
+    /// Enable client authentication, and accept certificates
+    /// signed by those roots provided in CERTFILE.
+    #[structopt(long, env = "MEILI_SSL_AUTH_PATH", parse(from_os_str))]
+    pub ssl_auth_path: Option<PathBuf>,
+
+    /// Read DER-encoded OCSP response from OCSPFILE and staple to certificate.
+    /// Optional
+    #[structopt(long, env = "MEILI_SSL_OCSP_PATH", parse(from_os_str))]
+    pub ssl_ocsp_path: Option<PathBuf>,
+
+    /// Send a fatal alert if the client does not complete client authentication.
+    #[structopt(long, env = "MEILI_SSL_REQUIRE_AUTH")]
+    pub ssl_require_auth: bool,
+
+    /// SSL support session resumption
+    #[structopt(long, env = "MEILI_SSL_RESUMPTION")]
+    pub ssl_resumption: bool,
+
+    /// SSL support tickets.
+    #[structopt(long, env = "MEILI_SSL_TICKETS")]
+    pub ssl_tickets: bool,
+}
+
+impl Opt {
+    pub fn get_ssl_config(&self) -> Result<Option<rustls::ServerConfig>, Box<dyn error::Error>> {
+        if let (Some(cert_path), Some(key_path)) = (&self.ssl_cert_path, &self.ssl_key_path) {
+            let client_auth = match &self.ssl_auth_path {
+                Some(auth_path) => {
+                    let roots = load_certs(auth_path.to_path_buf())?;
+                    let mut client_auth_roots = RootCertStore::empty();
+                    for root in roots {
+                        client_auth_roots.add(&root).unwrap();
+                    }
+                    if self.ssl_require_auth {
+                        AllowAnyAuthenticatedClient::new(client_auth_roots)
+                    } else {
+                        AllowAnyAnonymousOrAuthenticatedClient::new(client_auth_roots)
+                    }
+                }
+                None => NoClientAuth::new(),
+            };
+
+            let mut config = rustls::ServerConfig::new(client_auth);
+            config.key_log = Arc::new(rustls::KeyLogFile::new());
+
+            let certs = load_certs(cert_path.to_path_buf())?;
+            let privkey = load_private_key(key_path.to_path_buf())?;
+            let ocsp = load_ocsp(&self.ssl_ocsp_path)?;
+            config
+                .set_single_cert_with_ocsp_and_sct(certs, privkey, ocsp, vec![])
+                .map_err(|_| "bad certificates/private key")?;
+
+            if self.ssl_resumption {
+                config.set_persistence(rustls::ServerSessionMemoryCache::new(256));
+            }
+
+            if self.ssl_tickets {
+                config.ticketer = rustls::Ticketer::new();
+            }
+
+            Ok(Some(config))
+        } else {
+            Ok(None)
+        }
+    }
+}
+
+fn load_certs(filename: PathBuf) -> Result<Vec<rustls::Certificate>, Box<dyn error::Error>> {
+    let certfile = fs::File::open(filename).map_err(|_| "cannot open certificate file")?;
+    let mut reader = BufReader::new(certfile);
+    Ok(certs(&mut reader).map_err(|_| "cannot read certificate file")?)
+}
+
+fn load_private_key(filename: PathBuf) -> Result<rustls::PrivateKey, Box<dyn error::Error>> {
+    let rsa_keys = {
+        let keyfile =
+            fs::File::open(filename.clone()).map_err(|_| "cannot open private key file")?;
+        let mut reader = BufReader::new(keyfile);
+        rsa_private_keys(&mut reader).map_err(|_| "file contains invalid rsa private key")?
+    };
+
+    let pkcs8_keys = {
+        let keyfile = fs::File::open(filename).map_err(|_| "cannot open private key file")?;
+        let mut reader = BufReader::new(keyfile);
+        pkcs8_private_keys(&mut reader)
+            .map_err(|_| "file contains invalid pkcs8 private key (encrypted keys not supported)")?
+    };
+
+    // prefer to load pkcs8 keys
+    if !pkcs8_keys.is_empty() {
+        Ok(pkcs8_keys[0].clone())
+    } else {
+        assert!(!rsa_keys.is_empty());
+        Ok(rsa_keys[0].clone())
+    }
+}
+
+fn load_ocsp(filename: &Option<PathBuf>) -> Result<Vec<u8>, Box<dyn error::Error>> {
+    let mut ret = Vec::new();
+
+    if let Some(ref name) = filename {
+        fs::File::open(name)
+            .map_err(|_| "cannot open ocsp file")?
+            .read_to_end(&mut ret)
+            .map_err(|_| "cannot read oscp file")?;
+    }
+
+    Ok(ret)
 }

meilisearch-http/src/routes/document.rs

@@ -1,62 +1,83 @@
 use std::collections::{BTreeSet, HashSet};
 
+use actix_web::{web, HttpResponse};
+use actix_web_macros::{delete, get, post, put};
 use indexmap::IndexMap;
-use serde::{Deserialize, Serialize};
+use meilisearch_core::update;
+use serde::Deserialize;
 use serde_json::Value;
-use tide::{Request, Response};
 
-use crate::error::{ResponseError, SResult};
-use crate::helpers::tide::RequestExt;
-use crate::helpers::tide::ACL::*;
 use crate::Data;
+use crate::error::{Error, ResponseError};
+use crate::helpers::Authentication;
+use crate::routes::{IndexParam, IndexUpdateResponse};
 
-pub async fn get_document(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Public)?;
+type Document = IndexMap<String, Value>;
 
-    let index = ctx.index()?;
-
-    let original_document_id = ctx.document_id()?;
-    let document_id = meilisearch_core::serde::compute_document_id(original_document_id.clone());
-
-    let db = &ctx.state().db;
-    let reader = db.main_read_txn()?;
-
-    let response = index
-        .document::<IndexMap<String, Value>>(&reader, None, document_id)?
-        .ok_or(ResponseError::document_not_found(&original_document_id))?;
-
-    if response.is_empty() {
-        return Err(ResponseError::document_not_found(&original_document_id));
-    }
-
-    Ok(tide::Response::new(200).body_json(&response)?)
+#[derive(Deserialize)]
+struct DocumentParam {
+    index_uid: String,
+    document_id: String,
 }
 
-#[derive(Default, Serialize)]
-#[serde(rename_all = "camelCase")]
-pub struct IndexUpdateResponse {
-    pub update_id: u64,
+pub fn services(cfg: &mut web::ServiceConfig) {
+    cfg.service(get_document)
+        .service(delete_document)
+        .service(get_all_documents)
+        .service(add_documents)
+        .service(update_documents)
+        .service(delete_documents)
+        .service(clear_all_documents);
 }
 
-pub async fn delete_document(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
+#[get(
+    "/indexes/{index_uid}/documents/{document_id}",
+    wrap = "Authentication::Public"
+)]
+async fn get_document(
+    data: web::Data<Data>,
+    path: web::Path<DocumentParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
+
+    let reader = data.db.main_read_txn()?;
+
+    let internal_id = index.main
+        .external_to_internal_docid(&reader, &path.document_id)?
+        .ok_or(Error::document_not_found(&path.document_id))?;
+
+    let document: Document = index
+        .document(&reader, None, internal_id)?
+        .ok_or(Error::document_not_found(&path.document_id))?;
+
+    Ok(HttpResponse::Ok().json(document))
+}
+
+#[delete(
+    "/indexes/{index_uid}/documents/{document_id}",
+    wrap = "Authentication::Private"
+)]
+async fn delete_document(
+    data: web::Data<Data>,
+    path: web::Path<DocumentParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
-    let index = ctx.index()?;
-    let document_id = ctx.document_id()?;
-    let document_id = meilisearch_core::serde::compute_document_id(document_id);
-    let db = &ctx.state().db;
-    let mut update_writer = db.update_write_txn()?;
     let mut documents_deletion = index.documents_deletion();
-    documents_deletion.delete_document_by_id(document_id);
-    let update_id = documents_deletion.finalize(&mut update_writer)?;
+    documents_deletion.delete_document_by_external_docid(path.document_id.clone());
 
-    update_writer.commit()?;
+    let update_id = data.db.update_write(|w| documents_deletion.finalize(w))?;
 
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
 
-#[derive(Default, Deserialize)]
+#[derive(Deserialize)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 struct BrowseQuery {
     offset: Option<usize>,
@@ -64,18 +85,21 @@ struct BrowseQuery {
     attributes_to_retrieve: Option<String>,
 }
 
-pub async fn get_all_documents(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
+#[get("/indexes/{index_uid}/documents", wrap = "Authentication::Public")]
+async fn get_all_documents(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+    params: web::Query<BrowseQuery>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
-    let index = ctx.index()?;
-    let query: BrowseQuery = ctx.query().unwrap_or_default();
-
-    let offset = query.offset.unwrap_or(0);
-    let limit = query.limit.unwrap_or(20);
-
-    let db = &ctx.state().db;
-    let reader = db.main_read_txn()?;
+    let offset = params.offset.unwrap_or(0);
+    let limit = params.limit.unwrap_or(20);
 
+    let reader = data.db.main_read_txn()?;
     let documents_ids: Result<BTreeSet<_>, _> = index
         .documents_fields_counts
         .documents_ids(&reader)?
@@ -83,29 +107,21 @@ pub async fn get_all_documents(ctx: Request<Data>) -> SResult<Response> {
         .take(limit)
         .collect();
 
-    let documents_ids = match documents_ids {
-        Ok(documents_ids) => documents_ids,
-        Err(e) => return Err(ResponseError::internal(e)),
-    };
+    let attributes: Option<HashSet<&str>> = params
+        .attributes_to_retrieve
+        .as_ref()
+        .map(|a| a.split(',').collect());
 
-    let mut response_body = Vec::<IndexMap<String, Value>>::new();
-
-    if let Some(attributes) = query.attributes_to_retrieve {
-        let attributes = attributes.split(',').collect::<HashSet<&str>>();
-        for document_id in documents_ids {
-            if let Ok(Some(document)) = index.document(&reader, Some(&attributes), document_id) {
-                response_body.push(document);
-            }
-        }
-    } else {
-        for document_id in documents_ids {
-            if let Ok(Some(document)) = index.document(&reader, None, document_id) {
-                response_body.push(document);
-            }
+    let mut documents = Vec::new();
+    for document_id in documents_ids? {
+        if let Ok(Some(document)) =
+            index.document::<Document>(&reader, attributes.as_ref(), document_id)
+        {
+            documents.push(document);
         }
     }
 
-    Ok(tide::Response::new(200).body_json(&response_body)?)
+    Ok(HttpResponse::Ok().json(documents))
 }
 
 fn find_primary_key(document: &IndexMap<String, Value>) -> Option<String> {
@@ -117,42 +133,45 @@ fn find_primary_key(document: &IndexMap<String, Value>) -> Option<String> {
     None
 }
 
-#[derive(Default, Deserialize)]
+#[derive(Deserialize)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 struct UpdateDocumentsQuery {
     primary_key: Option<String>,
 }
 
-async fn update_multiple_documents(mut ctx: Request<Data>, is_partial: bool) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
+async fn update_multiple_documents(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+    params: web::Query<UpdateDocumentsQuery>,
+    body: web::Json<Vec<Document>>,
+    is_partial: bool,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
-    let index = ctx.index()?;
+    let reader = data.db.main_read_txn()?;
 
-    let data: Vec<IndexMap<String, Value>> =
-        ctx.body_json().await.map_err(ResponseError::bad_request)?;
-    let query: UpdateDocumentsQuery = ctx.query().unwrap_or_default();
-
-    let db = &ctx.state().db;
-
-    let reader = db.main_read_txn()?;
     let mut schema = index
         .main
         .schema(&reader)?
-        .ok_or(ResponseError::internal("schema not found"))?;
+        .ok_or(meilisearch_core::Error::SchemaMissing)?;
 
     if schema.primary_key().is_none() {
-        let id = match query.primary_key {
-            Some(id) => id,
-            None => match data.first().and_then(|docs| find_primary_key(docs)) {
-                Some(id) => id,
-                None => return Err(ResponseError::bad_request("Could not infer a primary key")),
-            },
+        let id = match &params.primary_key {
+            Some(id) => id.to_string(),
+            None => body
+                .first()
+                .and_then(find_primary_key)
+                .ok_or(meilisearch_core::Error::MissingPrimaryKey)?
         };
 
-        let mut writer = db.main_write_txn()?;
-        schema.set_primary_key(&id).map_err(ResponseError::bad_request)?;
-        index.main.put_schema(&mut writer, &schema)?;
-        writer.commit()?;
+        schema
+            .set_primary_key(&id)
+            .map_err(Error::bad_request)?;
+
+        data.db.main_write(|w| index.main.put_schema(w, &schema))?;
     }
 
     let mut document_addition = if is_partial {
@@ -161,63 +180,73 @@ async fn update_multiple_documents(mut ctx: Request<Data>, is_partial: bool) ->
         index.documents_addition()
     };
 
-    for document in data {
+    for document in body.into_inner() {
         document_addition.update_document(document);
     }
 
-    let mut update_writer = db.update_write_txn()?;
-    let update_id = document_addition.finalize(&mut update_writer)?;
-    update_writer.commit()?;
+    let update_id = data.db.update_write(|w| document_addition.finalize(w))?;
 
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
 
-pub async fn add_or_replace_multiple_documents(ctx: Request<Data>) -> SResult<Response> {
-    update_multiple_documents(ctx, false).await
+#[post("/indexes/{index_uid}/documents", wrap = "Authentication::Private")]
+async fn add_documents(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+    params: web::Query<UpdateDocumentsQuery>,
+    body: web::Json<Vec<Document>>,
+) -> Result<HttpResponse, ResponseError> {
+    update_multiple_documents(data, path, params, body, false).await
 }
 
-pub async fn add_or_update_multiple_documents(ctx: Request<Data>) -> SResult<Response> {
-    update_multiple_documents(ctx, true).await
+#[put("/indexes/{index_uid}/documents", wrap = "Authentication::Private")]
+async fn update_documents(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+    params: web::Query<UpdateDocumentsQuery>,
+    body: web::Json<Vec<Document>>,
+) -> Result<HttpResponse, ResponseError> {
+    update_multiple_documents(data, path, params, body, true).await
 }
 
-pub async fn delete_multiple_documents(mut ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
+#[post(
+    "/indexes/{index_uid}/documents/delete-batch",
+    wrap = "Authentication::Private"
+)]
+async fn delete_documents(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+    body: web::Json<Vec<Value>>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
-    let data: Vec<Value> = ctx.body_json().await.map_err(ResponseError::bad_request)?;
-    let index = ctx.index()?;
-
-    let db = &ctx.state().db;
-    let mut writer = db.update_write_txn()?;
 
     let mut documents_deletion = index.documents_deletion();
 
-    for document_id in data {
-        if let Some(document_id) = meilisearch_core::serde::value_to_string(&document_id) {
-            documents_deletion
-                .delete_document_by_id(meilisearch_core::serde::compute_document_id(document_id));
-        }
+    for document_id in body.into_inner() {
+        let document_id = update::value_to_string(&document_id);
+        documents_deletion.delete_document_by_external_docid(document_id);
     }
 
-    let update_id = documents_deletion.finalize(&mut writer)?;
+    let update_id = data.db.update_write(|w| documents_deletion.finalize(w))?;
 
-    writer.commit()?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
 
-pub async fn clear_all_documents(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
+#[delete("/indexes/{index_uid}/documents", wrap = "Authentication::Private")]
+async fn clear_all_documents(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
-    let index = ctx.index()?;
+    let update_id = data.db.update_write(|w| index.clear_all(w))?;
 
-    let db = &ctx.state().db;
-    let mut writer = db.update_write_txn()?;
-
-    let update_id = index.clear_all(&mut writer)?;
-    writer.commit()?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }

meilisearch-http/src/routes/health.rs

@@ -1,60 +1,47 @@
-use crate::error::{ResponseError, SResult};
-use crate::helpers::tide::RequestExt;
-use crate::helpers::tide::ACL::*;
+use actix_web::{web, HttpResponse};
+use actix_web_macros::{get, put};
+use serde::Deserialize;
+
+use crate::error::{Error, ResponseError};
+use crate::helpers::Authentication;
 use crate::Data;
 
-use heed::types::{Str, Unit};
-use serde::Deserialize;
-use tide::{Request, Response};
+pub fn services(cfg: &mut web::ServiceConfig) {
+    cfg.service(get_health).service(change_healthyness);
+}
 
-const UNHEALTHY_KEY: &str = "_is_unhealthy";
-
-pub async fn get_health(ctx: Request<Data>) -> SResult<Response> {
-    let db = &ctx.state().db;
-    let reader = db.main_read_txn()?;
-
-    let common_store = ctx.state().db.common_store();
-
-    if let Ok(Some(_)) = common_store.get::<_, Str, Unit>(&reader, UNHEALTHY_KEY) {
-        return Err(ResponseError::Maintenance);
+#[get("/health")]
+async fn get_health(data: web::Data<Data>) -> Result<HttpResponse, ResponseError> {
+    let reader = data.db.main_read_txn()?;
+    if let Ok(Some(_)) = data.db.get_health(&reader) {
+        return Err(Error::Maintenance.into());
     }
-
-    Ok(tide::Response::new(200))
+    Ok(HttpResponse::Ok().finish())
 }
 
-pub async fn set_healthy(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Admin)?;
-    let db = &ctx.state().db;
-    let mut writer = db.main_write_txn()?;
-    let common_store = ctx.state().db.common_store();
-    common_store.delete::<_, Str>(&mut writer, UNHEALTHY_KEY)?;
-    writer.commit()?;
-
-    Ok(tide::Response::new(200))
+async fn set_healthy(data: web::Data<Data>) -> Result<HttpResponse, ResponseError> {
+    data.db.main_write(|w| data.db.set_healthy(w))?;
+    Ok(HttpResponse::Ok().finish())
 }
 
-pub async fn set_unhealthy(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Admin)?;
-    let db = &ctx.state().db;
-    let mut writer = db.main_write_txn()?;
-    let common_store = ctx.state().db.common_store();
-    common_store.put::<_, Str, Unit>(&mut writer, UNHEALTHY_KEY, &())?;
-    writer.commit()?;
-
-    Ok(tide::Response::new(200))
+async fn set_unhealthy(data: web::Data<Data>) -> Result<HttpResponse, ResponseError> {
+    data.db.main_write(|w| data.db.set_unhealthy(w))?;
+    Ok(HttpResponse::Ok().finish())
 }
 
 #[derive(Deserialize, Clone)]
-struct HealtBody {
+struct HealthBody {
     health: bool,
 }
 
-pub async fn change_healthyness(mut ctx: Request<Data>) -> SResult<Response> {
-    let body: HealtBody = ctx.body_json().await.map_err(ResponseError::bad_request)?;
-
+#[put("/health", wrap = "Authentication::Private")]
+async fn change_healthyness(
+    data: web::Data<Data>,
+    body: web::Json<HealthBody>,
+) -> Result<HttpResponse, ResponseError> {
     if body.health {
-        set_healthy(ctx).await
+        set_healthy(data).await
     } else {
-        set_unhealthy(ctx).await
+        set_unhealthy(data).await
     }
 }

meilisearch-http/src/routes/index.rs

@@ -1,16 +1,25 @@
+use actix_web::{web, HttpResponse};
+use actix_web_macros::{delete, get, post, put};
 use chrono::{DateTime, Utc};
 use log::error;
-use meilisearch_core::ProcessedUpdateResult;
 use rand::seq::SliceRandom;
 use serde::{Deserialize, Serialize};
-use serde_json::json;
-use tide::{Request, Response};
 
-use crate::error::{IntoInternalError, ResponseError, SResult};
-use crate::helpers::tide::RequestExt;
-use crate::helpers::tide::ACL::*;
+use crate::error::{Error, ResponseError};
+use crate::helpers::Authentication;
+use crate::routes::IndexParam;
 use crate::Data;
 
+pub fn services(cfg: &mut web::ServiceConfig) {
+    cfg.service(list_indexes)
+        .service(get_index)
+        .service(create_index)
+        .service(update_index)
+        .service(delete_index)
+        .service(get_update_status)
+        .service(get_all_updates_status);
+}
+
 fn generate_uid() -> String {
     let mut rng = rand::thread_rng();
     let sample = b"abcdefghijklmnopqrstuvwxyz0123456789";
@@ -20,24 +29,41 @@ fn generate_uid() -> String {
         .collect()
 }
 
-pub async fn list_indexes(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct IndexResponse {
+    name: String,
+    uid: String,
+    created_at: DateTime<Utc>,
+    updated_at: DateTime<Utc>,
+    primary_key: Option<String>,
+}
 
-    let indexes_uids = ctx.state().db.indexes_uids();
+#[get("/indexes", wrap = "Authentication::Private")]
+async fn list_indexes(data: web::Data<Data>) -> Result<HttpResponse, ResponseError> {
+    let reader = data.db.main_read_txn()?;
+    let mut indexes = Vec::new();
 
-    let db = &ctx.state().db;
-    let reader = db.main_read_txn()?;
-
-    let mut response_body = Vec::new();
-
-    for index_uid in indexes_uids {
-        let index = ctx.state().db.open_index(&index_uid);
+    for index_uid in data.db.indexes_uids() {
+        let index = data.db.open_index(&index_uid);
 
         match index {
             Some(index) => {
-                let name = index.main.name(&reader)?.into_internal_error()?;
-                let created_at = index.main.created_at(&reader)?.into_internal_error()?;
-                let updated_at = index.main.updated_at(&reader)?.into_internal_error()?;
+                let name = index.main.name(&reader)?.ok_or(Error::internal(
+                        "Impossible to get the name of an index",
+                ))?;
+                let created_at = index
+                    .main
+                    .created_at(&reader)?
+                    .ok_or(Error::internal(
+                            "Impossible to get the create date of an index",
+                    ))?;
+                let updated_at = index
+                    .main
+                    .updated_at(&reader)?
+                    .ok_or(Error::internal(
+                            "Impossible to get the last update date of an index",
+                    ))?;
 
                 let primary_key = match index.main.schema(&reader) {
                     Ok(Some(schema)) => match schema.primary_key() {
@@ -54,7 +80,7 @@ pub async fn list_indexes(ctx: Request<Data>) -> SResult<Response> {
                     updated_at,
                     primary_key,
                 };
-                response_body.push(index_response);
+                indexes.push(index_response);
             }
             None => error!(
                 "Index {} is referenced in the indexes list but cannot be found",
@@ -63,31 +89,35 @@ pub async fn list_indexes(ctx: Request<Data>) -> SResult<Response> {
         }
     }
 
-    Ok(tide::Response::new(200).body_json(&response_body)?)
+    Ok(HttpResponse::Ok().json(indexes))
 }
 
-#[derive(Debug, Serialize)]
-#[serde(rename_all = "camelCase")]
-struct IndexResponse {
-    name: String,
-    uid: String,
-    created_at: DateTime<Utc>,
-    updated_at: DateTime<Utc>,
-    primary_key: Option<String>,
-}
+#[get("/indexes/{index_uid}", wrap = "Authentication::Private")]
+async fn get_index(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
-pub async fn get_index(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-
-    let index = ctx.index()?;
-
-    let db = &ctx.state().db;
-    let reader = db.main_read_txn()?;
-
-    let uid = ctx.url_param("index")?;
-    let name = index.main.name(&reader)?.into_internal_error()?;
-    let created_at = index.main.created_at(&reader)?.into_internal_error()?;
-    let updated_at = index.main.updated_at(&reader)?.into_internal_error()?;
+    let reader = data.db.main_read_txn()?;
+    let name = index.main.name(&reader)?.ok_or(Error::internal(
+            "Impossible to get the name of an index",
+    ))?;
+    let created_at = index
+        .main
+        .created_at(&reader)?
+        .ok_or(Error::internal(
+                "Impossible to get the create date of an index",
+        ))?;
+    let updated_at = index
+        .main
+        .updated_at(&reader)?
+        .ok_or(Error::internal(
+                "Impossible to get the last update date of an index",
+        ))?;
 
     let primary_key = match index.main.schema(&reader) {
         Ok(Some(schema)) => match schema.primary_key() {
@@ -96,16 +126,15 @@ pub async fn get_index(ctx: Request<Data>) -> SResult<Response> {
         },
         _ => None,
     };
-
-    let response_body = IndexResponse {
+    let index_response = IndexResponse {
         name,
-        uid,
+        uid: path.index_uid.clone(),
         created_at,
         updated_at,
         primary_key,
     };
 
-    Ok(tide::Response::new(200).body_json(&response_body)?)
+    Ok(HttpResponse::Ok().json(index_response))
 }
 
 #[derive(Debug, Deserialize)]
@@ -116,86 +145,77 @@ struct IndexCreateRequest {
     primary_key: Option<String>,
 }
 
-#[derive(Debug, Serialize)]
-#[serde(rename_all = "camelCase")]
-struct IndexCreateResponse {
-    name: String,
-    uid: String,
-    created_at: DateTime<Utc>,
-    updated_at: DateTime<Utc>,
-    primary_key: Option<String>,
-}
-
-pub async fn create_index(mut ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-
-    let body = ctx
-        .body_json::<IndexCreateRequest>()
-        .await
-        .map_err(ResponseError::bad_request)?;
-
+#[post("/indexes", wrap = "Authentication::Private")]
+async fn create_index(
+    data: web::Data<Data>,
+    body: web::Json<IndexCreateRequest>,
+) -> Result<HttpResponse, ResponseError> {
     if let (None, None) = (body.name.clone(), body.uid.clone()) {
-        return Err(ResponseError::bad_request(
+        return Err(Error::bad_request(
             "Index creation must have an uid",
-        ));
+        ).into());
     }
 
-    let db = &ctx.state().db;
-
-    let uid = match body.uid {
+    let uid = match &body.uid {
         Some(uid) => {
             if uid
                 .chars()
                 .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_')
             {
-                uid
+                uid.to_owned()
             } else {
-                return Err(ResponseError::InvalidIndexUid);
+                return Err(Error::InvalidIndexUid.into());
             }
         }
         None => loop {
             let uid = generate_uid();
-            if db.open_index(&uid).is_none() {
+            if data.db.open_index(&uid).is_none() {
                 break uid;
             }
         },
     };
 
-    let created_index = match db.create_index(&uid) {
-        Ok(index) => index,
-        Err(e) => return Err(ResponseError::create_index(e)),
-    };
+    let created_index = data
+        .db
+        .create_index(&uid)
+        .map_err(|e| match e {
+            meilisearch_core::Error::IndexAlreadyExists => e.into(),
+            _ => ResponseError::from(Error::create_index(e))
+        })?;
 
-    let mut writer = db.main_write_txn()?;
-    let name = body.name.unwrap_or(uid.clone());
-    created_index.main.put_name(&mut writer, &name)?;
-    let created_at = created_index
-        .main
-        .created_at(&writer)?
-        .into_internal_error()?;
-    let updated_at = created_index
-        .main
-        .updated_at(&writer)?
-        .into_internal_error()?;
+    let index_response = data.db.main_write::<_, _, ResponseError>(|mut writer| {
+        let name = body.name.as_ref().unwrap_or(&uid);
+        created_index.main.put_name(&mut writer, name)?;
 
-    if let Some(id) = body.primary_key.clone() {
-        if let Some(mut schema) = created_index.main.schema(&mut writer)? {
-            schema.set_primary_key(&id).map_err(ResponseError::bad_request)?;
-            created_index.main.put_schema(&mut writer, &schema)?;
+        let created_at = created_index
+            .main
+            .created_at(&writer)?
+            .ok_or(Error::internal("Impossible to read created at"))?;
+
+        let updated_at = created_index
+            .main
+            .updated_at(&writer)?
+            .ok_or(Error::internal("Impossible to read updated at"))?;
+
+        if let Some(id) = body.primary_key.clone() {
+            if let Some(mut schema) = created_index.main.schema(&writer)? {
+                schema
+                    .set_primary_key(&id)
+                    .map_err(Error::bad_request)?;
+                created_index.main.put_schema(&mut writer, &schema)?;
+            }
         }
-    }
+        let index_response = IndexResponse {
+            name: name.to_string(),
+            uid,
+            created_at,
+            updated_at,
+            primary_key: body.primary_key.clone(),
+        };
+        Ok(index_response)
+    })?;
 
-    writer.commit()?;
-
-    let response_body = IndexCreateResponse {
-        name,
-        uid,
-        created_at,
-        updated_at,
-        primary_key: body.primary_key,
-    };
-
-    Ok(tide::Response::new(201).body_json(&response_body)?)
+    Ok(HttpResponse::Created().json(index_response))
 }
 
 #[derive(Debug, Deserialize)]
@@ -215,49 +235,48 @@ struct UpdateIndexResponse {
     primary_key: Option<String>,
 }
 
-pub async fn update_index(mut ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
+#[put("/indexes/{index_uid}", wrap = "Authentication::Private")]
+async fn update_index(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+    body: web::Json<IndexCreateRequest>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
-    let body = ctx
-        .body_json::<UpdateIndexRequest>()
-        .await
-        .map_err(ResponseError::bad_request)?;
+    data.db.main_write::<_, _, ResponseError>(|writer| {
+        if let Some(name) = &body.name {
+            index.main.put_name(writer, name)?;
+        }
 
-    let index_uid = ctx.url_param("index")?;
-    let index = ctx.index()?;
-
-    let db = &ctx.state().db;
-    let mut writer = db.main_write_txn()?;
-
-    if let Some(name) = body.name {
-        index.main.put_name(&mut writer, &name)?;
-    }
-
-    if let Some(id) = body.primary_key.clone() {
-        if let Some(mut schema) = index.main.schema(&mut writer)? {
-            match schema.primary_key() {
-                Some(_) => {
-                    return Err(ResponseError::bad_request(
-                        "The primary key cannot be updated",
-                    ));
-                }
-                None => {
-                    schema
-                        .set_primary_key(&id)
-                        .map_err(ResponseError::bad_request)?;
-                    index.main.put_schema(&mut writer, &schema)?;
-                }
+        if let Some(id) = body.primary_key.clone() {
+            if let Some(mut schema) = index.main.schema(writer)? {
+                schema.set_primary_key(&id)?;
+                index.main.put_schema(writer, &schema)?;
             }
         }
-    }
+        index.main.put_updated_at(writer)?;
+        Ok(())
+    })?;
 
-    index.main.put_updated_at(&mut writer)?;
-    writer.commit()?;
-
-    let reader = db.main_read_txn()?;
-    let name = index.main.name(&reader)?.into_internal_error()?;
-    let created_at = index.main.created_at(&reader)?.into_internal_error()?;
-    let updated_at = index.main.updated_at(&reader)?.into_internal_error()?;
+    let reader = data.db.main_read_txn()?;
+    let name = index.main.name(&reader)?.ok_or(Error::internal(
+            "Impossible to get the name of an index",
+    ))?;
+    let created_at = index
+        .main
+        .created_at(&reader)?
+        .ok_or(Error::internal(
+                "Impossible to get the create date of an index",
+        ))?;
+    let updated_at = index
+        .main
+        .updated_at(&reader)?
+        .ok_or(Error::internal(
+                "Impossible to get the last update date of an index",
+        ))?;
 
     let primary_key = match index.main.schema(&reader) {
         Ok(Some(schema)) => match schema.primary_key() {
@@ -267,86 +286,74 @@ pub async fn update_index(mut ctx: Request<Data>) -> SResult<Response> {
         _ => None,
     };
 
-    let response_body = UpdateIndexResponse {
+    let index_response = IndexResponse {
         name,
-        uid: index_uid,
+        uid: path.index_uid.clone(),
         created_at,
         updated_at,
         primary_key,
     };
 
-    Ok(tide::Response::new(200).body_json(&response_body)?)
+    Ok(HttpResponse::Ok().json(index_response))
 }
 
-pub async fn get_update_status(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-
-    let db = &ctx.state().db;
-    let reader = db.update_read_txn()?;
-
-    let update_id = ctx
-        .param::<u64>("update_id")
-        .map_err(|e| ResponseError::bad_parameter("update_id", e))?;
-
-    let index = ctx.index()?;
-    let status = index.update_status(&reader, update_id)?;
-
-    let response = match status {
-        Some(status) => tide::Response::new(200).body_json(&status).unwrap(),
-        None => tide::Response::new(404)
-            .body_json(&json!({ "message": "unknown update id" }))
-            .unwrap(),
-    };
-
-    Ok(response)
+#[delete("/indexes/{index_uid}", wrap = "Authentication::Private")]
+async fn delete_index(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    if data.db.delete_index(&path.index_uid)? {
+        Ok(HttpResponse::NoContent().finish())
+    } else {
+        Err(Error::index_not_found(&path.index_uid).into())
+    }
 }
 
-pub async fn get_all_updates_status(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let db = &ctx.state().db;
-    let reader = db.update_read_txn()?;
-    let index = ctx.index()?;
+#[derive(Deserialize)]
+struct UpdateParam {
+    index_uid: String,
+    update_id: u64,
+}
+
+#[get(
+    "/indexes/{index_uid}/updates/{update_id}",
+    wrap = "Authentication::Private"
+)]
+async fn get_update_status(
+    data: web::Data<Data>,
+    path: web::Path<UpdateParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
+
+    let reader = data.db.update_read_txn()?;
+
+    let status = index.update_status(&reader, path.update_id)?;
+
+    match status {
+        Some(status) => Ok(HttpResponse::Ok().json(status)),
+        None => Err(Error::NotFound(format!(
+            "Update {}",
+            path.update_id
+        )).into()),
+    }
+}
+
+#[get("/indexes/{index_uid}/updates", wrap = "Authentication::Private")]
+async fn get_all_updates_status(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
+
+    let reader = data.db.update_read_txn()?;
+
     let response = index.all_updates_status(&reader)?;
-    Ok(tide::Response::new(200).body_json(&response).unwrap())
-}
-
-pub async fn delete_index(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let _ = ctx.index()?;
-    let index_uid = ctx.url_param("index")?;
-    ctx.state().db.delete_index(&index_uid)?;
-    Ok(tide::Response::new(204))
-}
-
-pub fn index_update_callback(index_uid: &str, data: &Data, status: ProcessedUpdateResult) {
-    if status.error.is_some() {
-        return;
-    }
-
-    if let Some(index) = data.db.open_index(&index_uid) {
-        let db = &data.db;
-        let mut writer = match db.main_write_txn() {
-            Ok(writer) => writer,
-            Err(e) => {
-                error!("Impossible to get write_txn; {}", e);
-                return;
-            }
-        };
-
-        if let Err(e) = data.compute_stats(&mut writer, &index_uid) {
-            error!("Impossible to compute stats; {}", e)
-        }
-
-        if let Err(e) = data.set_last_update(&mut writer) {
-            error!("Impossible to update last_update; {}", e)
-        }
-
-        if let Err(e) = index.main.put_updated_at(&mut writer) {
-            error!("Impossible to update updated_at; {}", e)
-        }
-
-        if let Err(e) = writer.commit() {
-            error!("Impossible to get write_txn; {}", e);
-        }
-    }
+
+    Ok(HttpResponse::Ok().json(response))
 }

meilisearch-http/src/routes/key.rs

@@ -1,17 +1,26 @@
-use crate::error::SResult;
-use crate::helpers::tide::RequestExt;
-use crate::helpers::tide::ACL::*;
+use actix_web::web;
+use actix_web::HttpResponse;
+use actix_web_macros::get;
+use serde::Serialize;
+
+use crate::helpers::Authentication;
 use crate::Data;
-use serde_json::json;
-use tide::{Request, Response};
 
-pub async fn list(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Admin)?;
-
-    let keys = &ctx.state().api_keys;
-
-    Ok(tide::Response::new(200).body_json(&json!({
-        "private": keys.private,
-        "public": keys.public,
-    }))?)
+pub fn services(cfg: &mut web::ServiceConfig) {
+    cfg.service(list);
+}
+
+#[derive(Serialize)]
+struct KeysResponse {
+    private: Option<String>,
+    public: Option<String>,
+}
+
+#[get("/keys", wrap = "Authentication::Admin")]
+async fn list(data: web::Data<Data>) -> HttpResponse {
+    let api_keys = data.api_keys.clone();
+    HttpResponse::Ok().json(KeysResponse {
+        private: api_keys.private,
+        public: api_keys.public,
+    })
 }

meilisearch-http/src/routes/mod.rs

@@ -1,7 +1,5 @@
-use crate::data::Data;
-use std::future::Future;
-use tide::IntoResponse;
-use tide::Response;
+use actix_web::{get, HttpResponse};
+use serde::{Deserialize, Serialize};
 
 pub mod document;
 pub mod health;
@@ -13,118 +11,33 @@ pub mod stats;
 pub mod stop_words;
 pub mod synonym;
 
-async fn into_response<T: IntoResponse, U: IntoResponse>(
-    x: impl Future<Output = Result<T, U>>,
-) -> Response {
-    match x.await {
-        Ok(resp) => resp.into_response(),
-        Err(resp) => resp.into_response(),
+#[derive(Deserialize)]
+pub struct IndexParam {
+    index_uid: String,
+}
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct IndexUpdateResponse {
+    pub update_id: u64,
+}
+
+impl IndexUpdateResponse {
+    pub fn with_id(update_id: u64) -> Self {
+        Self { update_id }
     }
 }
 
-pub fn load_routes(app: &mut tide::Server<Data>) {
-    app.at("/").get(|_| async {
-        tide::Response::new(200)
-            .body_string(include_str!("../../public/interface.html").to_string())
-            .set_mime(mime::TEXT_HTML_UTF_8)
-    });
-    app.at("/bulma.min.css").get(|_| async {
-        tide::Response::new(200)
-            .body_string(include_str!("../../public/bulma.min.css").to_string())
-            .set_mime(mime::TEXT_CSS_UTF_8)
-    });
-
-    app.at("/indexes")
-        .get(|ctx| into_response(index::list_indexes(ctx)))
-        .post(|ctx| into_response(index::create_index(ctx)));
-
-    app.at("/indexes/search")
-        .post(|ctx| into_response(search::search_multi_index(ctx)));
-
-    app.at("/indexes/:index")
-        .get(|ctx| into_response(index::get_index(ctx)))
-        .put(|ctx| into_response(index::update_index(ctx)))
-        .delete(|ctx| into_response(index::delete_index(ctx)));
-
-    app.at("/indexes/:index/search")
-        .get(|ctx| into_response(search::search_with_url_query(ctx)));
-
-    app.at("/indexes/:index/updates")
-        .get(|ctx| into_response(index::get_all_updates_status(ctx)));
-
-    app.at("/indexes/:index/updates/:update_id")
-        .get(|ctx| into_response(index::get_update_status(ctx)));
-
-    app.at("/indexes/:index/documents")
-        .get(|ctx| into_response(document::get_all_documents(ctx)))
-        .post(|ctx| into_response(document::add_or_replace_multiple_documents(ctx)))
-        .put(|ctx| into_response(document::add_or_update_multiple_documents(ctx)))
-        .delete(|ctx| into_response(document::clear_all_documents(ctx)));
-
-    app.at("/indexes/:index/documents/:document_id")
-        .get(|ctx| into_response(document::get_document(ctx)))
-        .delete(|ctx| into_response(document::delete_document(ctx)));
-
-    app.at("/indexes/:index/documents/delete-batch")
-        .post(|ctx| into_response(document::delete_multiple_documents(ctx)));
-
-    app.at("/indexes/:index/settings")
-        .get(|ctx| into_response(setting::get_all(ctx)))
-        .post(|ctx| into_response(setting::update_all(ctx)))
-        .delete(|ctx| into_response(setting::delete_all(ctx)));
-
-    app.at("/indexes/:index/settings/ranking-rules")
-        .get(|ctx| into_response(setting::get_rules(ctx)))
-        .post(|ctx| into_response(setting::update_rules(ctx)))
-        .delete(|ctx| into_response(setting::delete_rules(ctx)));
-
-    app.at("/indexes/:index/settings/distinct-attribute")
-        .get(|ctx| into_response(setting::get_distinct(ctx)))
-        .post(|ctx| into_response(setting::update_distinct(ctx)))
-        .delete(|ctx| into_response(setting::delete_distinct(ctx)));
-
-    app.at("/indexes/:index/settings/searchable-attributes")
-        .get(|ctx| into_response(setting::get_searchable(ctx)))
-        .post(|ctx| into_response(setting::update_searchable(ctx)))
-        .delete(|ctx| into_response(setting::delete_searchable(ctx)));
-
-    app.at("/indexes/:index/settings/displayed-attributes")
-        .get(|ctx| into_response(setting::displayed(ctx)))
-        .post(|ctx| into_response(setting::update_displayed(ctx)))
-        .delete(|ctx| into_response(setting::delete_displayed(ctx)));
-
-    app.at("/indexes/:index/settings/accept-new-fields")
-        .get(|ctx| into_response(setting::get_accept_new_fields(ctx)))
-        .post(|ctx| into_response(setting::update_accept_new_fields(ctx)));
-
-    app.at("/indexes/:index/settings/synonyms")
-        .get(|ctx| into_response(synonym::get(ctx)))
-        .post(|ctx| into_response(synonym::update(ctx)))
-        .delete(|ctx| into_response(synonym::delete(ctx)));
-
-    app.at("/indexes/:index/settings/stop-words")
-        .get(|ctx| into_response(stop_words::get(ctx)))
-        .post(|ctx| into_response(stop_words::update(ctx)))
-        .delete(|ctx| into_response(stop_words::delete(ctx)));
-
-    app.at("/indexes/:index/stats")
-        .get(|ctx| into_response(stats::index_stats(ctx)));
-
-    app.at("/keys").get(|ctx| into_response(key::list(ctx)));
-
-    app.at("/health")
-        .get(|ctx| into_response(health::get_health(ctx)))
-        .put(|ctx| into_response(health::change_healthyness(ctx)));
-
-    app.at("/stats")
-        .get(|ctx| into_response(stats::get_stats(ctx)));
-
-    app.at("/version")
-        .get(|ctx| into_response(stats::get_version(ctx)));
-
-    app.at("/sys-info")
-        .get(|ctx| into_response(stats::get_sys_info(ctx)));
-
-    app.at("/sys-info/pretty")
-        .get(|ctx| into_response(stats::get_sys_info_pretty(ctx)));
+#[get("/")]
+pub async fn load_html() -> HttpResponse {
+    HttpResponse::Ok()
+        .content_type("text/html; charset=utf-8")
+        .body(include_str!("../../public/interface.html").to_string())
+}
+
+#[get("/bulma.min.css")]
+pub async fn load_css() -> HttpResponse {
+    HttpResponse::Ok()
+        .content_type("text/css; charset=utf-8")
+        .body(include_str!("../../public/bulma.min.css").to_string())
 }

meilisearch-http/src/routes/search.rs

@@ -1,23 +1,30 @@
-use std::collections::HashMap;
-use std::collections::HashSet;
-use std::time::Duration;
+use std::collections::{HashSet, HashMap};
 
 use log::warn;
-use meilisearch_core::Index;
-use rayon::iter::{IntoParallelIterator, ParallelIterator};
+use actix_web::web;
+use actix_web::HttpResponse;
+use actix_web_macros::{get, post};
 use serde::{Deserialize, Serialize};
-use tide::{Request, Response};
+use serde_json::Value;
 
-use crate::error::{ResponseError, SResult};
-use crate::helpers::meilisearch::{Error, IndexSearchExt, SearchHit};
-use crate::helpers::tide::RequestExt;
-use crate::helpers::tide::ACL::*;
+use crate::error::{Error, FacetCountError, ResponseError};
+use crate::helpers::meilisearch::{IndexSearchExt, SearchResult};
+use crate::helpers::Authentication;
+use crate::routes::IndexParam;
 use crate::Data;
 
-#[derive(Deserialize)]
+use meilisearch_core::facets::FacetFilter;
+use meilisearch_schema::{Schema, FieldId};
+
+pub fn services(cfg: &mut web::ServiceConfig) {
+    cfg.service(search_with_post)
+        .service(search_with_url_query);
+}
+
+#[derive(Serialize, Deserialize)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
-struct SearchQuery {
-    q: String,
+pub struct SearchQuery {
+    q: Option<String>,
     offset: Option<usize>,
     limit: Option<usize>,
     attributes_to_retrieve: Option<String>,
@@ -25,244 +32,217 @@ struct SearchQuery {
     crop_length: Option<usize>,
     attributes_to_highlight: Option<String>,
     filters: Option<String>,
-    timeout_ms: Option<u64>,
     matches: Option<bool>,
+    facet_filters: Option<String>,
+    facets_distribution: Option<String>,
 }
 
-pub async fn search_with_url_query(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Public)?;
+#[get("/indexes/{index_uid}/search", wrap = "Authentication::Public")]
+async fn search_with_url_query(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+    params: web::Query<SearchQuery>,
+) -> Result<HttpResponse, ResponseError> {
+    let search_result = params.search(&path.index_uid, data)?;
+    Ok(HttpResponse::Ok().json(search_result))
+}
 
-    let index = ctx.index()?;
-    let db = &ctx.state().db;
-    let reader = db.main_read_txn()?;
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+pub struct SearchQueryPost {
+    q: Option<String>,
+    offset: Option<usize>,
+    limit: Option<usize>,
+    attributes_to_retrieve: Option<Vec<String>>,
+    attributes_to_crop: Option<Vec<String>>,
+    crop_length: Option<usize>,
+    attributes_to_highlight: Option<Vec<String>>,
+    filters: Option<String>,
+    matches: Option<bool>,
+    facet_filters: Option<Value>,
+    facets_distribution: Option<Vec<String>>,
+}
 
-    let schema = index
-        .main
-        .schema(&reader)?
-        .ok_or(ResponseError::open_index("No Schema found"))?;
-
-    let query: SearchQuery = ctx
-        .query()
-        .map_err(|_| ResponseError::bad_request("invalid query parameter"))?;
-
-    let mut search_builder = index.new_search(query.q.clone());
-
-    if let Some(offset) = query.offset {
-        search_builder.offset(offset);
-    }
-    if let Some(limit) = query.limit {
-        search_builder.limit(limit);
+impl From<SearchQueryPost> for SearchQuery {
+    fn from(other: SearchQueryPost) -> SearchQuery {
+        SearchQuery {
+            q: other.q,
+            offset: other.offset,
+            limit: other.limit,
+            attributes_to_retrieve: other.attributes_to_retrieve.map(|attrs| attrs.join(",")),
+            attributes_to_crop: other.attributes_to_crop.map(|attrs| attrs.join(",")),
+            crop_length: other.crop_length,
+            attributes_to_highlight: other.attributes_to_highlight.map(|attrs| attrs.join(",")),
+            filters: other.filters,
+            matches: other.matches,
+            facet_filters: other.facet_filters.map(|f| f.to_string()),
+            facets_distribution: other.facets_distribution.map(|f| format!("{:?}", f)),
+        }
     }
+}
 
-    let available_attributes = schema.displayed_name();
-    let mut restricted_attributes: HashSet<&str>;
-    match &query.attributes_to_retrieve {
-        Some(attributes_to_retrieve) => {
-            let attributes_to_retrieve: HashSet<&str> = attributes_to_retrieve.split(',').collect();
-            if attributes_to_retrieve.contains("*") {
+#[post("/indexes/{index_uid}/search", wrap = "Authentication::Public")]
+async fn search_with_post(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+    params: web::Json<SearchQueryPost>,
+) -> Result<HttpResponse, ResponseError> {
+    let query: SearchQuery = params.0.into();
+    let search_result = query.search(&path.index_uid, data)?;
+    Ok(HttpResponse::Ok().json(search_result))
+}
+
+impl SearchQuery {
+    fn search(&self, index_uid: &str, data: web::Data<Data>) -> Result<SearchResult, ResponseError> {
+        let index = data
+            .db
+            .open_index(index_uid)
+            .ok_or(Error::index_not_found(index_uid))?;
+
+        let reader = data.db.main_read_txn()?;
+        let schema = index
+            .main
+            .schema(&reader)?
+            .ok_or(Error::internal("Impossible to retrieve the schema"))?;
+
+        let mut search_builder = index.new_search(self.q.clone());
+
+        if let Some(offset) = self.offset {
+            search_builder.offset(offset);
+        }
+        if let Some(limit) = self.limit {
+            search_builder.limit(limit);
+        }
+
+        let available_attributes = schema.displayed_name();
+        let mut restricted_attributes: HashSet<&str>;
+        match &self.attributes_to_retrieve {
+            Some(attributes_to_retrieve) => {
+                let attributes_to_retrieve: HashSet<&str> = attributes_to_retrieve.split(',').collect();
+                if attributes_to_retrieve.contains("*") {
+                    restricted_attributes = available_attributes.clone();
+                } else {
+                    restricted_attributes = HashSet::new();
+                    for attr in attributes_to_retrieve {
+                        if available_attributes.contains(attr) {
+                            restricted_attributes.insert(attr);
+                            search_builder.add_retrievable_field(attr.to_string());
+                        } else {
+                            warn!("The attributes {:?} present in attributesToCrop parameter doesn't exist", attr);
+                        }
+                    }
+                }
+            },
+            None => {
                 restricted_attributes = available_attributes.clone();
-            } else {
-                restricted_attributes = HashSet::new();
-                for attr in attributes_to_retrieve {
-                    if available_attributes.contains(attr) {
-                        restricted_attributes.insert(attr);
-                        search_builder.add_retrievable_field(attr.to_string());
-                    } else {
-                        warn!("The attributes {:?} present in attributesToCrop parameter doesn't exist", attr);
-                    }
+            }
+        }
+
+        if let Some(ref facet_filters) = self.facet_filters {
+            let attrs = index.main.attributes_for_faceting(&reader)?.unwrap_or_default();
+            search_builder.add_facet_filters(FacetFilter::from_str(facet_filters, &schema, &attrs)?);
+        }
+
+        if let Some(facets) = &self.facets_distribution {
+            match index.main.attributes_for_faceting(&reader)? {
+                Some(ref attrs) => {
+                    let field_ids = prepare_facet_list(&facets, &schema, attrs)?;
+                    search_builder.add_facets(field_ids);
+                },
+                None => return Err(FacetCountError::NoFacetSet.into()),
+            }
+        }
+
+        if let Some(attributes_to_crop) = &self.attributes_to_crop {
+            let default_length = self.crop_length.unwrap_or(200);
+            let mut final_attributes: HashMap<String, usize> = HashMap::new();
+
+            for attribute in attributes_to_crop.split(',') {
+                let mut attribute = attribute.split(':');
+                let attr = attribute.next();
+                let length = attribute.next().and_then(|s| s.parse().ok()).unwrap_or(default_length);
+                match attr {
+                    Some("*") => {
+                        for attr in &restricted_attributes {
+                            final_attributes.insert(attr.to_string(), length);
+                        }
+                    },
+                    Some(attr) => {
+                        if available_attributes.contains(attr) {
+                            final_attributes.insert(attr.to_string(), length);
+                        } else {
+                            warn!("The attributes {:?} present in attributesToCrop parameter doesn't exist", attr);
+                        }
+                    },
+                    None => (),
                 }
             }
-        },
-        None => {
-            restricted_attributes = available_attributes.clone();
+            search_builder.attributes_to_crop(final_attributes);
         }
-    }
 
-    if let Some(attributes_to_crop) = query.attributes_to_crop {
-        let default_length = query.crop_length.unwrap_or(200);
-        let mut final_attributes: HashMap<String, usize> = HashMap::new();
-
-        for attribute in attributes_to_crop.split(',') {
-            let mut attribute = attribute.split(':');
-            let attr = attribute.next();
-            let length = attribute.next().and_then(|s| s.parse().ok()).unwrap_or(default_length);
-            match attr {
-                Some("*") => {
+        if let Some(attributes_to_highlight) = &self.attributes_to_highlight {
+            let mut final_attributes: HashSet<String> = HashSet::new();
+            for attribute in attributes_to_highlight.split(',') {
+                if attribute == "*" {
                     for attr in &restricted_attributes {
-                        final_attributes.insert(attr.to_string(), length);
+                        final_attributes.insert(attr.to_string());
                     }
-                },
-                Some(attr) => {
-                    if available_attributes.contains(attr) {
-                        final_attributes.insert(attr.to_string(), length);
-                    } else {
-                        warn!("The attributes {:?} present in attributesToCrop parameter doesn't exist", attr);
-                    }
-                },
-                None => (),
-            }
-        }
-
-        search_builder.attributes_to_crop(final_attributes);
-    }
-
-    if let Some(attributes_to_highlight) = query.attributes_to_highlight {
-        let mut final_attributes: HashSet<String> = HashSet::new();
-        for attribute in attributes_to_highlight.split(',') {
-            if attribute == "*" {
-                for attr in &restricted_attributes {
-                    final_attributes.insert(attr.to_string());
-                }
-            } else {
-                if available_attributes.contains(attribute) {
+                } else if available_attributes.contains(attribute) {
                     final_attributes.insert(attribute.to_string());
                 } else {
                     warn!("The attributes {:?} present in attributesToHighlight parameter doesn't exist", attribute);
                 }
             }
+
+            search_builder.attributes_to_highlight(final_attributes);
         }
 
-        search_builder.attributes_to_highlight(final_attributes);
-    }
-
-    if let Some(filters) = query.filters {
-        search_builder.filters(filters);
-    }
-
-    if let Some(timeout_ms) = query.timeout_ms {
-        search_builder.timeout(Duration::from_millis(timeout_ms));
-    }
-
-    if let Some(matches) = query.matches {
-        if matches {
-            search_builder.get_matches();
+        if let Some(filters) = &self.filters {
+            search_builder.filters(filters.to_string());
         }
+
+        if let Some(matches) = self.matches {
+            if matches {
+                search_builder.get_matches();
+            }
+        }
+        search_builder.search(&reader)
     }
-
-    let response = match search_builder.search(&reader) {
-        Ok(response) => response,
-        Err(Error::Internal(message)) => return Err(ResponseError::Internal(message)),
-        Err(others) => return Err(ResponseError::bad_request(others)),
-    };
-
-    Ok(tide::Response::new(200).body_json(&response).unwrap())
 }
 
-#[derive(Clone, Deserialize)]
-#[serde(rename_all = "camelCase", deny_unknown_fields)]
-struct SearchMultiBody {
-    indexes: HashSet<String>,
-    query: String,
-    offset: Option<usize>,
-    limit: Option<usize>,
-    attributes_to_retrieve: Option<HashSet<String>>,
-    searchable_attributes: Option<HashSet<String>>,
-    attributes_to_crop: Option<HashMap<String, usize>>,
-    attributes_to_highlight: Option<HashSet<String>>,
-    filters: Option<String>,
-    timeout_ms: Option<u64>,
-    matches: Option<bool>,
-}
-
-#[derive(Debug, Clone, Serialize)]
-#[serde(rename_all = "camelCase")]
-struct SearchMultiBodyResponse {
-    hits: HashMap<String, Vec<SearchHit>>,
-    offset: usize,
-    hits_per_page: usize,
-    processing_time_ms: usize,
-    query: String,
-}
-
-pub async fn search_multi_index(mut ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Public)?;
-    let body = ctx
-        .body_json::<SearchMultiBody>()
-        .await
-        .map_err(ResponseError::bad_request)?;
-
-    let mut index_list = body.clone().indexes;
-
-    for index in index_list.clone() {
-        if index == "*" {
-            index_list = ctx.state().db.indexes_uids().into_iter().collect();
-            break;
-        }
-    }
-
-    let mut offset = 0;
-    let mut count = 20;
-
-    if let Some(body_offset) = body.offset {
-        if let Some(limit) = body.limit {
-            offset = body_offset;
-            count = limit;
-        }
-    }
-
-    let offset = offset;
-    let count = count;
-    let db = &ctx.state().db;
-    let par_body = body.clone();
-    let responses_per_index: Vec<SResult<_>> = index_list
-        .into_par_iter()
-        .map(move |index_uid| {
-            let index: Index = db
-                .open_index(&index_uid)
-                .ok_or(ResponseError::index_not_found(&index_uid))?;
-
-            let mut search_builder = index.new_search(par_body.query.clone());
-
-            search_builder.offset(offset);
-            search_builder.limit(count);
-
-            if let Some(attributes_to_retrieve) = par_body.attributes_to_retrieve.clone() {
-                search_builder.attributes_to_retrieve(attributes_to_retrieve);
+/// Parses the incoming string into an array of attributes for which to return a count. It returns
+/// a Vec of attribute names ascociated with their id.
+///
+/// An error is returned if the array is malformed, or if it contains attributes that are
+/// unexisting, or not set as facets.
+fn prepare_facet_list(facets: &str, schema: &Schema, facet_attrs: &[FieldId]) -> Result<Vec<(FieldId, String)>, FacetCountError> {
+    let json_array = serde_json::from_str(facets)?;
+    match json_array {
+        Value::Array(vals) => {
+            let wildcard = Value::String("*".to_string());
+            if vals.iter().any(|f| f == &wildcard) {
+                let attrs = facet_attrs
+                    .iter()
+                    .filter_map(|&id| schema.name(id).map(|n| (id, n.to_string())))
+                    .collect();
+                return Ok(attrs);
             }
-            if let Some(attributes_to_crop) = par_body.attributes_to_crop.clone() {
-                search_builder.attributes_to_crop(attributes_to_crop);
-            }
-            if let Some(attributes_to_highlight) = par_body.attributes_to_highlight.clone() {
-                search_builder.attributes_to_highlight(attributes_to_highlight);
-            }
-            if let Some(filters) = par_body.filters.clone() {
-                search_builder.filters(filters);
-            }
-            if let Some(timeout_ms) = par_body.timeout_ms {
-                search_builder.timeout(Duration::from_millis(timeout_ms));
-            }
-            if let Some(matches) = par_body.matches {
-                if matches {
-                    search_builder.get_matches();
+            let mut field_ids = Vec::with_capacity(facet_attrs.len());
+            for facet in vals {
+                match facet {
+                    Value::String(facet) => {
+                        if let Some(id) = schema.id(&facet) {
+                            if !facet_attrs.contains(&id) {
+                                return Err(FacetCountError::AttributeNotSet(facet));
+                            }
+                            field_ids.push((id, facet));
+                        }
+                    }
+                    bad_val => return Err(FacetCountError::unexpected_token(bad_val, &["String"])),
                 }
             }
-
-            let reader = db.main_read_txn()?;
-            let response = search_builder.search(&reader)?;
-            Ok((index_uid, response))
-        })
-        .collect();
-
-    let mut hits_map = HashMap::new();
-
-    let mut max_query_time = 0;
-
-    for response in responses_per_index {
-        if let Ok((index_uid, response)) = response {
-            if response.processing_time_ms > max_query_time {
-                max_query_time = response.processing_time_ms;
-            }
-            hits_map.insert(index_uid, response.hits);
+            Ok(field_ids)
         }
+        bad_val => Err(FacetCountError::unexpected_token(bad_val, &["[String]"]))
     }
-
-    let response = SearchMultiBodyResponse {
-        hits: hits_map,
-        offset,
-        hits_per_page: count,
-        processing_time_ms: max_query_time,
-        query: body.query,
-    };
-
-    Ok(tide::Response::new(200).body_json(&response).unwrap())
 }

meilisearch-http/src/routes/setting.rs

@@ -1,34 +1,83 @@
+use actix_web::{web, HttpResponse};
+use actix_web_macros::{delete, get, post};
 use meilisearch_core::settings::{Settings, SettingsUpdate, UpdateState, DEFAULT_RANKING_RULES};
+use meilisearch_schema::Schema;
 use std::collections::{BTreeMap, BTreeSet, HashSet};
-use tide::{Request, Response};
 
-use crate::error::{ResponseError, SResult};
-use crate::helpers::tide::RequestExt;
-use crate::helpers::tide::ACL::*;
-use crate::routes::document::IndexUpdateResponse;
+use crate::error::{Error, ResponseError};
+use crate::helpers::Authentication;
+use crate::routes::{IndexParam, IndexUpdateResponse};
 use crate::Data;
 
-pub async fn get_all(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-    let db = &ctx.state().db;
-    let reader = db.main_read_txn()?;
+pub fn services(cfg: &mut web::ServiceConfig) {
+    cfg.service(update_all)
+        .service(get_all)
+        .service(delete_all)
+        .service(get_rules)
+        .service(update_rules)
+        .service(delete_rules)
+        .service(get_distinct)
+        .service(update_distinct)
+        .service(delete_distinct)
+        .service(get_searchable)
+        .service(update_searchable)
+        .service(delete_searchable)
+        .service(get_displayed)
+        .service(update_displayed)
+        .service(delete_displayed)
+        .service(get_attributes_for_faceting)
+        .service(delete_attributes_for_faceting)
+        .service(update_attributes_for_faceting);
+}
 
-    let stop_words_fst = index.main.stop_words_fst(&reader)?;
-    let stop_words = stop_words_fst.unwrap_or_default().stream().into_strs()?;
-    let stop_words: BTreeSet<String> = stop_words.into_iter().collect();
+#[post("/indexes/{index_uid}/settings", wrap = "Authentication::Private")]
+async fn update_all(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+    body: web::Json<Settings>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
-    let synonyms_fst = index.main.synonyms_fst(&reader)?.unwrap_or_default();
-    let synonyms_list = synonyms_fst.stream().into_strs()?;
+    let update_id = data.db.update_write::<_, _, ResponseError>(|writer| {
+        let settings = body
+            .into_inner()
+            .to_update()
+            .map_err(Error::bad_request)?;
+        let update_id = index.settings_update(writer, settings)?;
+        Ok(update_id)
+    })?;
+
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
+}
+
+#[get("/indexes/{index_uid}/settings", wrap = "Authentication::Private")]
+async fn get_all(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
+
+    let reader = data.db.main_read_txn()?;
+
+    let stop_words: BTreeSet<String> = index
+        .main
+        .stop_words(&reader)?
+        .into_iter()
+        .collect();
+
+    let synonyms_list = index.main.synonyms(&reader)?;
 
     let mut synonyms = BTreeMap::new();
     let index_synonyms = &index.synonyms;
     for synonym in synonyms_list {
-        let alternative_list = index_synonyms.synonyms(&reader, synonym.as_bytes())?;
-        if let Some(list) = alternative_list {
-            let list = list.stream().into_strs()?;
-            synonyms.insert(synonym, list);
-        }
+        let list = index_synonyms.synonyms(&reader, synonym.as_bytes())?;
+        synonyms.insert(synonym, list);
     }
 
     let ranking_rules = index
@@ -39,25 +88,27 @@ pub async fn get_all(ctx: Request<Data>) -> SResult<Response> {
         .map(|r| r.to_string())
         .collect();
 
-    let distinct_attribute = index.main.distinct_attribute(&reader)?;
 
     let schema = index.main.schema(&reader)?;
 
-    let searchable_attributes = schema.clone().map(|s| {
-        s.indexed_name()
-            .iter()
-            .map(|s| (*s).to_string())
-            .collect::<Vec<String>>()
-    });
+    let distinct_attribute = match (index.main.distinct_attribute(&reader)?, &schema) {
+        (Some(id), Some(schema)) => schema.name(id).map(str::to_string),
+        _ => None,
+    };
 
-    let displayed_attributes = schema.clone().map(|s| {
-        s.displayed_name()
-            .iter()
-            .map(|s| (*s).to_string())
-            .collect::<HashSet<String>>()
-    });
+    let attributes_for_faceting = match (&schema, &index.main.attributes_for_faceting(&reader)?) {
+        (Some(schema), Some(attrs)) => {
+            attrs
+                .iter()
+                .filter_map(|&id| schema.name(id))
+                .map(str::to_string)
+                .collect()
+        }
+        _ => vec![],
+    };
 
-    let accept_new_fields = schema.map(|s| s.accept_new_fields());
+    let searchable_attributes = schema.as_ref().map(get_indexed_attributes);
+    let displayed_attributes = schema.as_ref().map(get_displayed_attributes);
 
     let settings = Settings {
         ranking_rules: Some(Some(ranking_rules)),
@@ -66,33 +117,21 @@ pub async fn get_all(ctx: Request<Data>) -> SResult<Response> {
         displayed_attributes: Some(displayed_attributes),
         stop_words: Some(Some(stop_words)),
         synonyms: Some(Some(synonyms)),
-        accept_new_fields: Some(accept_new_fields),
+        attributes_for_faceting: Some(Some(attributes_for_faceting)),
     };
 
-    Ok(tide::Response::new(200).body_json(&settings).unwrap())
+    Ok(HttpResponse::Ok().json(settings))
 }
 
-pub async fn update_all(mut ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-    let settings: Settings =
-        ctx.body_json().await.map_err(ResponseError::bad_request)?;
-    let db = &ctx.state().db;
-
-    let mut writer = db.update_write_txn()?;
-    let settings = settings.into_update().map_err(ResponseError::bad_request)?;
-    let update_id = index.settings_update(&mut writer, settings)?;
-    writer.commit()?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
-}
-
-pub async fn delete_all(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-    let db = &ctx.state().db;
-    let mut writer = db.update_write_txn()?;
+#[delete("/indexes/{index_uid}/settings", wrap = "Authentication::Private")]
+async fn delete_all(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
     let settings = SettingsUpdate {
         ranking_rules: UpdateState::Clear,
@@ -102,22 +141,27 @@ pub async fn delete_all(ctx: Request<Data>) -> SResult<Response> {
         displayed_attributes: UpdateState::Clear,
         stop_words: UpdateState::Clear,
         synonyms: UpdateState::Clear,
-        accept_new_fields: UpdateState::Clear,
+        attributes_for_faceting: UpdateState::Clear,
     };
 
-    let update_id = index.settings_update(&mut writer, settings)?;
+    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
 
-    writer.commit()?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
 
-pub async fn get_rules(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-    let db = &ctx.state().db;
-    let reader = db.main_read_txn()?;
+#[get(
+    "/indexes/{index_uid}/settings/ranking-rules",
+    wrap = "Authentication::Private"
+)]
+async fn get_rules(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let reader = data.db.main_read_txn()?;
 
     let ranking_rules = index
         .main
@@ -127,248 +171,366 @@ pub async fn get_rules(ctx: Request<Data>) -> SResult<Response> {
         .map(|r| r.to_string())
         .collect::<Vec<String>>();
 
-    Ok(tide::Response::new(200).body_json(&ranking_rules).unwrap())
+    Ok(HttpResponse::Ok().json(ranking_rules))
 }
 
-pub async fn update_rules(mut ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-    let ranking_rules: Option<Vec<String>> =
-        ctx.body_json().await.map_err(ResponseError::bad_request)?;
-    let db = &ctx.state().db;
+#[post(
+    "/indexes/{index_uid}/settings/ranking-rules",
+    wrap = "Authentication::Private"
+)]
+async fn update_rules(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+    body: web::Json<Option<Vec<String>>>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
     let settings = Settings {
-        ranking_rules: Some(ranking_rules),
+        ranking_rules: Some(body.into_inner()),
         ..Settings::default()
     };
 
-    let mut writer = db.update_write_txn()?;
-    let settings = settings.into_update().map_err(ResponseError::bad_request)?;
-    let update_id = index.settings_update(&mut writer, settings)?;
-    writer.commit()?;
+    let settings = settings.to_update().map_err(Error::bad_request)?;
+    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
 
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
 
-pub async fn delete_rules(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-    let db = &ctx.state().db;
-    let mut writer = db.update_write_txn()?;
+#[delete(
+    "/indexes/{index_uid}/settings/ranking-rules",
+    wrap = "Authentication::Private"
+)]
+async fn delete_rules(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
     let settings = SettingsUpdate {
         ranking_rules: UpdateState::Clear,
         ..SettingsUpdate::default()
     };
 
-    let update_id = index.settings_update(&mut writer, settings)?;
+    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
 
-    writer.commit()?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
 
-pub async fn get_distinct(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-    let db = &ctx.state().db;
-    let reader = db.main_read_txn()?;
+#[get(
+    "/indexes/{index_uid}/settings/distinct-attribute",
+    wrap = "Authentication::Private"
+)]
+async fn get_distinct(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let reader = data.db.main_read_txn()?;
+    let distinct_attribute_id = index.main.distinct_attribute(&reader)?;
+    let schema = index.main.schema(&reader)?;
+    let distinct_attribute = match (schema, distinct_attribute_id) {
+        (Some(schema), Some(id)) => schema.name(id).map(str::to_string),
+        _ => None,
+    };
 
-    let distinct_attribute = index.main.distinct_attribute(&reader)?;
-
-    Ok(tide::Response::new(200)
-        .body_json(&distinct_attribute)
-        .unwrap())
+    Ok(HttpResponse::Ok().json(distinct_attribute))
 }
 
-pub async fn update_distinct(mut ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-    let distinct_attribute: Option<String> =
-        ctx.body_json().await.map_err(ResponseError::bad_request)?;
-    let db = &ctx.state().db;
+#[post(
+    "/indexes/{index_uid}/settings/distinct-attribute",
+    wrap = "Authentication::Private"
+)]
+async fn update_distinct(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+    body: web::Json<Option<String>>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
     let settings = Settings {
-        distinct_attribute: Some(distinct_attribute),
+        distinct_attribute: Some(body.into_inner()),
         ..Settings::default()
     };
 
-    let mut writer = db.update_write_txn()?;
-    let settings = settings.into_update().map_err(ResponseError::bad_request)?;
-    let update_id = index.settings_update(&mut writer, settings)?;
-    writer.commit()?;
+    let settings = settings.to_update().map_err(Error::bad_request)?;
+    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
 
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
 
-pub async fn delete_distinct(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-    let db = &ctx.state().db;
-    let mut writer = db.update_write_txn()?;
+#[delete(
+    "/indexes/{index_uid}/settings/distinct-attribute",
+    wrap = "Authentication::Private"
+)]
+async fn delete_distinct(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
     let settings = SettingsUpdate {
         distinct_attribute: UpdateState::Clear,
         ..SettingsUpdate::default()
     };
 
-    let update_id = index.settings_update(&mut writer, settings)?;
+    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
 
-    writer.commit()?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
 
-pub async fn get_searchable(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-    let db = &ctx.state().db;
-    let reader = db.main_read_txn()?;
-
+#[get(
+    "/indexes/{index_uid}/settings/searchable-attributes",
+    wrap = "Authentication::Private"
+)]
+async fn get_searchable(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let reader = data.db.main_read_txn()?;
     let schema = index.main.schema(&reader)?;
-
     let searchable_attributes: Option<Vec<String>> =
-        schema.map(|s| s.indexed_name().iter().map(|i| (*i).to_string()).collect());
+        schema.as_ref().map(get_indexed_attributes);
 
-    Ok(tide::Response::new(200)
-        .body_json(&searchable_attributes)
-        .unwrap())
+    Ok(HttpResponse::Ok().json(searchable_attributes))
 }
 
-pub async fn update_searchable(mut ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-    let searchable_attributes: Option<Vec<String>> =
-        ctx.body_json().await.map_err(ResponseError::bad_request)?;
-    let db = &ctx.state().db;
+#[post(
+    "/indexes/{index_uid}/settings/searchable-attributes",
+    wrap = "Authentication::Private"
+)]
+async fn update_searchable(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+    body: web::Json<Option<Vec<String>>>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
     let settings = Settings {
-        searchable_attributes: Some(searchable_attributes),
+        searchable_attributes: Some(body.into_inner()),
         ..Settings::default()
     };
 
-    let mut writer = db.update_write_txn()?;
-    let settings = settings.into_update().map_err(ResponseError::bad_request)?;
-    let update_id = index.settings_update(&mut writer, settings)?;
-    writer.commit()?;
+    let settings = settings.to_update().map_err(Error::bad_request)?;
 
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
+    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
 
-pub async fn delete_searchable(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-    let db = &ctx.state().db;
+#[delete(
+    "/indexes/{index_uid}/settings/searchable-attributes",
+    wrap = "Authentication::Private"
+)]
+async fn delete_searchable(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
     let settings = SettingsUpdate {
         searchable_attributes: UpdateState::Clear,
         ..SettingsUpdate::default()
     };
 
-    let mut writer = db.update_write_txn()?;
-    let update_id = index.settings_update(&mut writer, settings)?;
-    writer.commit()?;
+    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
 
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
 
-pub async fn displayed(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-    let db = &ctx.state().db;
-    let reader = db.main_read_txn()?;
+#[get(
+    "/indexes/{index_uid}/settings/displayed-attributes",
+    wrap = "Authentication::Private"
+)]
+async fn get_displayed(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let reader = data.db.main_read_txn()?;
 
     let schema = index.main.schema(&reader)?;
 
-    let displayed_attributes: Option<HashSet<String>> = schema.map(|s| {
-        s.displayed_name()
-            .iter()
-            .map(|i| (*i).to_string())
-            .collect()
-    });
+    let displayed_attributes = schema.as_ref().map(get_displayed_attributes);
 
-    Ok(tide::Response::new(200)
-        .body_json(&displayed_attributes)
-        .unwrap())
+    Ok(HttpResponse::Ok().json(displayed_attributes))
 }
 
-pub async fn update_displayed(mut ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-    let displayed_attributes: Option<HashSet<String>> =
-        ctx.body_json().await.map_err(ResponseError::bad_request)?;
-    let db = &ctx.state().db;
+#[post(
+    "/indexes/{index_uid}/settings/displayed-attributes",
+    wrap = "Authentication::Private"
+)]
+async fn update_displayed(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+    body: web::Json<Option<HashSet<String>>>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
     let settings = Settings {
-        displayed_attributes: Some(displayed_attributes),
+        displayed_attributes: Some(body.into_inner()),
         ..Settings::default()
     };
 
-    let mut writer = db.update_write_txn()?;
-    let settings = settings.into_update().map_err(ResponseError::bad_request)?;
-    let update_id = index.settings_update(&mut writer, settings)?;
-    writer.commit()?;
+    let settings = settings.to_update().map_err(Error::bad_request)?;
+    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
 
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
 
-pub async fn delete_displayed(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-    let db = &ctx.state().db;
+#[delete(
+    "/indexes/{index_uid}/settings/displayed-attributes",
+    wrap = "Authentication::Private"
+)]
+async fn delete_displayed(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
     let settings = SettingsUpdate {
         displayed_attributes: UpdateState::Clear,
         ..SettingsUpdate::default()
     };
 
-    let mut writer = db.update_write_txn()?;
-    let update_id = index.settings_update(&mut writer, settings)?;
-    writer.commit()?;
+    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
 
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
 
-pub async fn get_accept_new_fields(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-    let db = &ctx.state().db;
-    let reader = db.main_read_txn()?;
+#[get(
+    "/indexes/{index_uid}/settings/attributes-for-faceting",
+    wrap = "Authentication::Private"
+)]
+async fn get_attributes_for_faceting(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
-    let schema = index.main.schema(&reader)?;
+    let attributes_for_faceting = data
+        .db
+        .main_read::<_, _, ResponseError>(|reader| {
+        let schema = index.main.schema(reader)?;
+        let attrs = index.main.attributes_for_faceting(reader)?;
+        let attr_names = match (&schema, &attrs) {
+            (Some(schema), Some(attrs)) => {
+                attrs
+                    .iter()
+                    .filter_map(|&id| schema.name(id))
+                    .map(str::to_string)
+                    .collect()
+            }
+            _ => vec![]
+        };
+        Ok(attr_names)
+    })?;
 
-    let accept_new_fields = schema.map(|s| s.accept_new_fields());
-
-    Ok(tide::Response::new(200)
-        .body_json(&accept_new_fields)
-        .unwrap())
+    Ok(HttpResponse::Ok().json(attributes_for_faceting))
 }
 
-pub async fn update_accept_new_fields(mut ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-    let accept_new_fields: Option<bool> =
-        ctx.body_json().await.map_err(ResponseError::bad_request)?;
-    let db = &ctx.state().db;
+#[post(
+    "/indexes/{index_uid}/settings/attributes-for-faceting",
+    wrap = "Authentication::Private"
+)]
+async fn update_attributes_for_faceting(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+    body: web::Json<Option<Vec<String>>>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
     let settings = Settings {
-        accept_new_fields: Some(accept_new_fields),
+        attributes_for_faceting: Some(body.into_inner()),
         ..Settings::default()
     };
 
-    let mut writer = db.update_write_txn()?;
-    let settings = settings.into_update().map_err(ResponseError::bad_request)?;
-    let update_id = index.settings_update(&mut writer, settings)?;
-    writer.commit()?;
+    let settings = settings.to_update().map_err(Error::bad_request)?;
+    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
 
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
+}
+
+#[delete(
+    "/indexes/{index_uid}/settings/attributes-for-faceting",
+    wrap = "Authentication::Private"
+)]
+async fn delete_attributes_for_faceting(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
+
+    let settings = SettingsUpdate {
+        attributes_for_faceting: UpdateState::Clear,
+        ..SettingsUpdate::default()
+    };
+
+    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
+}
+
+fn get_indexed_attributes(schema: &Schema) -> Vec<String> {
+    if schema.is_indexed_all() {
+        ["*"].iter().map(|s| s.to_string()).collect()
+    } else {
+        schema.indexed_name()
+            .iter()
+            .map(|s| s.to_string())
+            .collect()
+    }
+}
+
+fn get_displayed_attributes(schema: &Schema) -> HashSet<String> {
+    if schema.is_displayed_all() {
+        ["*"].iter().map(|s| s.to_string()).collect()
+    } else {
+        schema.displayed_name()
+            .iter()
+            .map(|s| s.to_string())
+            .collect()
+    }
 }

meilisearch-http/src/routes/stats.rs

@@ -1,46 +1,61 @@
 use std::collections::HashMap;
 
+use actix_web::web;
+use actix_web::HttpResponse;
+use actix_web_macros::get;
 use chrono::{DateTime, Utc};
 use log::error;
-use pretty_bytes::converter::convert;
 use serde::Serialize;
-use sysinfo::{NetworkExt, Pid, ProcessExt, ProcessorExt, System, SystemExt};
-use tide::{Request, Response};
 use walkdir::WalkDir;
 
-use crate::error::{IntoInternalError, SResult};
-use crate::helpers::tide::RequestExt;
-use crate::helpers::tide::ACL::*;
+use crate::error::{Error, ResponseError};
+use crate::helpers::Authentication;
+use crate::routes::IndexParam;
 use crate::Data;
 
+pub fn services(cfg: &mut web::ServiceConfig) {
+    cfg.service(index_stats)
+        .service(get_stats)
+        .service(get_version);
+}
+
 #[derive(Serialize)]
 #[serde(rename_all = "camelCase")]
 struct IndexStatsResponse {
     number_of_documents: u64,
     is_indexing: bool,
-    fields_frequency: HashMap<String, usize>,
+    fields_distribution: HashMap<String, usize>,
 }
 
-pub async fn index_stats(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Admin)?;
-    let index_uid = ctx.url_param("index")?;
-    let index = ctx.index()?;
-    let db = &ctx.state().db;
-    let reader = db.main_read_txn()?;
-    let update_reader = db.update_read_txn()?;
-    let number_of_documents = index.main.number_of_documents(&reader)?;
-    let fields_frequency = index.main.fields_frequency(&reader)?.unwrap_or_default();
-    let is_indexing = ctx
-        .state()
-        .is_indexing(&update_reader, &index_uid)?
-        .into_internal_error()?;
+#[get("/indexes/{index_uid}/stats", wrap = "Authentication::Private")]
+async fn index_stats(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
-    let response = IndexStatsResponse {
+    let reader = data.db.main_read_txn()?;
+
+    let number_of_documents = index.main.number_of_documents(&reader)?;
+
+    let fields_distribution = index.main.fields_distribution(&reader)?.unwrap_or_default();
+
+    let update_reader = data.db.update_read_txn()?;
+
+    let is_indexing =
+        data.db.is_indexing(&update_reader, &path.index_uid)?
+            .ok_or(Error::internal(
+                "Impossible to know if the database is indexing",
+            ))?;
+
+    Ok(HttpResponse::Ok().json(IndexStatsResponse {
         number_of_documents,
         is_indexing,
-        fields_frequency,
-    };
-    Ok(tide::Response::new(200).body_json(&response).unwrap())
+        fields_distribution,
+    }))
 }
 
 #[derive(Serialize)]
@@ -51,34 +66,30 @@ struct StatsResult {
     indexes: HashMap<String, IndexStatsResponse>,
 }
 
-pub async fn get_stats(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Admin)?;
-
+#[get("/stats", wrap = "Authentication::Private")]
+async fn get_stats(data: web::Data<Data>) -> Result<HttpResponse, ResponseError> {
     let mut index_list = HashMap::new();
 
-    let db = &ctx.state().db;
-    let reader = db.main_read_txn()?;
-    let update_reader = db.update_read_txn()?;
+    let reader = data.db.main_read_txn()?;
+    let update_reader = data.db.update_read_txn()?;
 
-    let indexes_set = ctx.state().db.indexes_uids();
+    let indexes_set = data.db.indexes_uids();
     for index_uid in indexes_set {
-        let index = ctx.state().db.open_index(&index_uid);
-
+        let index = data.db.open_index(&index_uid);
         match index {
             Some(index) => {
                 let number_of_documents = index.main.number_of_documents(&reader)?;
 
-                let fields_frequency = index.main.fields_frequency(&reader)?.unwrap_or_default();
+                let fields_distribution = index.main.fields_distribution(&reader)?.unwrap_or_default();
 
-                let is_indexing = ctx
-                    .state()
-                    .is_indexing(&update_reader, &index_uid)?
-                    .into_internal_error()?;
+                let is_indexing = data.db.is_indexing(&update_reader, &index_uid)?.ok_or(
+                    Error::internal("Impossible to know if the database is indexing"),
+                )?;
 
                 let response = IndexStatsResponse {
                     number_of_documents,
                     is_indexing,
-                    fields_frequency,
+                    fields_distribution,
                 };
                 index_list.insert(index_uid, response);
             }
@@ -89,22 +100,20 @@ pub async fn get_stats(ctx: Request<Data>) -> SResult<Response> {
         }
     }
 
-    let database_size = WalkDir::new(ctx.state().db_path.clone())
+    let database_size = WalkDir::new(&data.db_path)
         .into_iter()
         .filter_map(|entry| entry.ok())
         .filter_map(|entry| entry.metadata().ok())
         .filter(|metadata| metadata.is_file())
         .fold(0, |acc, m| acc + m.len());
 
-    let last_update = ctx.state().last_update(&reader)?;
+    let last_update = data.db.last_update(&reader)?;
 
-    let response = StatsResult {
+    Ok(HttpResponse::Ok().json(StatsResult {
         database_size,
         last_update,
         indexes: index_list,
-    };
-
-    Ok(tide::Response::new(200).body_json(&response).unwrap())
+    }))
 }
 
 #[derive(Serialize)]
@@ -115,203 +124,11 @@ struct VersionResponse {
     pkg_version: String,
 }
 
-pub async fn get_version(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Admin)?;
-    let response = VersionResponse {
+#[get("/version", wrap = "Authentication::Private")]
+async fn get_version() -> HttpResponse {
+    HttpResponse::Ok().json(VersionResponse {
         commit_sha: env!("VERGEN_SHA").to_string(),
         build_date: env!("VERGEN_BUILD_TIMESTAMP").to_string(),
         pkg_version: env!("CARGO_PKG_VERSION").to_string(),
-    };
-
-    Ok(tide::Response::new(200).body_json(&response).unwrap())
-}
-
-#[derive(Serialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct SysGlobal {
-    total_memory: u64,
-    used_memory: u64,
-    total_swap: u64,
-    used_swap: u64,
-    input_data: u64,
-    output_data: u64,
-}
-
-impl SysGlobal {
-    fn new() -> SysGlobal {
-        SysGlobal {
-            total_memory: 0,
-            used_memory: 0,
-            total_swap: 0,
-            used_swap: 0,
-            input_data: 0,
-            output_data: 0,
-        }
-    }
-}
-
-#[derive(Serialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct SysProcess {
-    memory: u64,
-    cpu: f32,
-}
-
-impl SysProcess {
-    fn new() -> SysProcess {
-        SysProcess {
-            memory: 0,
-            cpu: 0.0,
-        }
-    }
-}
-
-#[derive(Serialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct SysInfo {
-    memory_usage: f64,
-    processor_usage: Vec<f32>,
-    global: SysGlobal,
-    process: SysProcess,
-}
-
-impl SysInfo {
-    fn new() -> SysInfo {
-        SysInfo {
-            memory_usage: 0.0,
-            processor_usage: Vec::new(),
-            global: SysGlobal::new(),
-            process: SysProcess::new(),
-        }
-    }
-}
-
-pub(crate) fn report(pid: Pid) -> SysInfo {
-    let mut sys = System::new();
-    let mut info = SysInfo::new();
-
-    info.memory_usage = sys.get_used_memory() as f64 / sys.get_total_memory() as f64 * 100.0;
-
-    for processor in sys.get_processors() {
-        info.processor_usage.push(processor.get_cpu_usage() * 100.0);
-    }
-
-    info.global.total_memory = sys.get_total_memory();
-    info.global.used_memory = sys.get_used_memory();
-    info.global.total_swap = sys.get_total_swap();
-    info.global.used_swap = sys.get_used_swap();
-    info.global.input_data = sys.get_networks().into_iter().map(|(_, n)| n.get_received()).sum::<u64>();
-    info.global.output_data = sys.get_networks().into_iter().map(|(_, n)| n.get_transmitted()).sum::<u64>();
-
-    if let Some(process) = sys.get_process(pid) {
-        info.process.memory = process.memory();
-        info.process.cpu = process.cpu_usage() * 100.0;
-    }
-
-    sys.refresh_all();
-
-    info
-}
-
-pub async fn get_sys_info(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Admin)?;
-    let response = report(ctx.state().server_pid);
-    Ok(tide::Response::new(200).body_json(&response).unwrap())
-}
-
-#[derive(Serialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct SysGlobalPretty {
-    total_memory: String,
-    used_memory: String,
-    total_swap: String,
-    used_swap: String,
-    input_data: String,
-    output_data: String,
-}
-
-impl SysGlobalPretty {
-    fn new() -> SysGlobalPretty {
-        SysGlobalPretty {
-            total_memory: "None".to_owned(),
-            used_memory: "None".to_owned(),
-            total_swap: "None".to_owned(),
-            used_swap: "None".to_owned(),
-            input_data: "None".to_owned(),
-            output_data: "None".to_owned(),
-        }
-    }
-}
-
-#[derive(Serialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct SysProcessPretty {
-    memory: String,
-    cpu: String,
-}
-
-impl SysProcessPretty {
-    fn new() -> SysProcessPretty {
-        SysProcessPretty {
-            memory: "None".to_owned(),
-            cpu: "None".to_owned(),
-        }
-    }
-}
-
-#[derive(Serialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct SysInfoPretty {
-    memory_usage: String,
-    processor_usage: Vec<String>,
-    global: SysGlobalPretty,
-    process: SysProcessPretty,
-}
-
-impl SysInfoPretty {
-    fn new() -> SysInfoPretty {
-        SysInfoPretty {
-            memory_usage: "None".to_owned(),
-            processor_usage: Vec::new(),
-            global: SysGlobalPretty::new(),
-            process: SysProcessPretty::new(),
-        }
-    }
-}
-
-pub(crate) fn report_pretty(pid: Pid) -> SysInfoPretty {
-    let mut sys = System::new();
-    let mut info = SysInfoPretty::new();
-
-    info.memory_usage = format!(
-        "{:.1} %",
-        sys.get_used_memory() as f64 / sys.get_total_memory() as f64 * 100.0
-    );
-
-    for processor in sys.get_processors() {
-        info.processor_usage
-            .push(format!("{:.1} %", processor.get_cpu_usage() * 100.0));
-    }
-
-    info.global.total_memory = convert(sys.get_total_memory() as f64 * 1024.0);
-    info.global.used_memory = convert(sys.get_used_memory() as f64 * 1024.0);
-    info.global.total_swap = convert(sys.get_total_swap() as f64 * 1024.0);
-    info.global.used_swap = convert(sys.get_used_swap() as f64 * 1024.0);
-    info.global.input_data = convert(sys.get_networks().into_iter().map(|(_, n)| n.get_received()).sum::<u64>() as f64);
-    info.global.output_data = convert(sys.get_networks().into_iter().map(|(_, n)| n.get_transmitted()).sum::<u64>() as f64);
-
-    if let Some(process) = sys.get_process(pid) {
-        info.process.memory = convert(process.memory() as f64 * 1024.0);
-        info.process.cpu = format!("{:.1} %", process.cpu_usage() * 100.0);
-    }
-
-    sys.refresh_all();
-
-    info
-}
-
-pub async fn get_sys_info_pretty(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Admin)?;
-    let response = report_pretty(ctx.state().server_pid);
-    Ok(tide::Response::new(200).body_json(&response).unwrap())
+    })
 }

meilisearch-http/src/routes/stop_words.rs

@@ -1,63 +1,78 @@
+use actix_web::{web, HttpResponse};
+use actix_web_macros::{delete, get, post};
+use meilisearch_core::settings::{SettingsUpdate, UpdateState};
 use std::collections::BTreeSet;
 
-use meilisearch_core::settings::{SettingsUpdate, UpdateState};
-use tide::{Request, Response};
-
-use crate::error::{ResponseError, SResult};
-use crate::helpers::tide::RequestExt;
-use crate::helpers::tide::ACL::*;
-use crate::routes::document::IndexUpdateResponse;
+use crate::error::{Error, ResponseError};
+use crate::helpers::Authentication;
+use crate::routes::{IndexParam, IndexUpdateResponse};
 use crate::Data;
 
-pub async fn get(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-    let db = &ctx.state().db;
-    let reader = db.main_read_txn()?;
-    let stop_words_fst = index.main.stop_words_fst(&reader)?;
-    let stop_words = stop_words_fst.unwrap_or_default().stream().into_strs()?;
-
-    Ok(tide::Response::new(200).body_json(&stop_words).unwrap())
+pub fn services(cfg: &mut web::ServiceConfig) {
+    cfg.service(get).service(update).service(delete);
 }
 
-pub async fn update(mut ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
+#[get(
+    "/indexes/{index_uid}/settings/stop-words",
+    wrap = "Authentication::Private"
+)]
+async fn get(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let reader = data.db.main_read_txn()?;
+    let stop_words = index.main.stop_words(&reader)?;
 
-    let data: BTreeSet<String> = ctx.body_json().await.map_err(ResponseError::bad_request)?;
+    Ok(HttpResponse::Ok().json(stop_words))
+}
 
-    let db = &ctx.state().db;
-    let mut writer = db.update_write_txn()?;
+#[post(
+    "/indexes/{index_uid}/settings/stop-words",
+    wrap = "Authentication::Private"
+)]
+async fn update(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+    body: web::Json<BTreeSet<String>>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
     let settings = SettingsUpdate {
-        stop_words: UpdateState::Update(data),
+        stop_words: UpdateState::Update(body.into_inner()),
         ..SettingsUpdate::default()
     };
 
-    let update_id = index.settings_update(&mut writer, settings)?;
+    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
 
-    writer.commit()?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
 
-pub async fn delete(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-
-    let db = &ctx.state().db;
-    let mut writer = db.update_write_txn()?;
+#[delete(
+    "/indexes/{index_uid}/settings/stop-words",
+    wrap = "Authentication::Private"
+)]
+async fn delete(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
     let settings = SettingsUpdate {
         stop_words: UpdateState::Clear,
         ..SettingsUpdate::default()
     };
 
-    let update_id = index.settings_update(&mut writer, settings)?;
+    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
 
-    writer.commit()?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }

meilisearch-http/src/routes/synonym.rs

@@ -1,82 +1,89 @@
 use std::collections::BTreeMap;
 
+use actix_web::{web, HttpResponse};
+use actix_web_macros::{delete, get, post};
 use indexmap::IndexMap;
 use meilisearch_core::settings::{SettingsUpdate, UpdateState};
-use tide::{Request, Response};
 
-use crate::error::{ResponseError, SResult};
-use crate::helpers::tide::RequestExt;
-use crate::helpers::tide::ACL::*;
-use crate::routes::document::IndexUpdateResponse;
+use crate::error::{Error, ResponseError};
+use crate::helpers::Authentication;
+use crate::routes::{IndexParam, IndexUpdateResponse};
 use crate::Data;
 
-pub async fn get(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-    let index = ctx.index()?;
-
-    let db = &ctx.state().db;
-    let reader = db.main_read_txn()?;
-
-    let synonyms_fst = index.main.synonyms_fst(&reader)?.unwrap_or_default();
-    let synonyms_list = synonyms_fst.stream().into_strs()?;
-
-    let mut synonyms = IndexMap::new();
-
-    let index_synonyms = &index.synonyms;
-
-    for synonym in synonyms_list {
-        let alternative_list = index_synonyms.synonyms(&reader, synonym.as_bytes())?;
-
-        if let Some(list) = alternative_list {
-            let list = list.stream().into_strs()?;
-            synonyms.insert(synonym, list);
-        }
-    }
-
-    Ok(tide::Response::new(200).body_json(&synonyms).unwrap())
+pub fn services(cfg: &mut web::ServiceConfig) {
+    cfg.service(get).service(update).service(delete);
 }
 
-pub async fn update(mut ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
+#[get(
+    "/indexes/{index_uid}/settings/synonyms",
+    wrap = "Authentication::Private"
+)]
+async fn get(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
-    let data: BTreeMap<String, Vec<String>> =
-        ctx.body_json().await.map_err(ResponseError::bad_request)?;
+    let reader = data.db.main_read_txn()?;
 
-    let index = ctx.index()?;
+    let synonyms_list = index.main.synonyms(&reader)?;
 
-    let db = &ctx.state().db;
-    let mut writer = db.update_write_txn()?;
+    let mut synonyms = IndexMap::new();
+    let index_synonyms = &index.synonyms;
+    for synonym in synonyms_list {
+        let list = index_synonyms.synonyms(&reader, synonym.as_bytes())?;
+        synonyms.insert(synonym, list);
+    }
+
+    Ok(HttpResponse::Ok().json(synonyms))
+}
+
+#[post(
+    "/indexes/{index_uid}/settings/synonyms",
+    wrap = "Authentication::Private"
+)]
+async fn update(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+    body: web::Json<BTreeMap<String, Vec<String>>>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
     let settings = SettingsUpdate {
-        synonyms: UpdateState::Update(data),
+        synonyms: UpdateState::Update(body.into_inner()),
         ..SettingsUpdate::default()
     };
 
-    let update_id = index.settings_update(&mut writer, settings)?;
+    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
 
-    writer.commit()?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
 
-pub async fn delete(ctx: Request<Data>) -> SResult<Response> {
-    ctx.is_allowed(Private)?;
-
-    let index = ctx.index()?;
-
-    let db = &ctx.state().db;
-    let mut writer = db.update_write_txn()?;
+#[delete(
+    "/indexes/{index_uid}/settings/synonyms",
+    wrap = "Authentication::Private"
+)]
+async fn delete(
+    data: web::Data<Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = data
+        .db
+        .open_index(&path.index_uid)
+        .ok_or(Error::index_not_found(&path.index_uid))?;
 
     let settings = SettingsUpdate {
         synonyms: UpdateState::Clear,
         ..SettingsUpdate::default()
     };
 
-    let update_id = index.settings_update(&mut writer, settings)?;
+    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
 
-    writer.commit()?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::Response::new(202).body_json(&response_body)?)
+    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }

meilisearch-http/tests/common.rs

@@ -1,403 +1,73 @@
 #![allow(dead_code)]
 
-use http::StatusCode;
-use serde_json::Value;
+use actix_web::{http::StatusCode, test};
+use serde_json::{json, Value};
 use std::time::Duration;
-
-use async_std::io::prelude::*;
-use async_std::task::{block_on, sleep};
-use http_service::Body;
-use http_service_mock::{make_server, TestBackend};
-use meilisearch_http::data::Data;
-use meilisearch_http::option::Opt;
-use meilisearch_http::routes;
-use serde_json::json;
 use tempdir::TempDir;
-use tide::server::Service;
+use tokio::time::delay_for;
+
+use meilisearch_core::DatabaseOptions;
+use meilisearch_http::data::Data;
+use meilisearch_http::helpers::NormalizePath;
+use meilisearch_http::option::Opt;
+
+/// Performs a search test on both post and get routes
+#[macro_export]
+macro_rules! test_post_get_search {
+    ($server:expr, $query:expr, |$response:ident, $status_code:ident | $block:expr) => {
+        let post_query: meilisearch_http::routes::search::SearchQueryPost = serde_json::from_str(&$query.clone().to_string()).unwrap();
+        let get_query: meilisearch_http::routes::search::SearchQuery = post_query.into();
+        let get_query = ::serde_url_params::to_string(&get_query).unwrap();
+        let ($response, $status_code) = $server.search_get(&get_query).await;
+        let _ =::std::panic::catch_unwind(|| $block)
+            .map_err(|e| panic!("panic in get route: {:?}", e.downcast_ref::<&str>().unwrap()));
+        let ($response, $status_code) = $server.search_post($query).await;
+        let _ = ::std::panic::catch_unwind(|| $block)
+            .map_err(|e| panic!("panic in post route: {:?}", e.downcast_ref::<&str>().unwrap()));
+    };
+}
 
 pub struct Server {
     uid: String,
-    mock: TestBackend<Service<Data>>,
+    data: Data,
 }
 
 impl Server {
     pub fn with_uid(uid: &str) -> Server {
         let tmp_dir = TempDir::new("meilisearch").unwrap();
 
+        let default_db_options = DatabaseOptions::default();
+
         let opt = Opt {
             db_path: tmp_dir.path().to_str().unwrap().to_string(),
             http_addr: "127.0.0.1:7700".to_owned(),
             master_key: None,
             env: "development".to_owned(),
             no_analytics: true,
+            main_map_size: default_db_options.main_map_size,
+            update_map_size: default_db_options.update_map_size,
+            http_payload_size_limit: 10000000,
+            ..Opt::default()
         };
 
-        let data = Data::new(opt.clone());
-        let mut app = tide::with_state(data);
-        routes::load_routes(&mut app);
-        let http_server = app.into_http_service();
-        let mock = make_server(http_server).unwrap();
+        let data = Data::new(opt.clone()).unwrap();
 
         Server {
             uid: uid.to_string(),
-            mock,
+            data,
         }
     }
 
-    pub fn wait_update_id(&mut self, update_id: u64) {
-        loop {
-            let (response, status_code) = self.get_update_status(update_id);
-            assert_eq!(status_code, 200);
+    pub async fn test_server() -> Self {
 
-            if response["status"] == "processed" || response["status"] == "error" {
-                eprintln!("{:#?}", response);
-                return;
-            }
+        let mut server = Self::with_uid("test");
 
-            block_on(sleep(Duration::from_secs(1)));
-        }
-    }
-
-    // Global Http request GET/POST/DELETE async or sync
-
-    pub fn get_request(&mut self, url: &str) -> (Value, StatusCode) {
-        eprintln!("get_request: {}", url);
-        let req = http::Request::get(url).body(Body::empty()).unwrap();
-        let res = self.mock.simulate(req).unwrap();
-        let status_code = res.status().clone();
-
-        let mut buf = Vec::new();
-        block_on(res.into_body().read_to_end(&mut buf)).unwrap();
-        let response = serde_json::from_slice(&buf).unwrap_or_default();
-        (response, status_code)
-    }
-
-    pub fn post_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) {
-        eprintln!("post_request: {}", url);
-        let body_bytes = body.to_string().into_bytes();
-
-        let req = http::Request::post(url)
-            .body(Body::from(body_bytes))
-            .unwrap();
-        let res = self.mock.simulate(req).unwrap();
-        let status_code = res.status().clone();
-
-        let mut buf = Vec::new();
-        block_on(res.into_body().read_to_end(&mut buf)).unwrap();
-        let response = serde_json::from_slice(&buf).unwrap_or_default();
-        (response, status_code)
-    }
-
-    pub fn post_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) {
-        eprintln!("post_request_async: {}", url);
-        let (response, status_code) = self.post_request(url, body);
-        assert_eq!(status_code, 202);
-        assert!(response["updateId"].as_u64().is_some());
-        self.wait_update_id(response["updateId"].as_u64().unwrap());
-        (response, status_code)
-    }
-
-    pub fn put_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) {
-        eprintln!("put_request: {}", url);
-        let body_bytes = body.to_string().into_bytes();
-
-        let req = http::Request::put(url)
-            .body(Body::from(body_bytes))
-            .unwrap();
-        let res = self.mock.simulate(req).unwrap();
-        let status_code = res.status().clone();
-
-        let mut buf = Vec::new();
-        block_on(res.into_body().read_to_end(&mut buf)).unwrap();
-        let response = serde_json::from_slice(&buf).unwrap_or_default();
-        (response, status_code)
-    }
-
-    pub fn put_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) {
-        eprintln!("put_request_async: {}", url);
-        let (response, status_code) = self.put_request(url, body);
-        assert!(response["updateId"].as_u64().is_some());
-        assert_eq!(status_code, 202);
-        self.wait_update_id(response["updateId"].as_u64().unwrap());
-        (response, status_code)
-    }
-
-    pub fn delete_request(&mut self, url: &str) -> (Value, StatusCode) {
-        eprintln!("delete_request: {}", url);
-        let req = http::Request::delete(url).body(Body::empty()).unwrap();
-        let res = self.mock.simulate(req).unwrap();
-        let status_code = res.status().clone();
-
-        let mut buf = Vec::new();
-        block_on(res.into_body().read_to_end(&mut buf)).unwrap();
-        let response = serde_json::from_slice(&buf).unwrap_or_default();
-        (response, status_code)
-    }
-
-    pub fn delete_request_async(&mut self, url: &str) -> (Value, StatusCode) {
-        eprintln!("delete_request_async: {}", url);
-        let (response, status_code) = self.delete_request(url);
-        assert!(response["updateId"].as_u64().is_some());
-        assert_eq!(status_code, 202);
-        self.wait_update_id(response["updateId"].as_u64().unwrap());
-        (response, status_code)
-    }
-
-    // All Routes
-
-    pub fn list_indexes(&mut self) -> (Value, StatusCode) {
-        self.get_request("/indexes")
-    }
-
-    pub fn create_index(&mut self, body: Value) -> (Value, StatusCode) {
-        self.post_request("/indexes", body)
-    }
-
-    pub fn search_multi_index(&mut self, query: &str) -> (Value, StatusCode) {
-        let url = format!("/indexes/search?{}", query);
-        self.get_request(&url)
-    }
-
-    pub fn get_index(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}", self.uid);
-        self.get_request(&url)
-    }
-
-    pub fn update_index(&mut self, body: Value) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}", self.uid);
-        self.put_request(&url, body)
-    }
-
-    pub fn delete_index(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}", self.uid);
-        self.delete_request(&url)
-    }
-
-    pub fn search(&mut self, query: &str) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/search?{}", self.uid, query);
-        self.get_request(&url)
-    }
-
-    pub fn get_all_updates_status(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/updates", self.uid);
-        self.get_request(&url)
-    }
-
-    pub fn get_update_status(&mut self, update_id: u64) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/updates/{}", self.uid, update_id);
-        self.get_request(&url)
-    }
-
-    pub fn get_all_documents(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/documents", self.uid);
-        self.get_request(&url)
-    }
-
-    pub fn add_or_replace_multiple_documents(&mut self, body: Value) {
-        let url = format!("/indexes/{}/documents", self.uid);
-        self.post_request_async(&url, body);
-    }
-
-    pub fn add_or_replace_multiple_documents_sync(&mut self, body: Value) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/documents", self.uid);
-        self.post_request(&url, body)
-    }
-
-    pub fn add_or_update_multiple_documents(&mut self, body: Value) {
-        let url = format!("/indexes/{}/documents", self.uid);
-        self.put_request_async(&url, body);
-    }
-
-    pub fn clear_all_documents(&mut self) {
-        let url = format!("/indexes/{}/documents", self.uid);
-        self.delete_request_async(&url);
-    }
-
-    pub fn get_document(&mut self, document_id: impl ToString) -> (Value, StatusCode) {
-        let url = format!(
-            "/indexes/{}/documents/{}",
-            self.uid,
-            document_id.to_string()
-        );
-        self.get_request(&url)
-    }
-
-    pub fn delete_document(&mut self, document_id: impl ToString) -> (Value, StatusCode) {
-        let url = format!(
-            "/indexes/{}/documents/{}",
-            self.uid,
-            document_id.to_string()
-        );
-        self.delete_request_async(&url)
-    }
-
-    pub fn delete_multiple_documents(&mut self, body: Value) {
-        let url = format!("/indexes/{}/documents/delete-batch", self.uid);
-        self.post_request_async(&url, body);
-    }
-
-    pub fn get_all_settings(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings", self.uid);
-        self.get_request(&url)
-    }
-
-    pub fn update_all_settings(&mut self, body: Value) {
-        let url = format!("/indexes/{}/settings", self.uid);
-        self.post_request_async(&url, body);
-    }
-
-    pub fn delete_all_settings(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings", self.uid);
-        self.delete_request_async(&url)
-    }
-
-    pub fn get_ranking_rules(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings/ranking-rules", self.uid);
-        self.get_request(&url)
-    }
-
-    pub fn update_ranking_rules(&mut self, body: Value) {
-        let url = format!("/indexes/{}/settings/ranking-rules", self.uid);
-        self.post_request_async(&url, body);
-    }
-
-    pub fn update_ranking_rules_sync(&mut self, body: Value) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings/ranking-rules", self.uid);
-        self.post_request(&url, body)
-    }
-
-    pub fn delete_ranking_rules(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings/ranking-rules", self.uid);
-        self.delete_request_async(&url)
-    }
-
-    pub fn get_distinct_attribute(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings/distinct-attribute", self.uid);
-        self.get_request(&url)
-    }
-
-    pub fn update_distinct_attribute(&mut self, body: Value) {
-        let url = format!("/indexes/{}/settings/distinct-attribute", self.uid);
-        self.post_request_async(&url, body);
-    }
-
-    pub fn delete_distinct_attribute(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings/distinct-attribute", self.uid);
-        self.delete_request_async(&url)
-    }
-
-    pub fn get_primary_key(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings/primary_key", self.uid);
-        self.get_request(&url)
-    }
-
-    pub fn get_searchable_attributes(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings/searchable-attributes", self.uid);
-        self.get_request(&url)
-    }
-
-    pub fn update_searchable_attributes(&mut self, body: Value) {
-        let url = format!("/indexes/{}/settings/searchable-attributes", self.uid);
-        self.post_request_async(&url, body);
-    }
-
-    pub fn delete_searchable_attributes(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings/searchable-attributes", self.uid);
-        self.delete_request_async(&url)
-    }
-
-    pub fn get_displayed_attributes(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings/displayed-attributes", self.uid);
-        self.get_request(&url)
-    }
-
-    pub fn update_displayed_attributes(&mut self, body: Value) {
-        let url = format!("/indexes/{}/settings/displayed-attributes", self.uid);
-        self.post_request_async(&url, body);
-    }
-
-    pub fn delete_displayed_attributes(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings/displayed-attributes", self.uid);
-        self.delete_request_async(&url)
-    }
-
-    pub fn get_accept_new_fields(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings/accept-new-fields", self.uid);
-        self.get_request(&url)
-    }
-
-    pub fn update_accept_new_fields(&mut self, body: Value) {
-        let url = format!("/indexes/{}/settings/accept-new-fields", self.uid);
-        self.post_request_async(&url, body);
-    }
-
-    pub fn get_synonyms(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings/synonyms", self.uid);
-        self.get_request(&url)
-    }
-
-    pub fn update_synonyms(&mut self, body: Value) {
-        let url = format!("/indexes/{}/settings/synonyms", self.uid);
-        self.post_request_async(&url, body);
-    }
-
-    pub fn delete_synonyms(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings/synonyms", self.uid);
-        self.delete_request_async(&url)
-    }
-
-    pub fn get_stop_words(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings/stop-words", self.uid);
-        self.get_request(&url)
-    }
-
-    pub fn update_stop_words(&mut self, body: Value) {
-        let url = format!("/indexes/{}/settings/stop-words", self.uid);
-        self.post_request_async(&url, body);
-    }
-
-    pub fn delete_stop_words(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings/stop-words", self.uid);
-        self.delete_request_async(&url)
-    }
-
-    pub fn get_index_stats(&mut self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/stats", self.uid);
-        self.get_request(&url)
-    }
-
-    pub fn list_keys(&mut self) -> (Value, StatusCode) {
-        self.get_request("/keys")
-    }
-
-    pub fn get_health(&mut self) -> (Value, StatusCode) {
-        self.get_request("/health")
-    }
-
-    pub fn update_health(&mut self, body: Value) -> (Value, StatusCode) {
-        self.put_request("/health", body)
-    }
-
-    pub fn get_version(&mut self) -> (Value, StatusCode) {
-        self.get_request("/version")
-    }
-
-    pub fn get_sys_info(&mut self) -> (Value, StatusCode) {
-        self.get_request("/sys-info")
-    }
-
-    pub fn get_sys_info_pretty(&mut self) -> (Value, StatusCode) {
-        self.get_request("/sys-info/pretty")
-    }
-
-    // Populate routes
-
-    pub fn populate_movies(&mut self) {
         let body = json!({
-            "uid": "movies",
+            "uid": "test",
             "primaryKey": "id",
         });
-        self.create_index(body);
+
+        server.create_index(body).await;
 
         let body = json!({
             "rankingRules": [
@@ -406,42 +76,411 @@ impl Server {
                 "proximity",
                 "attribute",
                 "wordsPosition",
-                "desc(popularity)",
                 "exactness",
-                "desc(vote_average)",
             ],
             "searchableAttributes": [
-                "title",
-                "tagline",
-                "overview",
-                "cast",
-                "director",
-                "producer",
-                "production_companies",
-                "genres",
+                "balance",
+                "picture",
+                "age",
+                "color",
+                "name",
+                "gender",
+                "email",
+                "phone",
+                "address",
+                "about",
+                "registered",
+                "latitude",
+                "longitude",
+                "tags",
             ],
             "displayedAttributes": [
-                "title",
-                "director",
-                "producer",
-                "tagline",
-                "genres",
                 "id",
-                "overview",
-                "vote_count",
-                "vote_average",
-                "poster_path",
-                "popularity",
+                "isActive",
+                "balance",
+                "picture",
+                "age",
+                "color",
+                "name",
+                "gender",
+                "email",
+                "phone",
+                "address",
+                "about",
+                "registered",
+                "latitude",
+                "longitude",
+                "tags",
             ],
-            "acceptNewFields": false,
         });
 
-        self.update_all_settings(body);
+        server.update_all_settings(body).await;
 
-        let dataset = include_bytes!("assets/movies.json");
+        let dataset = include_bytes!("assets/test_set.json");
 
         let body: Value = serde_json::from_slice(dataset).unwrap();
 
-        self.add_or_replace_multiple_documents(body);
+        server.add_or_replace_multiple_documents(body).await;
+        server
+    }
+
+
+    pub async fn wait_update_id(&mut self, update_id: u64) {
+        // try 10 times to get status, or panic to not wait forever
+        for _ in 0..10 {
+            let (response, status_code) = self.get_update_status(update_id).await;
+            assert_eq!(status_code, 200);
+
+            if response["status"] == "processed" || response["status"] == "failed" {
+                eprintln!("{:#?}", response);
+                return;
+            }
+
+            delay_for(Duration::from_secs(1)).await;
+        }
+        panic!("Timeout waiting for update id");
+    }
+
+    // Global Http request GET/POST/DELETE async or sync
+
+    pub async fn get_request(&mut self, url: &str) -> (Value, StatusCode) {
+        eprintln!("get_request: {}", url);
+
+        let mut app = test::init_service(meilisearch_http::create_app(&self.data).wrap(NormalizePath)).await;
+
+        let req = test::TestRequest::get().uri(url).to_request();
+        let res = test::call_service(&mut app, req).await;
+        let status_code = res.status().clone();
+
+        let body = test::read_body(res).await;
+        let response = serde_json::from_slice(&body).unwrap_or_default();
+        (response, status_code)
+    }
+
+    pub async fn post_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) {
+        eprintln!("post_request: {}", url);
+
+        let mut app = test::init_service(meilisearch_http::create_app(&self.data).wrap(NormalizePath)).await;
+
+        let req = test::TestRequest::post()
+            .uri(url)
+            .set_json(&body)
+            .to_request();
+        let res = test::call_service(&mut app, req).await;
+        let status_code = res.status().clone();
+
+        let body = test::read_body(res).await;
+        let response = serde_json::from_slice(&body).unwrap_or_default();
+        (response, status_code)
+    }
+
+    pub async fn post_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) {
+        eprintln!("post_request_async: {}", url);
+
+        let (response, status_code) = self.post_request(url, body).await;
+        eprintln!("response: {}", response);
+        assert_eq!(status_code, 202);
+        assert!(response["updateId"].as_u64().is_some());
+        self.wait_update_id(response["updateId"].as_u64().unwrap())
+            .await;
+        (response, status_code)
+    }
+
+    pub async fn put_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) {
+        eprintln!("put_request: {}", url);
+
+        let mut app = test::init_service(meilisearch_http::create_app(&self.data).wrap(NormalizePath)).await;
+
+        let req = test::TestRequest::put()
+            .uri(url)
+            .set_json(&body)
+            .to_request();
+        let res = test::call_service(&mut app, req).await;
+        let status_code = res.status().clone();
+
+        let body = test::read_body(res).await;
+        let response = serde_json::from_slice(&body).unwrap_or_default();
+        (response, status_code)
+    }
+
+    pub async fn put_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) {
+        eprintln!("put_request_async: {}", url);
+
+        let (response, status_code) = self.put_request(url, body).await;
+        assert!(response["updateId"].as_u64().is_some());
+        assert_eq!(status_code, 202);
+        self.wait_update_id(response["updateId"].as_u64().unwrap())
+            .await;
+        (response, status_code)
+    }
+
+    pub async fn delete_request(&mut self, url: &str) -> (Value, StatusCode) {
+        eprintln!("delete_request: {}", url);
+
+        let mut app = test::init_service(meilisearch_http::create_app(&self.data).wrap(NormalizePath)).await;
+
+        let req = test::TestRequest::delete().uri(url).to_request();
+        let res = test::call_service(&mut app, req).await;
+        let status_code = res.status().clone();
+
+        let body = test::read_body(res).await;
+        let response = serde_json::from_slice(&body).unwrap_or_default();
+        (response, status_code)
+    }
+
+    pub async fn delete_request_async(&mut self, url: &str) -> (Value, StatusCode) {
+        eprintln!("delete_request_async: {}", url);
+
+        let (response, status_code) = self.delete_request(url).await;
+        assert!(response["updateId"].as_u64().is_some());
+        assert_eq!(status_code, 202);
+        self.wait_update_id(response["updateId"].as_u64().unwrap())
+            .await;
+        (response, status_code)
+    }
+
+    // All Routes
+
+    pub async fn list_indexes(&mut self) -> (Value, StatusCode) {
+        self.get_request("/indexes").await
+    }
+
+    pub async fn create_index(&mut self, body: Value) -> (Value, StatusCode) {
+        self.post_request("/indexes", body).await
+    }
+
+    pub async fn search_multi_index(&mut self, query: &str) -> (Value, StatusCode) {
+        let url = format!("/indexes/search?{}", query);
+        self.get_request(&url).await
+    }
+
+    pub async fn get_index(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}", self.uid);
+        self.get_request(&url).await
+    }
+
+    pub async fn update_index(&mut self, body: Value) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}", self.uid);
+        self.put_request(&url, body).await
+    }
+
+    pub async fn delete_index(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}", self.uid);
+        self.delete_request(&url).await
+    }
+
+    pub async fn search_get(&mut self, query: &str) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/search?{}", self.uid, query);
+        self.get_request(&url).await
+    }
+
+    pub async fn search_post(&mut self, body: Value) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/search", self.uid);
+        self.post_request(&url, body).await
+    }
+
+    pub async fn get_all_updates_status(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/updates", self.uid);
+        self.get_request(&url).await
+    }
+
+    pub async fn get_update_status(&mut self, update_id: u64) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/updates/{}", self.uid, update_id);
+        self.get_request(&url).await
+    }
+
+    pub async fn get_all_documents(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/documents", self.uid);
+        self.get_request(&url).await
+    }
+
+    pub async fn add_or_replace_multiple_documents(&mut self, body: Value) {
+        let url = format!("/indexes/{}/documents", self.uid);
+        self.post_request_async(&url, body).await;
+    }
+
+    pub async fn add_or_replace_multiple_documents_sync(
+        &mut self,
+        body: Value,
+    ) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/documents", self.uid);
+        self.post_request(&url, body).await
+    }
+
+    pub async fn add_or_update_multiple_documents(&mut self, body: Value) {
+        let url = format!("/indexes/{}/documents", self.uid);
+        self.put_request_async(&url, body).await;
+    }
+
+    pub async fn clear_all_documents(&mut self) {
+        let url = format!("/indexes/{}/documents", self.uid);
+        self.delete_request_async(&url).await;
+    }
+
+    pub async fn get_document(&mut self, document_id: impl ToString) -> (Value, StatusCode) {
+        let url = format!(
+            "/indexes/{}/documents/{}",
+            self.uid,
+            document_id.to_string()
+        );
+        self.get_request(&url).await
+    }
+
+    pub async fn delete_document(&mut self, document_id: impl ToString) -> (Value, StatusCode) {
+        let url = format!(
+            "/indexes/{}/documents/{}",
+            self.uid,
+            document_id.to_string()
+        );
+        self.delete_request_async(&url).await
+    }
+
+    pub async fn delete_multiple_documents(&mut self, body: Value) {
+        let url = format!("/indexes/{}/documents/delete-batch", self.uid);
+        self.post_request_async(&url, body).await;
+    }
+
+    pub async fn get_all_settings(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings", self.uid);
+        self.get_request(&url).await
+    }
+
+    pub async fn update_all_settings(&mut self, body: Value) {
+        let url = format!("/indexes/{}/settings", self.uid);
+        self.post_request_async(&url, body).await;
+    }
+
+    pub async fn delete_all_settings(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings", self.uid);
+        self.delete_request_async(&url).await
+    }
+
+    pub async fn get_ranking_rules(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/ranking-rules", self.uid);
+        self.get_request(&url).await
+    }
+
+    pub async fn update_ranking_rules(&mut self, body: Value) {
+        let url = format!("/indexes/{}/settings/ranking-rules", self.uid);
+        self.post_request_async(&url, body).await;
+    }
+
+    pub async fn update_ranking_rules_sync(&mut self, body: Value) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/ranking-rules", self.uid);
+        self.post_request(&url, body).await
+    }
+
+    pub async fn delete_ranking_rules(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/ranking-rules", self.uid);
+        self.delete_request_async(&url).await
+    }
+
+    pub async fn get_distinct_attribute(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/distinct-attribute", self.uid);
+        self.get_request(&url).await
+    }
+
+    pub async fn update_distinct_attribute(&mut self, body: Value) {
+        let url = format!("/indexes/{}/settings/distinct-attribute", self.uid);
+        self.post_request_async(&url, body).await;
+    }
+
+    pub async fn delete_distinct_attribute(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/distinct-attribute", self.uid);
+        self.delete_request_async(&url).await
+    }
+
+    pub async fn get_primary_key(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/primary_key", self.uid);
+        self.get_request(&url).await
+    }
+
+    pub async fn get_searchable_attributes(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/searchable-attributes", self.uid);
+        self.get_request(&url).await
+    }
+
+    pub async fn update_searchable_attributes(&mut self, body: Value) {
+        let url = format!("/indexes/{}/settings/searchable-attributes", self.uid);
+        self.post_request_async(&url, body).await;
+    }
+
+    pub async fn delete_searchable_attributes(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/searchable-attributes", self.uid);
+        self.delete_request_async(&url).await
+    }
+
+    pub async fn get_displayed_attributes(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/displayed-attributes", self.uid);
+        self.get_request(&url).await
+    }
+
+    pub async fn update_displayed_attributes(&mut self, body: Value) {
+        let url = format!("/indexes/{}/settings/displayed-attributes", self.uid);
+        self.post_request_async(&url, body).await;
+    }
+
+    pub async fn delete_displayed_attributes(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/displayed-attributes", self.uid);
+        self.delete_request_async(&url).await
+    }
+
+    pub async fn get_synonyms(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/synonyms", self.uid);
+        self.get_request(&url).await
+    }
+
+    pub async fn update_synonyms(&mut self, body: Value) {
+        let url = format!("/indexes/{}/settings/synonyms", self.uid);
+        self.post_request_async(&url, body).await;
+    }
+
+    pub async fn delete_synonyms(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/synonyms", self.uid);
+        self.delete_request_async(&url).await
+    }
+
+    pub async fn get_stop_words(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/stop-words", self.uid);
+        self.get_request(&url).await
+    }
+
+    pub async fn update_stop_words(&mut self, body: Value) {
+        let url = format!("/indexes/{}/settings/stop-words", self.uid);
+        self.post_request_async(&url, body).await;
+    }
+
+    pub async fn delete_stop_words(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/stop-words", self.uid);
+        self.delete_request_async(&url).await
+    }
+
+    pub async fn get_index_stats(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/stats", self.uid);
+        self.get_request(&url).await
+    }
+
+    pub async fn list_keys(&mut self) -> (Value, StatusCode) {
+        self.get_request("/keys").await
+    }
+
+    pub async fn get_health(&mut self) -> (Value, StatusCode) {
+        self.get_request("/health").await
+    }
+
+    pub async fn update_health(&mut self, body: Value) -> (Value, StatusCode) {
+        self.put_request("/health", body).await
+    }
+
+    pub async fn get_version(&mut self) -> (Value, StatusCode) {
+        self.get_request("/version").await
+    }
+
+    pub async fn get_sys_info(&mut self) -> (Value, StatusCode) {
+        self.get_request("/sys-info").await
+    }
+
+    pub async fn get_sys_info_pretty(&mut self) -> (Value, StatusCode) {
+        self.get_request("/sys-info/pretty").await
     }
 }

meilisearch-http/tests/dashboard.rs

@@ -0,0 +1,12 @@
+mod common;
+
+#[actix_rt::test]
+async fn dashboard() {
+    let mut server = common::Server::with_uid("movies");
+
+    let (_response, status_code) = server.get_request("/").await;
+    assert_eq!(status_code, 200);
+
+    let (_response, status_code) = server.get_request("/bulma.min.css").await;
+    assert_eq!(status_code, 200);
+}

meilisearch-http/tests/documents_add.rs

@@ -3,8 +3,8 @@ use serde_json::json;
 mod common;
 
 // Test issue https://github.com/meilisearch/MeiliSearch/issues/519
-#[test]
-fn check_add_documents_with_primary_key_param() {
+#[actix_rt::test]
+async fn check_add_documents_with_primary_key_param() {
     let mut server = common::Server::with_uid("movies");
 
     // 1 - Create the index with no primary_key
@@ -12,7 +12,7 @@ fn check_add_documents_with_primary_key_param() {
     let body = json!({
         "uid": "movies",
     });
-    let (response, status_code) = server.create_index(body);
+    let (response, status_code) = server.create_index(body).await;
     assert_eq!(status_code, 201);
     assert_eq!(response["primaryKey"], json!(null));
 
@@ -24,28 +24,28 @@ fn check_add_documents_with_primary_key_param() {
     }]);
 
     let url = "/indexes/movies/documents?primaryKey=title";
-    let (response, status_code) = server.post_request(&url, body);
+    let (response, status_code) = server.post_request(&url, body).await;
     eprintln!("{:#?}", response);
     assert_eq!(status_code, 202);
     let update_id = response["updateId"].as_u64().unwrap();
-    server.wait_update_id(update_id);
+    server.wait_update_id(update_id).await;
 
     // 3 - Check update success
 
-    let (response, status_code) = server.get_update_status(update_id);
+    let (response, status_code) = server.get_update_status(update_id).await;
     assert_eq!(status_code, 200);
     assert_eq!(response["status"], "processed");
 }
 
 // Test issue https://github.com/meilisearch/MeiliSearch/issues/568
-#[test]
-fn check_add_documents_with_nested_boolean() {
+#[actix_rt::test]
+async fn check_add_documents_with_nested_boolean() {
     let mut server = common::Server::with_uid("tasks");
 
     // 1 - Create the index with no primary_key
 
     let body = json!({ "uid": "tasks" });
-    let (response, status_code) = server.create_index(body);
+    let (response, status_code) = server.create_index(body).await;
     assert_eq!(status_code, 201);
     assert_eq!(response["primaryKey"], json!(null));
 
@@ -64,71 +64,71 @@ fn check_add_documents_with_nested_boolean() {
     }]);
 
     let url = "/indexes/tasks/documents";
-    let (response, status_code) = server.post_request(&url, body);
+    let (response, status_code) = server.post_request(&url, body).await;
     eprintln!("{:#?}", response);
     assert_eq!(status_code, 202);
     let update_id = response["updateId"].as_u64().unwrap();
-    server.wait_update_id(update_id);
+    server.wait_update_id(update_id).await;
 
     // 3 - Check update success
 
-    let (response, status_code) = server.get_update_status(update_id);
+    let (response, status_code) = server.get_update_status(update_id).await;
     assert_eq!(status_code, 200);
     assert_eq!(response["status"], "processed");
 }
 
 // Test issue https://github.com/meilisearch/MeiliSearch/issues/571
-#[test]
-fn check_add_documents_with_nested_null() {
+#[actix_rt::test]
+async fn check_add_documents_with_nested_null() {
     let mut server = common::Server::with_uid("tasks");
 
     // 1 - Create the index with no primary_key
 
     let body = json!({ "uid": "tasks" });
-    let (response, status_code) = server.create_index(body);
+    let (response, status_code) = server.create_index(body).await;
     assert_eq!(status_code, 201);
     assert_eq!(response["primaryKey"], json!(null));
 
     // 2 - Add a document that contains a null in a nested object
 
-    let body = json!([{ 
-        "id": 0, 
-        "foo": { 
+    let body = json!([{
+        "id": 0,
+        "foo": {
             "bar": null
-        } 
+        }
     }]);
 
     let url = "/indexes/tasks/documents";
-    let (response, status_code) = server.post_request(&url, body);
+    let (response, status_code) = server.post_request(&url, body).await;
     eprintln!("{:#?}", response);
     assert_eq!(status_code, 202);
     let update_id = response["updateId"].as_u64().unwrap();
-    server.wait_update_id(update_id);
+    server.wait_update_id(update_id).await;
 
     // 3 - Check update success
 
-    let (response, status_code) = server.get_update_status(update_id);
+    let (response, status_code) = server.get_update_status(update_id).await;
     assert_eq!(status_code, 200);
     assert_eq!(response["status"], "processed");
 }
 
 // Test issue https://github.com/meilisearch/MeiliSearch/issues/574
-#[test]
-fn check_add_documents_with_nested_sequence() {
+#[actix_rt::test]
+async fn check_add_documents_with_nested_sequence() {
     let mut server = common::Server::with_uid("tasks");
 
     // 1 - Create the index with no primary_key
 
     let body = json!({ "uid": "tasks" });
-    let (response, status_code) = server.create_index(body);
+    let (response, status_code) = server.create_index(body).await;
     assert_eq!(status_code, 201);
     assert_eq!(response["primaryKey"], json!(null));
 
     // 2 - Add a document that contains a seq in a nested object
 
-    let body = json!([{ 
-        "id": 0, 
-        "foo": { 
+    let body = json!([{
+        "id": 0,
+        "foo": {
             "bar": [123,456],
             "fez": [{
                 "id": 255,
@@ -158,20 +158,60 @@ fn check_add_documents_with_nested_sequence() {
     }]);
 
     let url = "/indexes/tasks/documents";
-    let (response, status_code) = server.post_request(&url, body.clone());
+    let (response, status_code) = server.post_request(&url, body.clone()).await;
     eprintln!("{:#?}", response);
     assert_eq!(status_code, 202);
     let update_id = response["updateId"].as_u64().unwrap();
-    server.wait_update_id(update_id);
+    server.wait_update_id(update_id).await;
 
     // 3 - Check update success
 
-    let (response, status_code) = server.get_update_status(update_id);
+    let (response, status_code) = server.get_update_status(update_id).await;
     assert_eq!(status_code, 200);
     assert_eq!(response["status"], "processed");
 
     let url = "/indexes/tasks/search?q=leesz";
-    let (response, status_code) = server.get_request(&url);
+    let (response, status_code) = server.get_request(&url).await;
     assert_eq!(status_code, 200);
     assert_eq!(response["hits"], body);
 }
+
+#[actix_rt::test]
+// test sample from #807
+async fn add_document_with_long_field() {
+    let mut server = common::Server::with_uid("test");
+    server.create_index(json!({ "uid": "test" })).await;
+    let body = json!([{
+        "documentId":"de1c2adbb897effdfe0deae32a01035e46f932ce",
+        "rank":1,
+        "relurl":"/configuration/app/web.html#locations",
+        "section":"Web",
+        "site":"docs",
+        "text":" The locations block is the most powerful, and potentially most involved, section of the .platform.app.yaml file. It allows you to control how the application container responds to incoming requests at a very fine-grained level. Common patterns also vary between language containers due to the way PHP-FPM handles incoming requests.\nEach entry of the locations block is an absolute URI path (with leading /) and its value includes the configuration directives for how the web server should handle matching requests. That is, if your domain is example.com then '/' means “requests for example.com/”, while '/admin' means “requests for example.com/admin”. If multiple blocks could match an incoming request then the most-specific will apply.\nweb:locations:'/':# Rules for all requests that don't otherwise match....'/sites/default/files':# Rules for any requests that begin with /sites/default/files....The simplest possible locations configuration is one that simply passes all requests on to your application unconditionally:\nweb:locations:'/':passthru:trueThat is, all requests to /* should be forwarded to the process started by web.commands.start above. Note that for PHP containers the passthru key must specify what PHP file the request should be forwarded to, and must also specify a docroot under which the file lives. For example:\nweb:locations:'/':root:'web'passthru:'/app.php'This block will serve requests to / from the web directory in the application, and if a file doesn’t exist on disk then the request will be forwarded to the /app.php script.\nA full list of the possible subkeys for locations is below.\n  root: The folder from which to serve static assets for this location relative to the application root. The application root is the directory in which the .platform.app.yaml file is located. Typical values for this property include public or web. Setting it to '' is not recommended, and its behavior may vary depending on the type of application. Absolute paths are not supported.\n  passthru: Whether to forward disallowed and missing resources from this location to the application and can be true, false or an absolute URI path (with leading /). The default value is false. For non-PHP applications it will generally be just true or false. In a PHP application this will typically be the front controller such as /index.php or /app.php. This entry works similar to mod_rewrite under Apache. Note: If the value of passthru does not begin with the same value as the location key it is under, the passthru may evaluate to another entry. That may be useful when you want different cache settings for different paths, for instance, but want missing files in all of them to map back to the same front controller. See the example block below.\n  index: The files to consider when serving a request for a directory: an array of file names or null. (typically ['index.html']). Note that in order for this to work, access to the static files named must be allowed by the allow or rules keys for this location.\n  expires: How long to allow static assets from this location to be cached (this enables the Cache-Control and Expires headers) and can be a time or -1 for no caching (default). Times can be suffixed with “ms” (milliseconds), “s” (seconds), “m” (minutes), “h” (hours), “d” (days), “w” (weeks), “M” (months, 30d) or “y” (years, 365d).\n  scripts: Whether to allow loading scripts in that location (true or false). This directive is only meaningful on PHP.\n  allow: Whether to allow serving files which don’t match a rule (true or false, default: true).\n  headers: Any additional headers to apply to static assets. This section is a mapping of header names to header values. Responses from the application aren’t affected, to avoid overlap with the application’s own ability to include custom headers in the response.\n  rules: Specific overrides for a specific location. The key is a PCRE (regular expression) that is matched against the full request path.\n  request_buffering: Most application servers do not support chunked requests (e.g. fpm, uwsgi), so Platform.sh enables request_buffering by default to handle them. That default configuration would look like this if it was present in .platform.app.yaml:\nweb:locations:'/':passthru:truerequest_buffering:enabled:truemax_request_size:250mIf the application server can already efficiently handle chunked requests, the request_buffering subkey can be modified to disable it entirely (enabled: false). Additionally, applications that frequently deal with uploads greater than 250MB in size can update the max_request_size key to the application’s needs. Note that modifications to request_buffering will need to be specified at each location where it is desired.\n ",
+        "title":"Locations",
+        "url":"/configuration/app/web.html#locations"
+    }]);
+    server.add_or_replace_multiple_documents(body).await;
+    let (response, _status) = server.search_post(json!({ "q": "request_buffering" })).await;
+    assert!(!response["hits"].as_array().unwrap().is_empty());
+}
+
+#[actix_rt::test]
+async fn documents_with_same_id_are_overwritten() {
+    let mut server = common::Server::with_uid("test");
+    server.create_index(json!({ "uid": "test"})).await;
+    let documents = json!([
+        {
+            "id": 1,
+            "content": "test1"
+        },
+        {
+            "id": 1,
+            "content": "test2"
+        },
+    ]);
+    server.add_or_replace_multiple_documents(documents).await;
+    let (response, _status) = server.get_all_documents().await;
+    assert_eq!(response.as_array().unwrap().len(), 1);
+    assert_eq!(response.as_array().unwrap()[0].as_object().unwrap()["content"], "test2");
+}

meilisearch-http/tests/documents_delete.rs

@@ -1,31 +1,34 @@
 mod common;
 
-#[test]
-fn delete() {
-    let mut server = common::Server::with_uid("movies");
-    server.populate_movies();
+#[actix_rt::test]
+async fn delete() {
+    let mut server = common::Server::test_server().await;
 
-    let (_response, status_code) = server.get_document(419704);
+    let (_response, status_code) = server.get_document(50).await;
     assert_eq!(status_code, 200);
 
-    server.delete_document(419704);
+    server.delete_document(50).await;
 
-    let (_response, status_code) = server.get_document(419704);
+    let (_response, status_code) = server.get_document(50).await;
     assert_eq!(status_code, 404);
 }
 
-// Resolve teh issue https://github.com/meilisearch/MeiliSearch/issues/493
-#[test]
-fn delete_batch() {
-    let mut server = common::Server::with_uid("movies");
-    server.populate_movies();
+// Resolve the issue https://github.com/meilisearch/MeiliSearch/issues/493
+#[actix_rt::test]
+async fn delete_batch() {
+    let mut server = common::Server::test_server().await;
 
-    let (_response, status_code) = server.get_document(419704);
-    assert_eq!(status_code, 200);
+    let doc_ids = vec!(50, 55, 60);
+    for doc_id in &doc_ids {
+        let (_response, status_code) = server.get_document(doc_id).await;
+        assert_eq!(status_code, 200);
+    }
 
-    let body = serde_json::json!([419704, 512200, 181812]);
-    server.delete_multiple_documents(body);
+    let body = serde_json::json!(&doc_ids);
+    server.delete_multiple_documents(body).await;
 
-    let (_response, status_code) = server.get_document(419704);
-    assert_eq!(status_code, 404);
+    for doc_id in &doc_ids {
+        let (_response, status_code) = server.get_document(doc_id).await;
+        assert_eq!(status_code, 404);
+    }
 }

meilisearch-http/tests/errors.rs

@@ -0,0 +1,196 @@
+mod common;
+
+use std::thread;
+use std::time::Duration;
+
+use actix_http::http::StatusCode;
+use serde_json::{json, Map, Value};
+
+macro_rules! assert_error {
+    ($code:literal, $type:literal, $status:path, $req:expr) => {
+        let (response, status_code) = $req;
+        assert_eq!(status_code, $status);
+        assert_eq!(response["errorCode"].as_str().unwrap(), $code);
+        assert_eq!(response["errorType"].as_str().unwrap(), $type);
+    };
+}
+
+macro_rules! assert_error_async {
+    ($code:literal, $type:literal, $server:expr, $req:expr) => {
+        let (response, _) = $req;
+        let update_id = response["updateId"].as_u64().unwrap();
+        for _ in 1..10 {
+            let (response, status_code) = $server.get_update_status(update_id).await;
+            assert_eq!(status_code, StatusCode::OK);
+            if response["status"] == "processed" || response["status"] == "failed" {
+                println!("response: {}", response);
+                assert_eq!(response["status"], "failed");
+                assert_eq!(response["errorCode"], $code);
+                assert_eq!(response["errorType"], $type);
+                return
+            }
+            thread::sleep(Duration::from_secs(1));
+        }
+    };
+}
+
+#[actix_rt::test]
+async fn index_already_exists_error() {
+    let mut server = common::Server::with_uid("test");
+    let body = json!({
+        "uid": "test"
+    });
+    let (response, status_code) = server.create_index(body.clone()).await;
+    println!("{}", response);
+    assert_eq!(status_code, StatusCode::CREATED);
+    assert_error!(
+        "index_already_exists",
+        "invalid_request_error",
+        StatusCode::BAD_REQUEST,
+        server.create_index(body).await);
+}
+
+#[actix_rt::test]
+async fn index_not_found_error() {
+    let mut server = common::Server::with_uid("test");
+    assert_error!(
+        "index_not_found",
+        "invalid_request_error",
+        StatusCode::NOT_FOUND,
+        server.get_index().await);
+}
+
+#[actix_rt::test]
+async fn primary_key_already_present_error() {
+    let mut server = common::Server::with_uid("test");
+    let body = json!({
+        "uid": "test",
+        "primaryKey": "test"
+    });
+    server.create_index(body.clone()).await;
+    let body = json!({
+        "primaryKey": "t"
+    });
+    assert_error!(
+        "primary_key_already_present",
+        "invalid_request_error",
+        StatusCode::BAD_REQUEST,
+        server.update_index(body).await);
+}
+
+#[actix_rt::test]
+async fn max_field_limit_exceeded_error() {
+    let mut server = common::Server::test_server().await;
+    let body = json!({
+        "uid": "test",
+    });
+    server.create_index(body).await;
+    let mut doc = Map::with_capacity(70_000);
+    doc.insert("id".into(), Value::String("foo".into()));
+    for i in 0..69_999 {
+        doc.insert(format!("field{}", i), Value::String("foo".into()));
+    }
+    let docs = json!([doc]);
+    assert_error_async!(
+        "max_fields_limit_exceeded",
+        "invalid_request_error",
+        server,
+        server.add_or_replace_multiple_documents_sync(docs).await);
+}
+
+#[actix_rt::test]
+async fn missing_document_id() {
+    let mut server = common::Server::test_server().await;
+    let body = json!({
+        "uid": "test",
+        "primaryKey": "test"
+    });
+    server.create_index(body).await;
+    let docs = json!([
+        {
+            "foo": "bar",
+        }
+    ]);
+    assert_error_async!(
+        "missing_document_id",
+        "invalid_request_error",
+        server,
+        server.add_or_replace_multiple_documents_sync(docs).await);
+}
+
+#[actix_rt::test]
+async fn facet_error() {
+    let mut server = common::Server::test_server().await;
+    let search = json!({
+        "q": "foo",
+        "facetFilters": ["test:hello"]
+    });
+    assert_error!(
+        "invalid_facet",
+        "invalid_request_error",
+        StatusCode::BAD_REQUEST,
+        server.search_post(search).await);
+}
+
+#[actix_rt::test]
+async fn filters_error() {
+    let mut server = common::Server::test_server().await;
+    let search = json!({
+        "q": "foo",
+        "filters": "fo:12"
+    });
+    assert_error!(
+        "invalid_filter",
+        "invalid_request_error",
+        StatusCode::BAD_REQUEST,
+        server.search_post(search).await);
+}
+
+#[actix_rt::test]
+async fn bad_request_error() {
+    let mut server = common::Server::with_uid("test");
+    let body = json!({
+        "foo": "bar",
+    });
+    assert_error!(
+        "bad_request",
+        "invalid_request_error",
+        StatusCode::BAD_REQUEST,
+        server.search_post(body).await);
+}
+
+#[actix_rt::test]
+async fn document_not_found_error() {
+    let mut server = common::Server::with_uid("test");
+    server.create_index(json!({"uid": "test"})).await;
+    assert_error!(
+        "document_not_found",
+        "invalid_request_error",
+        StatusCode::NOT_FOUND,
+        server.get_document(100).await);
+}
+
+#[actix_rt::test]
+async fn payload_too_large_error() {
+    let mut server = common::Server::with_uid("test");
+    let bigvec = vec![0u64; 10_000_000]; // 80mb
+    assert_error!(
+        "payload_too_large",
+        "invalid_request_error",
+        StatusCode::PAYLOAD_TOO_LARGE,
+        server.create_index(json!(bigvec)).await);
+}
+
+#[actix_rt::test]
+async fn missing_primary_key_error() {
+    let mut server = common::Server::with_uid("test");
+    server.create_index(json!({"uid": "test"})).await;
+    let document = json!([{
+        "content": "test"
+    }]);
+    assert_error!(
+        "missing_primary_key",
+        "invalid_request_error",
+        StatusCode::BAD_REQUEST,
+        server.add_or_replace_multiple_documents_sync(document).await);
+}

meilisearch-http/tests/health.rs

@@ -3,36 +3,36 @@ use std::convert::Into;
 
 mod common;
 
-#[test]
-fn test_healthyness() {
+#[actix_rt::test]
+async fn test_healthyness() {
     let mut server = common::Server::with_uid("movies");
 
     // Check that the server is healthy
 
-    let (_response, status_code) = server.get_health();
+    let (_response, status_code) = server.get_health().await;
     assert_eq!(status_code, 200);
 
     // Set the serve Unhealthy
     let body = json!({
         "health": false,
     });
-    let (_response, status_code) = server.update_health(body);
+    let (_response, status_code) = server.update_health(body).await;
     assert_eq!(status_code, 200);
 
     // Check that the server is unhealthy
 
-    let (_response, status_code) = server.get_health();
+    let (_response, status_code) = server.get_health().await;
     assert_eq!(status_code, 503);
 
     // Set the server healthy
     let body = json!({
         "health": true,
     });
-    let (_response, status_code) = server.update_health(body);
+    let (_response, status_code) = server.update_health(body).await;
     assert_eq!(status_code, 200);
 
     // Check if the server is healthy
 
-    let (_response, status_code) = server.get_health();
+    let (_response, status_code) = server.get_health().await;
     assert_eq!(status_code, 200);
 }

meilisearch-http/tests/index.rs

@@ -4,8 +4,8 @@ use serde_json::Value;
 
 mod common;
 
-#[test]
-fn create_index_with_name() {
+#[actix_rt::test]
+async fn create_index_with_name() {
     let mut server = common::Server::with_uid("movies");
 
     // 1 - Create a new index
@@ -14,7 +14,7 @@ fn create_index_with_name() {
         "name": "movies",
     });
 
-    let (res1_value, status_code) = server.create_index(body);
+    let (res1_value, status_code) = server.create_index(body).await;
 
     assert_eq!(status_code, 201);
     assert_eq!(res1_value.as_object().unwrap().len(), 5);
@@ -29,7 +29,7 @@ fn create_index_with_name() {
 
     // 2 - Check the list of indexes
 
-    let (res2_value, status_code) = server.list_indexes();
+    let (res2_value, status_code) = server.list_indexes().await;
 
     assert_eq!(status_code, 200);
     assert_eq!(res2_value.as_array().unwrap().len(), 1);
@@ -44,8 +44,8 @@ fn create_index_with_name() {
     assert_eq!(r2_updated_at.len(), r1_updated_at.len());
 }
 
-#[test]
-fn create_index_with_uid() {
+#[actix_rt::test]
+async fn create_index_with_uid() {
     let mut server = common::Server::with_uid("movies");
 
     // 1 - Create a new index
@@ -54,7 +54,7 @@ fn create_index_with_uid() {
         "uid": "movies",
     });
 
-    let (res1_value, status_code) = server.create_index(body);
+    let (res1_value, status_code) = server.create_index(body.clone()).await;
 
     assert_eq!(status_code, 201);
     assert_eq!(res1_value.as_object().unwrap().len(), 5);
@@ -67,9 +67,16 @@ fn create_index_with_uid() {
     assert!(r1_created_at.len() > 1);
     assert!(r1_updated_at.len() > 1);
 
+    // 1.5 verify that error is thrown when trying to create the same index
+
+    let (response, status_code) = server.create_index(body).await;
+
+    assert_eq!(status_code, 400);
+    assert_eq!(response["errorCode"].as_str().unwrap(), "index_already_exists");
+
     // 2 - Check the list of indexes
 
-    let (res2_value, status_code) = server.list_indexes();
+    let (res2_value, status_code) = server.list_indexes().await;
 
     assert_eq!(status_code, 200);
     assert_eq!(res2_value.as_array().unwrap().len(), 1);
@@ -84,8 +91,8 @@ fn create_index_with_uid() {
     assert_eq!(r2_updated_at.len(), r1_updated_at.len());
 }
 
-#[test]
-fn create_index_with_name_and_uid() {
+#[actix_rt::test]
+async fn create_index_with_name_and_uid() {
     let mut server = common::Server::with_uid("movies");
 
     // 1 - Create a new index
@@ -94,7 +101,7 @@ fn create_index_with_name_and_uid() {
         "name": "Films",
         "uid": "fr_movies",
     });
-    let (res1_value, status_code) = server.create_index(body);
+    let (res1_value, status_code) = server.create_index(body).await;
 
     assert_eq!(status_code, 201);
     assert_eq!(res1_value.as_object().unwrap().len(), 5);
@@ -109,7 +116,7 @@ fn create_index_with_name_and_uid() {
 
     // 2 - Check the list of indexes
 
-    let (res2_value, status_code) = server.list_indexes();
+    let (res2_value, status_code) = server.list_indexes().await;
 
     assert_eq!(status_code, 200);
     assert_eq!(res2_value.as_array().unwrap().len(), 1);
@@ -124,8 +131,8 @@ fn create_index_with_name_and_uid() {
     assert_eq!(r2_updated_at.len(), r1_updated_at.len());
 }
 
-#[test]
-fn rename_index() {
+#[actix_rt::test]
+async fn rename_index() {
     let mut server = common::Server::with_uid("movies");
 
     // 1 - Create a new index
@@ -135,7 +142,7 @@ fn rename_index() {
         "uid": "movies",
     });
 
-    let (res1_value, status_code) = server.create_index(body);
+    let (res1_value, status_code) = server.create_index(body).await;
 
     assert_eq!(status_code, 201);
     assert_eq!(res1_value.as_object().unwrap().len(), 5);
@@ -154,7 +161,7 @@ fn rename_index() {
         "name": "TV Shows",
     });
 
-    let (res2_value, status_code) = server.update_index(body);
+    let (res2_value, status_code) = server.update_index(body).await;
 
     assert_eq!(status_code, 200);
     assert_eq!(res2_value.as_object().unwrap().len(), 5);
@@ -169,7 +176,7 @@ fn rename_index() {
 
     // 3 - Check the list of indexes
 
-    let (res3_value, status_code) = server.list_indexes();
+    let (res3_value, status_code) = server.list_indexes().await;
 
     assert_eq!(status_code, 200);
     assert_eq!(res3_value.as_array().unwrap().len(), 1);
@@ -184,10 +191,16 @@ fn rename_index() {
     assert_eq!(r3_updated_at.len(), r2_updated_at.len());
 }
 
-#[test]
-fn delete_index_and_recreate_it() {
+#[actix_rt::test]
+async fn delete_index_and_recreate_it() {
     let mut server = common::Server::with_uid("movies");
 
+    // 0 - delete unexisting index is error
+
+    let (response, status_code) = server.delete_request("/indexes/test").await;
+    assert_eq!(status_code, 404);
+    assert_eq!(&response["errorCode"], "index_not_found");
+
     // 1 - Create a new index
 
     let body = json!({
@@ -195,7 +208,7 @@ fn delete_index_and_recreate_it() {
         "uid": "movies",
     });
 
-    let (res1_value, status_code) = server.create_index(body);
+    let (res1_value, status_code) = server.create_index(body).await;
 
     assert_eq!(status_code, 201);
     assert_eq!(res1_value.as_object().unwrap().len(), 5);
@@ -210,7 +223,7 @@ fn delete_index_and_recreate_it() {
 
     // 2 - Check the list of indexes
 
-    let (res2_value, status_code) = server.list_indexes();
+    let (res2_value, status_code) = server.list_indexes().await;
 
     assert_eq!(status_code, 200);
     assert_eq!(res2_value.as_array().unwrap().len(), 1);
@@ -226,13 +239,13 @@ fn delete_index_and_recreate_it() {
 
     // 3- Delete an index
 
-    let (_res2_value, status_code) = server.delete_index();
+    let (_res2_value, status_code) = server.delete_index().await;
 
     assert_eq!(status_code, 204);
 
     // 4 - Check the list of indexes
 
-    let (res2_value, status_code) = server.list_indexes();
+    let (res2_value, status_code) = server.list_indexes().await;
 
     assert_eq!(status_code, 200);
     assert_eq!(res2_value.as_array().unwrap().len(), 0);
@@ -243,7 +256,7 @@ fn delete_index_and_recreate_it() {
         "name": "movies",
     });
 
-    let (res1_value, status_code) = server.create_index(body);
+    let (res1_value, status_code) = server.create_index(body).await;
 
     assert_eq!(status_code, 201);
     assert_eq!(res1_value.as_object().unwrap().len(), 5);
@@ -258,7 +271,7 @@ fn delete_index_and_recreate_it() {
 
     // 6 - Check the list of indexes
 
-    let (res2_value, status_code) = server.list_indexes();
+    let (res2_value, status_code) = server.list_indexes().await;
     assert_eq!(status_code, 200);
     assert_eq!(res2_value.as_array().unwrap().len(), 1);
     assert_eq!(res2_value[0].as_object().unwrap().len(), 5);
@@ -272,8 +285,8 @@ fn delete_index_and_recreate_it() {
     assert_eq!(r2_updated_at.len(), r1_updated_at.len());
 }
 
-#[test]
-fn check_multiples_indexes() {
+#[actix_rt::test]
+async fn check_multiples_indexes() {
     let mut server = common::Server::with_uid("movies");
 
     // 1 - Create a new index
@@ -282,7 +295,7 @@ fn check_multiples_indexes() {
         "name": "movies",
     });
 
-    let (res1_value, status_code) = server.create_index(body);
+    let (res1_value, status_code) = server.create_index(body).await;
 
     assert_eq!(status_code, 201);
     assert_eq!(res1_value.as_object().unwrap().len(), 5);
@@ -297,7 +310,7 @@ fn check_multiples_indexes() {
 
     // 2 - Check the list of indexes
 
-    let (res2_value, status_code) = server.list_indexes();
+    let (res2_value, status_code) = server.list_indexes().await;
 
     assert_eq!(status_code, 200);
     assert_eq!(res2_value.as_array().unwrap().len(), 1);
@@ -317,7 +330,7 @@ fn check_multiples_indexes() {
         "name": "films",
     });
 
-    let (res3_value, status_code) = server.create_index(body);
+    let (res3_value, status_code) = server.create_index(body).await;
 
     assert_eq!(status_code, 201);
     assert_eq!(res3_value.as_object().unwrap().len(), 5);
@@ -332,7 +345,7 @@ fn check_multiples_indexes() {
 
     // 4 - Check the list of indexes
 
-    let (res4_value, status_code) = server.list_indexes();
+    let (res4_value, status_code) = server.list_indexes().await;
 
     assert_eq!(status_code, 200);
     assert_eq!(res4_value.as_array().unwrap().len(), 2);
@@ -370,19 +383,19 @@ fn check_multiples_indexes() {
     }
 }
 
-#[test]
-fn create_index_failed() {
+#[actix_rt::test]
+async fn create_index_failed() {
     let mut server = common::Server::with_uid("movies");
 
     // 2 - Push index creation with empty json body
 
     let body = json!({});
 
-    let (res_value, status_code) = server.create_index(body);
+    let (res_value, status_code) = server.create_index(body).await;
 
     assert_eq!(status_code, 400);
     let message = res_value["message"].as_str().unwrap();
-    assert_eq!(res_value.as_object().unwrap().len(), 1);
+    assert_eq!(res_value.as_object().unwrap().len(), 4);
     assert_eq!(message, "Index creation must have an uid");
 
     // 3 - Create a index with extra data
@@ -392,12 +405,9 @@ fn create_index_failed() {
         "active": true
     });
 
-    let (res_value, status_code) = server.create_index(body);
+    let (_res_value, status_code) = server.create_index(body).await;
 
     assert_eq!(status_code, 400);
-    let message = res_value["message"].as_str().unwrap();
-    assert_eq!(res_value.as_object().unwrap().len(), 1);
-    assert_eq!(message, "invalid data");
 
     // 3 - Create a index with wrong data type
 
@@ -406,17 +416,14 @@ fn create_index_failed() {
         "uid": 0
     });
 
-    let (res_value, status_code) = server.create_index(body);
+    let (_res_value, status_code) = server.create_index(body).await;
 
     assert_eq!(status_code, 400);
-    let message = res_value["message"].as_str().unwrap();
-    assert_eq!(res_value.as_object().unwrap().len(), 1);
-    assert_eq!(message, "invalid data");
 }
 
 // Resolve issue https://github.com/meilisearch/MeiliSearch/issues/492
-#[test]
-fn create_index_with_primary_key_and_index() {
+#[actix_rt::test]
+async fn create_index_with_primary_key_and_index() {
     let mut server = common::Server::with_uid("movies");
 
     // 1 - Create the index
@@ -426,7 +433,7 @@ fn create_index_with_primary_key_and_index() {
         "primaryKey": "id",
     });
 
-    let (_response, status_code) = server.create_index(body);
+    let (_response, status_code) = server.create_index(body).await;
     assert_eq!(status_code, 201);
 
     // 2 - Add content
@@ -436,11 +443,11 @@ fn create_index_with_primary_key_and_index() {
         "text": "The mask"
     }]);
 
-    server.add_or_replace_multiple_documents(body.clone());
+    server.add_or_replace_multiple_documents(body.clone()).await;
 
     // 3 - Retreive document
 
-    let (response, _status_code) = server.get_document(123);
+    let (response, _status_code) = server.get_document(123).await;
 
     let expect = json!({
         "id": 123,
@@ -454,8 +461,8 @@ fn create_index_with_primary_key_and_index() {
 // Test when the given index uid is not valid
 // Should have a 400 status code
 // Should have the right error message
-#[test]
-fn create_index_with_invalid_uid() {
+#[actix_rt::test]
+async fn create_index_with_invalid_uid() {
     let mut server = common::Server::with_uid("");
 
     // 1 - Create the index with invalid uid
@@ -464,11 +471,11 @@ fn create_index_with_invalid_uid() {
         "uid": "the movies"
     });
 
-    let (response, status_code) = server.create_index(body);
+    let (response, status_code) = server.create_index(body).await;
 
     assert_eq!(status_code, 400);
     let message = response["message"].as_str().unwrap();
-    assert_eq!(response.as_object().unwrap().len(), 1);
+    assert_eq!(response.as_object().unwrap().len(), 4);
     assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_).");
 
     // 2 - Create the index with invalid uid
@@ -477,11 +484,11 @@ fn create_index_with_invalid_uid() {
         "uid": "%$#"
     });
 
-    let (response, status_code) = server.create_index(body);
+    let (response, status_code) = server.create_index(body).await;
 
     assert_eq!(status_code, 400);
     let message = response["message"].as_str().unwrap();
-    assert_eq!(response.as_object().unwrap().len(), 1);
+    assert_eq!(response.as_object().unwrap().len(), 4);
     assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_).");
 
     // 3 - Create the index with invalid uid
@@ -490,11 +497,11 @@ fn create_index_with_invalid_uid() {
         "uid": "the~movies"
     });
 
-    let (response, status_code) = server.create_index(body);
+    let (response, status_code) = server.create_index(body).await;
 
     assert_eq!(status_code, 400);
     let message = response["message"].as_str().unwrap();
-    assert_eq!(response.as_object().unwrap().len(), 1);
+    assert_eq!(response.as_object().unwrap().len(), 4);
     assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_).");
 
     // 4 - Create the index with invalid uid
@@ -503,17 +510,17 @@ fn create_index_with_invalid_uid() {
         "uid": "🎉"
     });
 
-    let (response, status_code) = server.create_index(body);
+    let (response, status_code) = server.create_index(body).await;
 
     assert_eq!(status_code, 400);
     let message = response["message"].as_str().unwrap();
-    assert_eq!(response.as_object().unwrap().len(), 1);
+    assert_eq!(response.as_object().unwrap().len(), 4);
     assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_).");
 }
 
 // Test that it's possible to add primary_key if it's not already set on index creation
-#[test]
-fn create_index_and_add_indentifier_after() {
+#[actix_rt::test]
+async fn create_index_and_add_indentifier_after() {
     let mut server = common::Server::with_uid("movies");
 
     // 1 - Create the index with no primary_key
@@ -521,7 +528,7 @@ fn create_index_and_add_indentifier_after() {
     let body = json!({
         "uid": "movies",
     });
-    let (response, status_code) = server.create_index(body);
+    let (response, status_code) = server.create_index(body).await;
     assert_eq!(status_code, 201);
     assert_eq!(response["primaryKey"], json!(null));
 
@@ -531,21 +538,21 @@ fn create_index_and_add_indentifier_after() {
         "primaryKey": "id",
     });
 
-    let (response, status_code) = server.update_index(body);
+    let (response, status_code) = server.update_index(body).await;
     assert_eq!(status_code, 200);
     eprintln!("response: {:#?}", response);
     assert_eq!(response["primaryKey"].as_str().unwrap(), "id");
 
     // 3 - Get index to verify if the primary_key is good
 
-    let (response, status_code) = server.get_index();
+    let (response, status_code) = server.get_index().await;
     assert_eq!(status_code, 200);
     assert_eq!(response["primaryKey"].as_str().unwrap(), "id");
 }
 
 // Test that it's impossible to change the primary_key
-#[test]
-fn create_index_and_update_indentifier_after() {
+#[actix_rt::test]
+async fn create_index_and_update_indentifier_after() {
     let mut server = common::Server::with_uid("movies");
 
     // 1 - Create the index with no primary_key
@@ -554,7 +561,7 @@ fn create_index_and_update_indentifier_after() {
         "uid": "movies",
         "primaryKey": "id",
     });
-    let (response, status_code) = server.create_index(body);
+    let (response, status_code) = server.create_index(body).await;
     assert_eq!(status_code, 201);
     assert_eq!(response["primaryKey"].as_str().unwrap(), "id");
 
@@ -564,19 +571,19 @@ fn create_index_and_update_indentifier_after() {
         "primaryKey": "skuid",
     });
 
-    let (_response, status_code) = server.update_index(body);
+    let (_response, status_code) = server.update_index(body).await;
     assert_eq!(status_code, 400);
 
     // 3 - Get index to verify if the primary_key still the first one
 
-    let (response, status_code) = server.get_index();
+    let (response, status_code) = server.get_index().await;
     assert_eq!(status_code, 200);
     assert_eq!(response["primaryKey"].as_str().unwrap(), "id");
 }
 
 // Test that schema inference work well
-#[test]
-fn create_index_without_primary_key_and_add_document() {
+#[actix_rt::test]
+async fn create_index_without_primary_key_and_add_document() {
     let mut server = common::Server::with_uid("movies");
 
     // 1 - Create the index with no primary_key
@@ -584,7 +591,7 @@ fn create_index_without_primary_key_and_add_document() {
     let body = json!({
         "uid": "movies",
     });
-    let (response, status_code) = server.create_index(body);
+    let (response, status_code) = server.create_index(body).await;
     assert_eq!(status_code, 201);
     assert_eq!(response["primaryKey"], json!(null));
 
@@ -595,18 +602,18 @@ fn create_index_without_primary_key_and_add_document() {
         "title": "I'm a legend",
     }]);
 
-    server.add_or_update_multiple_documents(body);
+    server.add_or_update_multiple_documents(body).await;
 
     // 3 - Get index to verify if the primary_key is good
 
-    let (response, status_code) = server.get_index();
+    let (response, status_code) = server.get_index().await;
     assert_eq!(status_code, 200);
     assert_eq!(response["primaryKey"].as_str().unwrap(), "id");
 }
 
 // Test search with no primary_key
-#[test]
-fn create_index_without_primary_key_and_search() {
+#[actix_rt::test]
+async fn create_index_without_primary_key_and_search() {
     let mut server = common::Server::with_uid("movies");
 
     // 1 - Create the index with no primary_key
@@ -614,7 +621,7 @@ fn create_index_without_primary_key_and_search() {
     let body = json!({
         "uid": "movies",
     });
-    let (response, status_code) = server.create_index(body);
+    let (response, status_code) = server.create_index(body).await;
     assert_eq!(status_code, 201);
     assert_eq!(response["primaryKey"], json!(null));
 
@@ -622,15 +629,15 @@ fn create_index_without_primary_key_and_search() {
 
     let query = "q=captain&limit=3";
 
-    let (response, status_code) = server.search(&query);
+    let (response, status_code) = server.search_get(&query).await;
     assert_eq!(status_code, 200);
     assert_eq!(response["hits"].as_array().unwrap().len(), 0);
 }
 
 // Test the error message when we push an document update and impossibility to find primary key
 // Test issue https://github.com/meilisearch/MeiliSearch/issues/517
-#[test]
-fn check_add_documents_without_primary_key() {
+#[actix_rt::test]
+async fn check_add_documents_without_primary_key() {
     let mut server = common::Server::with_uid("movies");
 
     // 1 - Create the index with no primary_key
@@ -638,7 +645,7 @@ fn check_add_documents_without_primary_key() {
     let body = json!({
         "uid": "movies",
     });
-    let (response, status_code) = server.create_index(body);
+    let (response, status_code) = server.create_index(body).await;
     assert_eq!(status_code, 201);
     assert_eq!(response["primaryKey"], json!(null));
 
@@ -649,40 +656,37 @@ fn check_add_documents_without_primary_key() {
       "comment": "comment test"
     }]);
 
-    let (response, status_code) = server.add_or_replace_multiple_documents_sync(body);
-
-    let expected = json!({
-        "message": "Could not infer a primary key"
-    });
+    let (response, status_code) = server.add_or_replace_multiple_documents_sync(body).await;
 
+    assert_eq!(response.as_object().unwrap().len(), 4);
+    assert_eq!(response["errorCode"], "missing_primary_key");
     assert_eq!(status_code, 400);
-    assert_json_eq!(response, expected, ordered: false);
 }
 
-#[test]
-fn check_first_update_should_bring_up_processed_status_after_first_docs_addition(){
-    let mut server = common::Server::with_uid("movies");
+#[actix_rt::test]
+async fn check_first_update_should_bring_up_processed_status_after_first_docs_addition() {
+    let mut server = common::Server::with_uid("test");
 
     let body = json!({
-        "uid": "movies",
+        "uid": "test",
     });
 
     // 1. Create Index
-    let (response, status_code) = server.create_index(body);
+    let (response, status_code) = server.create_index(body).await;
     assert_eq!(status_code, 201);
     assert_eq!(response["primaryKey"], json!(null));
 
-    let dataset = include_bytes!("assets/movies.json");
+    let dataset = include_bytes!("assets/test_set.json");
 
     let body: Value = serde_json::from_slice(dataset).unwrap();
 
     // 2. Index the documents from movies.json, present inside of assets directory
-    server.add_or_replace_multiple_documents(body);
-    
+    server.add_or_replace_multiple_documents(body).await;
+
     // 3. Fetch the status of the indexing done above.
-    let (response, status_code) = server.get_all_updates_status();
-    
+    let (response, status_code) = server.get_all_updates_status().await;
+
     // 4. Verify the fetch is successful and indexing status is 'processed'
     assert_eq!(status_code, 200);
-    assert_eq!(response[0]["status"], "processed");    
+    assert_eq!(response[0]["status"], "processed");
 }

meilisearch-http/tests/placeholder_search.rs

@@ -0,0 +1,497 @@
+use std::convert::Into;
+
+use serde_json::json;
+use serde_json::Value;
+use std::sync::Mutex;
+use std::cell::RefCell;
+
+#[macro_use] mod common;
+
+#[actix_rt::test]
+async fn placeholder_search_with_limit() {
+    let mut server = common::Server::test_server().await;
+
+    let query = json! ({
+        "limit": 3
+    });
+
+    test_post_get_search!(server, query, |response, status_code| {
+        assert_eq!(status_code, 200);
+        assert_eq!(response["hits"].as_array().unwrap().len(), 3);
+    });
+}
+
+#[actix_rt::test]
+async fn placeholder_search_with_offset() {
+    let mut server = common::Server::test_server().await;
+
+    let query = json!({
+        "limit": 6,
+    });
+
+    // hack to take a value out of macro (must implement UnwindSafe)
+    let expected = Mutex::new(RefCell::new(Vec::new()));
+
+    test_post_get_search!(server, query, |response, status_code| {
+        assert_eq!(status_code, 200);
+        // take results at offset 3 as reference
+        let lock = expected.lock().unwrap();
+        lock.replace(response["hits"].as_array().unwrap()[3..6].iter().cloned().collect());
+    });
+
+    let expected = expected.into_inner().unwrap().into_inner();
+
+    let query = json!({
+        "limit": 3,
+        "offset": 3,
+    });
+    test_post_get_search!(server, query, |response, status_code| {
+        assert_eq!(status_code, 200);
+        let response = response["hits"].as_array().unwrap();
+        assert_eq!(&expected, response);
+    });
+}
+
+#[actix_rt::test]
+async fn placeholder_search_with_attribute_to_highlight_wildcard() {
+    // there should be no highlight in placeholder search
+    let mut server = common::Server::test_server().await;
+
+    let query = json!({
+        "limit": 1,
+        "attributesToHighlight": ["*"]
+    });
+
+    test_post_get_search!(server, query, |response, status_code| {
+        assert_eq!(status_code, 200);
+        let result = response["hits"]
+            .as_array()
+            .unwrap()[0]
+            .as_object()
+            .unwrap();
+        for value in result.values() {
+            assert!(value.to_string().find("<em>").is_none());
+        }
+    });
+}
+
+#[actix_rt::test]
+async fn placeholder_search_with_matches() {
+    // matches is always empty
+    let mut server = common::Server::test_server().await;
+
+    let query = json!({
+        "matches": true
+    });
+
+    test_post_get_search!(server, query, |response, status_code| {
+        assert_eq!(status_code, 200);
+        let result = response["hits"]
+            .as_array()
+            .unwrap()
+            .iter()
+            .map(|v| v.as_object().unwrap()["_matchesInfo"].clone())
+            .all(|m| m.as_object().unwrap().is_empty());
+        assert!(result);
+    });
+}
+
+#[actix_rt::test]
+async fn placeholder_search_witch_crop() {
+    // placeholder search crop always crop from beggining
+    let mut server = common::Server::test_server().await;
+
+    let query = json!({
+        "attributesToCrop": ["about"],
+        "cropLength": 20
+    });
+
+    test_post_get_search!(server, query, |response, status_code| {
+        assert_eq!(status_code, 200);
+
+        let hits = response["hits"].as_array().unwrap();
+
+        for hit in hits {
+            let hit = hit.as_object().unwrap();
+            let formatted = hit["_formatted"].as_object().unwrap();
+
+            let about = hit["about"].as_str().unwrap();
+            let about_formatted = formatted["about"].as_str().unwrap();
+            // the formatted about length should be about 20 characters long
+            assert!(about_formatted.len() < 20 + 10);
+            // the formatted part should be located at the beginning of the original one
+            assert_eq!(about.find(&about_formatted).unwrap(), 0);
+        }
+    });
+}
+
+#[actix_rt::test]
+async fn placeholder_search_with_attributes_to_retrieve() {
+    let mut server = common::Server::test_server().await;
+
+    let query = json!({
+        "limit": 1,
+        "attributesToRetrieve": ["gender", "about"],
+    });
+
+    test_post_get_search!(server, query, |response, _status_code| {
+        let hit = response["hits"]
+            .as_array()
+            .unwrap()[0]
+            .as_object()
+            .unwrap();
+        assert_eq!(hit.values().count(), 2);
+        let _ = hit["gender"];
+        let _ = hit["about"];
+    });
+}
+
+#[actix_rt::test]
+async fn placeholder_search_with_filter() {
+    let mut server = common::Server::test_server().await;
+
+    let query = json!({
+        "filters": "color='green'"
+    });
+
+    test_post_get_search!(server, query, |response, _status_code| {
+        let hits = response["hits"].as_array().unwrap();
+        assert!(hits.iter().all(|v| v["color"].as_str().unwrap() == "green"));
+    });
+
+    let query = json!({
+        "filters": "tags=bug"
+    });
+
+    test_post_get_search!(server, query, |response, _status_code| {
+        let hits = response["hits"].as_array().unwrap();
+        let value = Value::String(String::from("bug"));
+        assert!(hits.iter().all(|v| v["tags"].as_array().unwrap().contains(&value)));
+    });
+
+    let query = json!({
+        "filters": "color='green' AND (tags='bug' OR tags='wontfix')"
+    });
+    test_post_get_search!(server, query, |response, _status_code| {
+        let hits = response["hits"].as_array().unwrap();
+        let bug = Value::String(String::from("bug"));
+        let wontfix = Value::String(String::from("wontfix"));
+        assert!(hits.iter().all(|v|
+                v["color"].as_str().unwrap() == "green" &&
+                v["tags"].as_array().unwrap().contains(&bug) ||
+                v["tags"].as_array().unwrap().contains(&wontfix)));
+    });
+}
+
+#[actix_rt::test]
+async fn placeholder_test_faceted_search_valid() {
+    let mut server = common::Server::test_server().await;
+
+    // simple tests on attributes with string value
+    let body = json!({
+        "attributesForFaceting": ["color"]
+    });
+
+    server.update_all_settings(body).await;
+
+    let query = json!({
+        "facetFilters": ["color:green"]
+    });
+
+    test_post_get_search!(server, query, |response, _status_code| {
+        assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty());
+        assert!(response
+            .get("hits")
+            .unwrap()
+            .as_array()
+            .unwrap()
+            .iter()
+            .all(|value| value.get("color").unwrap() == "green"));
+    });
+
+    let query = json!({
+        "facetFilters": [["color:blue"]]
+    });
+
+    test_post_get_search!(server, query, |response, _status_code| {
+        assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty());
+        assert!(response
+            .get("hits")
+            .unwrap()
+            .as_array()
+            .unwrap()
+            .iter()
+            .all(|value| value.get("color").unwrap() == "blue"));
+    });
+
+    let query = json!({
+        "facetFilters": ["color:Blue"]
+    });
+
+    test_post_get_search!(server, query, |response, _status_code| {
+        assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty());
+        assert!(response
+            .get("hits")
+            .unwrap()
+            .as_array()
+            .unwrap()
+            .iter()
+            .all(|value| value.get("color").unwrap() == "blue"));
+    });
+
+    // test on arrays: ["tags:bug"]
+    let body = json!({
+        "attributesForFaceting": ["color", "tags"]
+    });
+
+    server.update_all_settings(body).await;
+
+    let query = json!({
+        "facetFilters": ["tags:bug"]
+    });
+    test_post_get_search!(server, query, |response, _status_code| {
+        assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty());
+        assert!(response
+            .get("hits")
+            .unwrap()
+            .as_array()
+            .unwrap()
+            .iter()
+            .all(|value| value.get("tags").unwrap().as_array().unwrap().contains(&Value::String("bug".to_owned()))));
+    });
+
+    // test and: ["color:blue", "tags:bug"]
+    let query = json!({
+        "facetFilters": ["color:blue", "tags:bug"]
+    });
+    test_post_get_search!(server, query, |response, _status_code| {
+        assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty());
+        assert!(response
+            .get("hits")
+            .unwrap()
+            .as_array()
+            .unwrap()
+            .iter()
+            .all(|value| value
+                .get("color")
+                .unwrap() == "blue"
+                && value.get("tags").unwrap().as_array().unwrap().contains(&Value::String("bug".to_owned()))));
+    });
+
+    // test or: [["color:blue", "color:green"]]
+    let query = json!({
+        "facetFilters": [["color:blue", "color:green"]]
+    });
+    test_post_get_search!(server, query, |response, _status_code| {
+        assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty());
+        assert!(response
+            .get("hits")
+            .unwrap()
+            .as_array()
+            .unwrap()
+            .iter()
+            .all(|value|
+                value
+                .get("color")
+                .unwrap() == "blue"
+                || value
+                .get("color")
+                .unwrap() == "green"));
+    });
+    // test and-or: ["tags:bug", ["color:blue", "color:green"]]
+    let query = json!({
+        "facetFilters": ["tags:bug", ["color:blue", "color:green"]]
+    });
+    test_post_get_search!(server, query, |response, _status_code| {
+        assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty());
+        assert!(response
+            .get("hits")
+            .unwrap()
+            .as_array()
+            .unwrap()
+            .iter()
+            .all(|value|
+                value
+                .get("tags")
+                .unwrap()
+                .as_array()
+                .unwrap()
+                .contains(&Value::String("bug".to_owned()))
+                && (value
+                    .get("color")
+                    .unwrap() == "blue"
+                    || value
+                    .get("color")
+                    .unwrap() == "green")));
+
+    });
+}
+
+#[actix_rt::test]
+async fn placeholder_test_faceted_search_invalid() {
+    let mut server = common::Server::test_server().await;
+
+    //no faceted attributes set
+    let query = json!({
+        "facetFilters": ["color:blue"]
+    });
+    test_post_get_search!(server, query, |_response, status_code| assert_ne!(status_code, 202));
+
+    let body = json!({
+        "attributesForFaceting": ["color", "tags"]
+    });
+    server.update_all_settings(body).await;
+    // empty arrays are error
+    // []
+    let query = json!({
+        "facetFilters": []
+    });
+    test_post_get_search!(server, query, |_response, status_code| assert_ne!(status_code, 202));
+    // [[]]
+    let query = json!({
+        "facetFilters": [[]]
+    });
+    test_post_get_search!(server, query, |_response, status_code| assert_ne!(status_code, 202));
+    // ["color:green", []]
+    let query = json!({
+        "facetFilters": ["color:green", []]
+    });
+    test_post_get_search!(server, query, |_response, status_code| assert_ne!(status_code, 202));
+
+    // too much depth
+    // [[[]]]
+    let query = json!({
+        "facetFilters": [[[]]]
+    });
+    test_post_get_search!(server, query, |_response, status_code| assert_ne!(status_code, 202));
+    // [["color:green", ["color:blue"]]]
+    let query = json!({
+        "facetFilters": [["color:green", ["color:blue"]]]
+    });
+    test_post_get_search!(server, query, |_response, status_code| assert_ne!(status_code, 202));
+    // "color:green"
+    let query = json!({
+        "facetFilters": "color:green"
+    });
+    test_post_get_search!(server, query, |_response, status_code| assert_ne!(status_code, 202));
+}
+
+#[actix_rt::test]
+async fn placeholder_test_facet_count() {
+    let mut server = common::Server::test_server().await;
+
+    // test without facet distribution
+    let query = json!({
+    });
+    test_post_get_search!(server, query, |response, _status_code|{
+        assert!(response.get("exhaustiveFacetsCount").is_none());
+        assert!(response.get("facetsDistribution").is_none());
+    });
+
+    // test no facets set, search on color
+    let query = json!({
+        "facetsDistribution": ["color"]
+    });
+    test_post_get_search!(server, query.clone(), |_response, status_code|{
+        assert_eq!(status_code, 400);
+    });
+
+    let body = json!({
+        "attributesForFaceting": ["color", "tags"]
+    });
+    server.update_all_settings(body).await;
+    // same as before, but now facets are set:
+    test_post_get_search!(server, query, |response, _status_code|{
+        println!("{}", response);
+        assert!(response.get("exhaustiveFacetsCount").is_some());
+        assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 1);
+    });
+    // searching on color and tags
+    let query = json!({
+        "facetsDistribution": ["color", "tags"]
+    });
+    test_post_get_search!(server, query, |response, _status_code|{
+        let facets = response.get("facetsDistribution").unwrap().as_object().unwrap();
+        assert_eq!(facets.values().count(), 2);
+        assert_ne!(!facets.get("color").unwrap().as_object().unwrap().values().count(), 0);
+        assert_ne!(!facets.get("tags").unwrap().as_object().unwrap().values().count(), 0);
+    });
+    // wildcard
+    let query = json!({
+        "facetsDistribution": ["*"]
+    });
+    test_post_get_search!(server, query, |response, _status_code|{
+        assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 2);
+    });
+    // wildcard with other attributes:
+    let query = json!({
+        "facetsDistribution": ["color", "*"]
+    });
+    test_post_get_search!(server, query, |response, _status_code|{
+        assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 2);
+    });
+
+    // empty facet list
+    let query = json!({
+        "facetsDistribution": []
+    });
+    test_post_get_search!(server, query, |response, _status_code|{
+        assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 0);
+    });
+
+    // attr not set as facet passed:
+    let query = json!({
+        "facetsDistribution": ["gender"]
+    });
+    test_post_get_search!(server, query, |_response, status_code|{
+        assert_eq!(status_code, 400);
+    });
+
+}
+
+#[actix_rt::test]
+#[should_panic]
+async fn placeholder_test_bad_facet_distribution() {
+    let mut server = common::Server::test_server().await;
+    // string instead of array:
+    let query = json!({
+        "facetsDistribution": "color"
+    });
+    test_post_get_search!(server, query, |_response, _status_code| {});
+
+    // invalid value in array:
+    let query = json!({
+        "facetsDistribution": ["color", true]
+    });
+    test_post_get_search!(server, query, |_response, _status_code| {});
+}
+
+#[actix_rt::test]
+async fn placeholder_test_sort() {
+    let mut server = common::Server::test_server().await;
+
+    let body = json!({
+        "rankingRules": ["asc(age)"],
+        "attributesForFaceting": ["color"]
+    });
+    server.update_all_settings(body).await;
+    let query = json!({ });
+    test_post_get_search!(server, query, |response, _status_code| {
+        let hits = response["hits"].as_array().unwrap();
+        hits.iter().map(|v| v["age"].as_u64().unwrap()).fold(0, |prev, cur| {
+            assert!(cur >= prev);
+            cur
+        });
+    });
+
+    let query = json!({
+        "facetFilters": ["color:green"]
+    });
+    test_post_get_search!(server, query, |response, _status_code| {
+        let hits = response["hits"].as_array().unwrap();
+        hits.iter().map(|v| v["age"].as_u64().unwrap()).fold(0, |prev, cur| {
+            assert!(cur >= prev);
+            cur
+        });
+    });
+}

meilisearch-http/tests/search_settings.rs

@@ -4,10 +4,9 @@ use std::convert::Into;
 
 mod common;
 
-#[test]
-fn search_with_settings_basic() {
-    let mut server = common::Server::with_uid("movies");
-    server.populate_movies();
+#[actix_rt::test]
+async fn search_with_settings_basic() {
+    let mut server = common::Server::test_server().await;
 
     let config = json!({
       "rankingRules": [
@@ -16,104 +15,78 @@ fn search_with_settings_basic() {
         "proximity",
         "attribute",
         "wordsPosition",
-        "desc(popularity)",
+        "desc(age)",
         "exactness",
-        "desc(vote_average)"
+        "desc(balance)"
       ],
       "distinctAttribute": null,
       "searchableAttributes": [
-        "title",
-        "tagline",
-        "overview",
-        "cast",
-        "director",
-        "producer",
-        "production_companies",
-        "genres"
+        "name",
+        "age",
+        "color",
+        "gender",
+        "email",
+        "address",
+        "about"
       ],
       "displayedAttributes": [
-        "title",
-        "director",
-        "producer",
-        "tagline",
-        "genres",
-        "id",
-        "overview",
-        "vote_count",
-        "vote_average",
-        "poster_path",
-        "popularity"
+        "name",
+        "age",
+        "gender",
+        "color",
+        "email",
+        "phone",
+        "address",
+        "balance"
       ],
       "stopWords": null,
       "synonyms": null,
-      "acceptNewFields": false,
     });
 
-    server.update_all_settings(config);
+    server.update_all_settings(config).await;
+
+    let query = "q=ea%20exercitation&limit=3";
 
-    let query = "q=the%20avangers&limit=3";
     let expect = json!([
       {
-        "id": 24428,
-        "popularity": 44.506,
-        "vote_average": 7.7,
-        "title": "The Avengers",
-        "tagline": "Some assembly required.",
-        "overview": "When an unexpected enemy emerges and threatens global safety and security, Nick Fury, director of the international peacekeeping agency known as S.H.I.E.L.D., finds himself in need of a team to pull the world back from the brink of disaster. Spanning the globe, a daring recruitment effort begins!",
-        "director": "Joss Whedon",
-        "producer": "Kevin Feige",
-        "genres": [
-          "Science Fiction",
-          "Action",
-          "Adventure"
-        ],
-        "poster_path": "https://image.tmdb.org/t/p/w500/cezWGskPY5x7GaglTTRN4Fugfb8.jpg",
-        "vote_count": 21079
+        "balance": "$2,467.47",
+        "age": 34,
+        "color": "blue",
+        "name": "Patricia Goff",
+        "gender": "female",
+        "email": "patriciagoff@chorizon.com",
+        "phone": "+1 (864) 463-2277",
+        "address": "866 Hornell Loop, Cresaptown, Ohio, 1700"
       },
       {
-        "id": 299534,
-        "popularity": 38.659,
-        "vote_average": 8.3,
-        "title": "Avengers: Endgame",
-        "tagline": "Part of the journey is the end.",
-        "overview": "After the devastating events of Avengers: Infinity War, the universe is in ruins due to the efforts of the Mad Titan, Thanos. With the help of remaining allies, the Avengers must assemble once more in order to undo Thanos' actions and restore order to the universe once and for all, no matter what consequences may be in store.",
-        "director": "Anthony Russo",
-        "producer": "Kevin Feige",
-        "genres": [
-          "Adventure",
-          "Science Fiction",
-          "Action"
-        ],
-        "poster_path": "https://image.tmdb.org/t/p/w500/or06FN3Dka5tukK1e9sl16pB3iy.jpg",
-        "vote_count": 10497
+        "balance": "$3,344.40",
+        "age": 35,
+        "color": "blue",
+        "name": "Adeline Flynn",
+        "gender": "female",
+        "email": "adelineflynn@chorizon.com",
+        "phone": "+1 (994) 600-2840",
+        "address": "428 Paerdegat Avenue, Hollymead, Pennsylvania, 948"
       },
       {
-        "id": 299536,
-        "popularity": 65.013,
-        "vote_average": 8.3,
-        "title": "Avengers: Infinity War",
-        "tagline": "An entire universe. Once and for all.",
-        "overview": "As the Avengers and their allies have continued to protect the world from threats too large for any one hero to handle, a new danger has emerged from the cosmic shadows: Thanos. A despot of intergalactic infamy, his goal is to collect all six Infinity Stones, artifacts of unimaginable power, and use them to inflict his twisted will on all of reality. Everything the Avengers have fought for has led up to this moment - the fate of Earth and existence itself has never been more uncertain.",
-        "director": "Anthony Russo",
-        "producer": "Kevin Feige",
-        "genres": [
-          "Adventure",
-          "Action",
-          "Science Fiction"
-        ],
-        "poster_path": "https://image.tmdb.org/t/p/w500/7WsyChQLEftFiDOVTGkv3hFpyyt.jpg",
-        "vote_count": 16056
+        "balance": "$3,394.96",
+        "age": 25,
+        "color": "blue",
+        "name": "Aida Kirby",
+        "gender": "female",
+        "email": "aidakirby@chorizon.com",
+        "phone": "+1 (942) 532-2325",
+        "address": "797 Engert Avenue, Wilsonia, Idaho, 6532"
       }
     ]);
 
-    let (response, _status_code) = server.search(query);
+    let (response, _status_code) = server.search_get(query).await;
     assert_json_eq!(expect, response["hits"].clone(), ordered: false);
 }
 
-#[test]
-fn search_with_settings_stop_words() {
-    let mut server = common::Server::with_uid("movies");
-    server.populate_movies();
+#[actix_rt::test]
+async fn search_with_settings_stop_words() {
+    let mut server = common::Server::test_server().await;
 
     let config = json!({
       "rankingRules": [
@@ -122,104 +95,77 @@ fn search_with_settings_stop_words() {
         "proximity",
         "attribute",
         "wordsPosition",
-        "desc(popularity)",
+        "desc(age)",
         "exactness",
-        "desc(vote_average)"
+        "desc(balance)"
       ],
       "distinctAttribute": null,
       "searchableAttributes": [
-        "title",
-        "tagline",
-        "overview",
-        "cast",
-        "director",
-        "producer",
-        "production_companies",
-        "genres"
+        "name",
+        "age",
+        "color",
+        "gender",
+        "email",
+        "address",
+        "about"
       ],
       "displayedAttributes": [
-        "title",
-        "director",
-        "producer",
-        "tagline",
-        "genres",
-        "id",
-        "overview",
-        "vote_count",
-        "vote_average",
-        "poster_path",
-        "popularity"
+        "name",
+        "age",
+        "gender",
+        "color",
+        "email",
+        "phone",
+        "address",
+        "balance"
       ],
-      "stopWords": ["the"],
+      "stopWords": ["ea"],
       "synonyms": null,
-      "acceptNewFields": false,
     });
 
-    server.update_all_settings(config);
+    server.update_all_settings(config).await;
 
-    let query = "q=the%20avangers&limit=3";
+    let query = "q=ea%20exercitation&limit=3";
     let expect = json!([
       {
-        "id": 299536,
-        "popularity": 65.013,
-        "vote_average": 8.3,
-        "title": "Avengers: Infinity War",
-        "tagline": "An entire universe. Once and for all.",
-        "overview": "As the Avengers and their allies have continued to protect the world from threats too large for any one hero to handle, a new danger has emerged from the cosmic shadows: Thanos. A despot of intergalactic infamy, his goal is to collect all six Infinity Stones, artifacts of unimaginable power, and use them to inflict his twisted will on all of reality. Everything the Avengers have fought for has led up to this moment - the fate of Earth and existence itself has never been more uncertain.",
-        "director": "Anthony Russo",
-        "producer": "Kevin Feige",
-        "genres": [
-          "Adventure",
-          "Action",
-          "Science Fiction"
-        ],
-        "poster_path": "https://image.tmdb.org/t/p/w500/7WsyChQLEftFiDOVTGkv3hFpyyt.jpg",
-        "vote_count": 16056
+        "balance": "$1,921.58",
+        "age": 31,
+        "color": "green",
+        "name": "Harper Carson",
+        "gender": "male",
+        "email": "harpercarson@chorizon.com",
+        "phone": "+1 (912) 430-3243",
+        "address": "883 Dennett Place, Knowlton, New Mexico, 9219"
       },
       {
-        "id": 299534,
-        "popularity": 38.659,
-        "vote_average": 8.3,
-        "title": "Avengers: Endgame",
-        "tagline": "Part of the journey is the end.",
-        "overview": "After the devastating events of Avengers: Infinity War, the universe is in ruins due to the efforts of the Mad Titan, Thanos. With the help of remaining allies, the Avengers must assemble once more in order to undo Thanos' actions and restore order to the universe once and for all, no matter what consequences may be in store.",
-        "director": "Anthony Russo",
-        "producer": "Kevin Feige",
-        "genres": [
-          "Adventure",
-          "Science Fiction",
-          "Action"
-        ],
-        "poster_path": "https://image.tmdb.org/t/p/w500/or06FN3Dka5tukK1e9sl16pB3iy.jpg",
-        "vote_count": 10497
+        "balance": "$1,706.13",
+        "age": 27,
+        "color": "green",
+        "name": "Cherry Orr",
+        "gender": "female",
+        "email": "cherryorr@chorizon.com",
+        "phone": "+1 (995) 479-3174",
+        "address": "442 Beverly Road, Ventress, New Mexico, 3361"
       },
       {
-        "id": 99861,
-        "popularity": 33.938,
-        "vote_average": 7.3,
-        "title": "Avengers: Age of Ultron",
-        "tagline": "A New Age Has Come.",
-        "overview": "When Tony Stark tries to jumpstart a dormant peacekeeping program, things go awry and Earth’s Mightiest Heroes are put to the ultimate test as the fate of the planet hangs in the balance. As the villainous Ultron emerges, it is up to The Avengers to stop him from enacting his terrible plans, and soon uneasy alliances and unexpected action pave the way for an epic and unique global adventure.",
-        "director": "Joss Whedon",
-        "producer": "Kevin Feige",
-        "genres": [
-          "Action",
-          "Adventure",
-          "Science Fiction"
-        ],
-        "poster_path": "https://image.tmdb.org/t/p/w500/t90Y3G8UGQp0f0DrP60wRu9gfrH.jpg",
-        "vote_count": 14661
+        "balance": "$1,476.39",
+        "age": 28,
+        "color": "brown",
+        "name": "Maureen Dale",
+        "gender": "female",
+        "email": "maureendale@chorizon.com",
+        "phone": "+1 (984) 538-3684",
+        "address": "817 Newton Street, Bannock, Wyoming, 1468"
       }
     ]);
 
-    let (response, _status_code) = server.search(query);
+    let (response, _status_code) = server.search_get(query).await;
     assert_json_eq!(expect, response["hits"].clone(), ordered: false);
 }
 
-#[test]
-fn search_with_settings_synonyms() {
-    let mut server = common::Server::with_uid("movies");
-    server.populate_movies();
+#[actix_rt::test]
+async fn search_with_settings_synonyms() {
+    let mut server = common::Server::test_server().await;
 
     let config = json!({
       "rankingRules": [
@@ -228,109 +174,81 @@ fn search_with_settings_synonyms() {
         "proximity",
         "attribute",
         "wordsPosition",
-        "desc(popularity)",
+        "desc(age)",
         "exactness",
-        "desc(vote_average)"
+        "desc(balance)"
       ],
       "distinctAttribute": null,
       "searchableAttributes": [
-        "title",
-        "tagline",
-        "overview",
-        "cast",
-        "director",
-        "producer",
-        "production_companies",
-        "genres"
+        "name",
+        "age",
+        "color",
+        "gender",
+        "email",
+        "address",
+        "about"
       ],
       "displayedAttributes": [
-        "title",
-        "director",
-        "producer",
-        "tagline",
-        "genres",
-        "id",
-        "overview",
-        "vote_count",
-        "vote_average",
-        "poster_path",
-        "popularity"
+        "name",
+        "age",
+        "gender",
+        "color",
+        "email",
+        "phone",
+        "address",
+        "balance"
       ],
       "stopWords": null,
       "synonyms": {
-        "avangers": [
-          "Captain America",
-          "Iron Man"
-        ]
+          "application": [
+              "exercitation"
+          ]
       },
-      "acceptNewFields": false,
     });
 
-    server.update_all_settings(config);
+    server.update_all_settings(config).await;
 
-    let query = "q=avangers&limit=3";
+    let query = "q=application&limit=3";
     let expect = json!([
       {
-        "id": 299536,
-        "popularity": 65.013,
-        "vote_average": 8.3,
-        "title": "Avengers: Infinity War",
-        "tagline": "An entire universe. Once and for all.",
-        "overview": "As the Avengers and their allies have continued to protect the world from threats too large for any one hero to handle, a new danger has emerged from the cosmic shadows: Thanos. A despot of intergalactic infamy, his goal is to collect all six Infinity Stones, artifacts of unimaginable power, and use them to inflict his twisted will on all of reality. Everything the Avengers have fought for has led up to this moment - the fate of Earth and existence itself has never been more uncertain.",
-        "director": "Anthony Russo",
-        "producer": "Kevin Feige",
-        "genres": [
-          "Adventure",
-          "Action",
-          "Science Fiction"
-        ],
-        "vote_count": 16056,
-        "poster_path": "https://image.tmdb.org/t/p/w500/7WsyChQLEftFiDOVTGkv3hFpyyt.jpg"
+        "balance": "$1,921.58",
+        "age": 31,
+        "color": "green",
+        "name": "Harper Carson",
+        "gender": "male",
+        "email": "harpercarson@chorizon.com",
+        "phone": "+1 (912) 430-3243",
+        "address": "883 Dennett Place, Knowlton, New Mexico, 9219"
       },
       {
-        "id": 299534,
-        "popularity": 38.659,
-        "vote_average": 8.3,
-        "title": "Avengers: Endgame",
-        "tagline": "Part of the journey is the end.",
-        "overview": "After the devastating events of Avengers: Infinity War, the universe is in ruins due to the efforts of the Mad Titan, Thanos. With the help of remaining allies, the Avengers must assemble once more in order to undo Thanos' actions and restore order to the universe once and for all, no matter what consequences may be in store.",
-        "director": "Anthony Russo",
-        "producer": "Kevin Feige",
-        "genres": [
-          "Adventure",
-          "Science Fiction",
-          "Action"
-        ],
-        "vote_count": 10497,
-        "poster_path": "https://image.tmdb.org/t/p/w500/or06FN3Dka5tukK1e9sl16pB3iy.jpg"
+        "balance": "$1,706.13",
+        "age": 27,
+        "color": "green",
+        "name": "Cherry Orr",
+        "gender": "female",
+        "email": "cherryorr@chorizon.com",
+        "phone": "+1 (995) 479-3174",
+        "address": "442 Beverly Road, Ventress, New Mexico, 3361"
       },
       {
-        "id": 99861,
-        "popularity": 33.938,
-        "vote_average": 7.3,
-        "title": "Avengers: Age of Ultron",
-        "tagline": "A New Age Has Come.",
-        "overview": "When Tony Stark tries to jumpstart a dormant peacekeeping program, things go awry and Earth’s Mightiest Heroes are put to the ultimate test as the fate of the planet hangs in the balance. As the villainous Ultron emerges, it is up to The Avengers to stop him from enacting his terrible plans, and soon uneasy alliances and unexpected action pave the way for an epic and unique global adventure.",
-        "director": "Joss Whedon",
-        "producer": "Kevin Feige",
-        "genres": [
-          "Action",
-          "Adventure",
-          "Science Fiction"
-        ],
-        "vote_count": 14661,
-        "poster_path": "https://image.tmdb.org/t/p/w500/t90Y3G8UGQp0f0DrP60wRu9gfrH.jpg"
+        "balance": "$1,476.39",
+        "age": 28,
+        "color": "brown",
+        "name": "Maureen Dale",
+        "gender": "female",
+        "email": "maureendale@chorizon.com",
+        "phone": "+1 (984) 538-3684",
+        "address": "817 Newton Street, Bannock, Wyoming, 1468"
       }
     ]);
 
-    let (response, _status_code) = server.search(query);
+    let (response, _status_code) = server.search_get(query).await;
     assert_json_eq!(expect, response["hits"].clone(), ordered: false);
 }
 
-#[test]
-fn search_with_settings_ranking_rules() {
-    let mut server = common::Server::with_uid("movies");
-    server.populate_movies();
+#[actix_rt::test]
+async fn search_with_settings_ranking_rules() {
+    let mut server = common::Server::test_server().await;
 
     let config = json!({
       "rankingRules": [
@@ -339,104 +257,78 @@ fn search_with_settings_ranking_rules() {
         "proximity",
         "attribute",
         "wordsPosition",
-        "asc(vote_average)",
+        "desc(age)",
         "exactness",
-        "desc(popularity)"
+        "desc(balance)"
       ],
       "distinctAttribute": null,
       "searchableAttributes": [
-        "title",
-        "tagline",
-        "overview",
-        "cast",
-        "director",
-        "producer",
-        "production_companies",
-        "genres"
+        "name",
+        "age",
+        "color",
+        "gender",
+        "email",
+        "address",
+        "about"
       ],
       "displayedAttributes": [
-        "title",
-        "director",
-        "producer",
-        "tagline",
-        "genres",
-        "id",
-        "overview",
-        "vote_count",
-        "vote_average",
-        "poster_path",
-        "popularity"
+        "name",
+        "age",
+        "gender",
+        "color",
+        "email",
+        "phone",
+        "address",
+        "balance"
       ],
       "stopWords": null,
       "synonyms": null,
-      "acceptNewFields": false,
     });
 
-    server.update_all_settings(config);
+    server.update_all_settings(config).await;
 
-    let query = "q=avangers&limit=3";
+    let query = "q=exarcitation&limit=3";
     let expect = json!([
       {
-        "id": 99861,
-        "popularity": 33.938,
-        "vote_average": 7.3,
-        "title": "Avengers: Age of Ultron",
-        "tagline": "A New Age Has Come.",
-        "overview": "When Tony Stark tries to jumpstart a dormant peacekeeping program, things go awry and Earth’s Mightiest Heroes are put to the ultimate test as the fate of the planet hangs in the balance. As the villainous Ultron emerges, it is up to The Avengers to stop him from enacting his terrible plans, and soon uneasy alliances and unexpected action pave the way for an epic and unique global adventure.",
-        "director": "Joss Whedon",
-        "producer": "Kevin Feige",
-        "genres": [
-          "Action",
-          "Adventure",
-          "Science Fiction"
-        ],
-        "poster_path": "https://image.tmdb.org/t/p/w500/t90Y3G8UGQp0f0DrP60wRu9gfrH.jpg",
-        "vote_count": 14661
+        "balance": "$1,921.58",
+        "age": 31,
+        "color": "green",
+        "name": "Harper Carson",
+        "gender": "male",
+        "email": "harpercarson@chorizon.com",
+        "phone": "+1 (912) 430-3243",
+        "address": "883 Dennett Place, Knowlton, New Mexico, 9219"
       },
       {
-        "id": 299536,
-        "popularity": 65.013,
-        "vote_average": 8.3,
-        "title": "Avengers: Infinity War",
-        "tagline": "An entire universe. Once and for all.",
-        "overview": "As the Avengers and their allies have continued to protect the world from threats too large for any one hero to handle, a new danger has emerged from the cosmic shadows: Thanos. A despot of intergalactic infamy, his goal is to collect all six Infinity Stones, artifacts of unimaginable power, and use them to inflict his twisted will on all of reality. Everything the Avengers have fought for has led up to this moment - the fate of Earth and existence itself has never been more uncertain.",
-        "director": "Anthony Russo",
-        "producer": "Kevin Feige",
-        "genres": [
-          "Adventure",
-          "Action",
-          "Science Fiction"
-        ],
-        "poster_path": "https://image.tmdb.org/t/p/w500/7WsyChQLEftFiDOVTGkv3hFpyyt.jpg",
-        "vote_count": 16056
+        "balance": "$1,706.13",
+        "age": 27,
+        "color": "green",
+        "name": "Cherry Orr",
+        "gender": "female",
+        "email": "cherryorr@chorizon.com",
+        "phone": "+1 (995) 479-3174",
+        "address": "442 Beverly Road, Ventress, New Mexico, 3361"
       },
       {
-        "id": 299534,
-        "popularity": 38.659,
-        "vote_average": 8.3,
-        "title": "Avengers: Endgame",
-        "tagline": "Part of the journey is the end.",
-        "overview": "After the devastating events of Avengers: Infinity War, the universe is in ruins due to the efforts of the Mad Titan, Thanos. With the help of remaining allies, the Avengers must assemble once more in order to undo Thanos' actions and restore order to the universe once and for all, no matter what consequences may be in store.",
-        "director": "Anthony Russo",
-        "producer": "Kevin Feige",
-        "genres": [
-          "Adventure",
-          "Science Fiction",
-          "Action"
-        ],
-        "poster_path": "https://image.tmdb.org/t/p/w500/or06FN3Dka5tukK1e9sl16pB3iy.jpg",
-        "vote_count": 10497
+        "balance": "$1,476.39",
+        "age": 28,
+        "color": "brown",
+        "name": "Maureen Dale",
+        "gender": "female",
+        "email": "maureendale@chorizon.com",
+        "phone": "+1 (984) 538-3684",
+        "address": "817 Newton Street, Bannock, Wyoming, 1468"
       }
     ]);
 
-    let (response, _status_code) = server.search(query);
+    let (response, _status_code) = server.search_get(query).await;
+    println!("{}", response["hits"].clone());
     assert_json_eq!(expect, response["hits"].clone(), ordered: false);
 }
 
-#[test]
-fn search_with_settings_searchable_attributes() {
-    let mut server = common::Server::with_uid("movies");
-    server.populate_movies();
+#[actix_rt::test]
+async fn search_with_settings_searchable_attributes() {
+    let mut server = common::Server::test_server().await;
 
     let config = json!({
       "rankingRules": [
@@ -445,103 +337,69 @@ fn search_with_settings_searchable_attributes() {
         "proximity",
         "attribute",
         "wordsPosition",
-        "desc(popularity)",
+        "desc(age)",
         "exactness",
-        "desc(vote_average)"
+        "desc(balance)"
       ],
       "distinctAttribute": null,
       "searchableAttributes": [
-        "tagline",
-        "overview",
-        "cast",
-        "director",
-        "producer",
-        "production_companies",
-        "genres"
+        "age",
+        "color",
+        "gender",
+        "address",
+        "about"
       ],
       "displayedAttributes": [
-        "title",
-        "director",
-        "producer",
-        "tagline",
-        "genres",
-        "id",
-        "overview",
-        "vote_count",
-        "vote_average",
-        "poster_path",
-        "popularity"
+        "name",
+        "age",
+        "gender",
+        "color",
+        "email",
+        "phone",
+        "address",
+        "balance"
       ],
       "stopWords": null,
-      "synonyms": null,
-      "acceptNewFields": false,
+      "synonyms": {
+          "exarcitation": [
+              "exercitation"
+          ]
+      },
     });
 
-    server.update_all_settings(config);
+    server.update_all_settings(config).await;
 
-    let query = "q=avangers&limit=3";
+    let query = "q=Carol&limit=3";
     let expect = json!([
       {
-        "id": 299536,
-        "popularity": 65.013,
-        "vote_average": 8.3,
-        "title": "Avengers: Infinity War",
-        "tagline": "An entire universe. Once and for all.",
-        "overview": "As the Avengers and their allies have continued to protect the world from threats too large for any one hero to handle, a new danger has emerged from the cosmic shadows: Thanos. A despot of intergalactic infamy, his goal is to collect all six Infinity Stones, artifacts of unimaginable power, and use them to inflict his twisted will on all of reality. Everything the Avengers have fought for has led up to this moment - the fate of Earth and existence itself has never been more uncertain.",
-        "director": "Anthony Russo",
-        "producer": "Kevin Feige",
-        "genres": [
-          "Adventure",
-          "Action",
-          "Science Fiction"
-        ],
-        "poster_path": "https://image.tmdb.org/t/p/w500/7WsyChQLEftFiDOVTGkv3hFpyyt.jpg",
-        "vote_count": 16056
+        "balance": "$1,440.09",
+        "age": 40,
+        "color": "blue",
+        "name": "Levy Whitley",
+        "gender": "male",
+        "email": "levywhitley@chorizon.com",
+        "phone": "+1 (911) 458-2411",
+        "address": "187 Thomas Street, Hachita, North Carolina, 2989"
       },
       {
-        "id": 299534,
-        "popularity": 38.659,
-        "vote_average": 8.3,
-        "title": "Avengers: Endgame",
-        "tagline": "Part of the journey is the end.",
-        "overview": "After the devastating events of Avengers: Infinity War, the universe is in ruins due to the efforts of the Mad Titan, Thanos. With the help of remaining allies, the Avengers must assemble once more in order to undo Thanos' actions and restore order to the universe once and for all, no matter what consequences may be in store.",
-        "director": "Anthony Russo",
-        "producer": "Kevin Feige",
-        "genres": [
-          "Adventure",
-          "Science Fiction",
-          "Action"
-        ],
-        "poster_path": "https://image.tmdb.org/t/p/w500/or06FN3Dka5tukK1e9sl16pB3iy.jpg",
-        "vote_count": 10497
-      },
-      {
-        "id": 100402,
-        "popularity": 16.418,
-        "vote_average": 7.7,
-        "title": "Captain America: The Winter Soldier",
-        "tagline": "In heroes we trust.",
-        "overview": "After the cataclysmic events in New York with The Avengers, Steve Rogers, aka Captain America is living quietly in Washington, D.C. and trying to adjust to the modern world. But when a S.H.I.E.L.D. colleague comes under attack, Steve becomes embroiled in a web of intrigue that threatens to put the world at risk. Joining forces with the Black Widow, Captain America struggles to expose the ever-widening conspiracy while fighting off professional assassins sent to silence him at every turn. When the full scope of the villainous plot is revealed, Captain America and the Black Widow enlist the help of a new ally, the Falcon. However, they soon find themselves up against an unexpected and formidable enemy—the Winter Soldier.",
-        "director": "Anthony Russo",
-        "producer": "Kevin Feige",
-        "genres": [
-          "Action",
-          "Adventure",
-          "Science Fiction"
-        ],
-        "poster_path": "https://image.tmdb.org/t/p/w500/5TQ6YDmymBpnF005OyoB7ohZps9.jpg",
-        "vote_count": 11972
+        "balance": "$1,977.66",
+        "age": 36,
+        "color": "brown",
+        "name": "Combs Stanley",
+        "gender": "male",
+        "email": "combsstanley@chorizon.com",
+        "phone": "+1 (827) 419-2053",
+        "address": "153 Beverley Road, Siglerville, South Carolina, 3666"
       }
     ]);
 
-    let (response, _status_code) = server.search(query);
+    let (response, _status_code) = server.search_get(query).await;
     assert_json_eq!(expect, response["hits"].clone(), ordered: false);
 }
 
-#[test]
-fn search_with_settings_displayed_attributes() {
-    let mut server = common::Server::with_uid("movies");
-    server.populate_movies();
+#[actix_rt::test]
+async fn search_with_settings_displayed_attributes() {
+    let mut server = common::Server::test_server().await;
 
     let config = json!({
       "rankingRules": [
@@ -550,68 +408,67 @@ fn search_with_settings_displayed_attributes() {
         "proximity",
         "attribute",
         "wordsPosition",
-        "desc(popularity)",
+        "desc(age)",
         "exactness",
-        "desc(vote_average)"
+        "desc(balance)"
       ],
       "distinctAttribute": null,
       "searchableAttributes": [
-        "title",
-        "tagline",
-        "overview",
-        "cast",
-        "director",
-        "producer",
-        "production_companies",
-        "genres"
+        "age",
+        "color",
+        "gender",
+        "address",
+        "about"
       ],
       "displayedAttributes": [
-        "title",
-        "tagline",
-        "id",
-        "overview",
-        "poster_path"
+        "name",
+        "age",
+        "gender",
+        "color",
+        "email",
+        "phone"
       ],
       "stopWords": null,
       "synonyms": null,
-      "acceptNewFields": false,
     });
 
-    server.update_all_settings(config);
+    server.update_all_settings(config).await;
 
-    let query = "q=avangers&limit=3";
+    let query = "q=exercitation&limit=3";
     let expect = json!([
       {
-        "id": 299536,
-        "title": "Avengers: Infinity War",
-        "tagline": "An entire universe. Once and for all.",
-        "overview": "As the Avengers and their allies have continued to protect the world from threats too large for any one hero to handle, a new danger has emerged from the cosmic shadows: Thanos. A despot of intergalactic infamy, his goal is to collect all six Infinity Stones, artifacts of unimaginable power, and use them to inflict his twisted will on all of reality. Everything the Avengers have fought for has led up to this moment - the fate of Earth and existence itself has never been more uncertain.",
-        "poster_path": "https://image.tmdb.org/t/p/w500/7WsyChQLEftFiDOVTGkv3hFpyyt.jpg"
+        "age": 31,
+        "color": "green",
+        "name": "Harper Carson",
+        "gender": "male",
+        "email": "harpercarson@chorizon.com",
+        "phone": "+1 (912) 430-3243"
       },
       {
-        "id": 299534,
-        "title": "Avengers: Endgame",
-        "tagline": "Part of the journey is the end.",
-        "overview": "After the devastating events of Avengers: Infinity War, the universe is in ruins due to the efforts of the Mad Titan, Thanos. With the help of remaining allies, the Avengers must assemble once more in order to undo Thanos' actions and restore order to the universe once and for all, no matter what consequences may be in store.",
-        "poster_path": "https://image.tmdb.org/t/p/w500/or06FN3Dka5tukK1e9sl16pB3iy.jpg"
+        "age": 27,
+        "color": "green",
+        "name": "Cherry Orr",
+        "gender": "female",
+        "email": "cherryorr@chorizon.com",
+        "phone": "+1 (995) 479-3174"
       },
       {
-        "id": 99861,
-        "title": "Avengers: Age of Ultron",
-        "tagline": "A New Age Has Come.",
-        "overview": "When Tony Stark tries to jumpstart a dormant peacekeeping program, things go awry and Earth’s Mightiest Heroes are put to the ultimate test as the fate of the planet hangs in the balance. As the villainous Ultron emerges, it is up to The Avengers to stop him from enacting his terrible plans, and soon uneasy alliances and unexpected action pave the way for an epic and unique global adventure.",
-        "poster_path": "https://image.tmdb.org/t/p/w500/t90Y3G8UGQp0f0DrP60wRu9gfrH.jpg"
+        "age": 28,
+        "color": "brown",
+        "name": "Maureen Dale",
+        "gender": "female",
+        "email": "maureendale@chorizon.com",
+        "phone": "+1 (984) 538-3684"
       }
     ]);
 
-    let (response, _status_code) = server.search(query);
+    let (response, _status_code) = server.search_get(query).await;
     assert_json_eq!(expect, response["hits"].clone(), ordered: false);
 }
 
-#[test]
-fn search_with_settings_searchable_attributes_2() {
-    let mut server = common::Server::with_uid("movies");
-    server.populate_movies();
+#[actix_rt::test]
+async fn search_with_settings_searchable_attributes_2() {
+    let mut server = common::Server::test_server().await;
 
     let config = json!({
       "rankingRules": [
@@ -620,60 +477,62 @@ fn search_with_settings_searchable_attributes_2() {
         "proximity",
         "attribute",
         "wordsPosition",
-        "desc(popularity)",
+        "desc(age)",
         "exactness",
-        "desc(vote_average)"
+        "desc(balance)"
       ],
       "distinctAttribute": null,
       "searchableAttributes": [
-        "tagline",
-        "overview",
-        "title",
-        "cast",
-        "director",
-        "producer",
-        "production_companies",
-        "genres"
+        "age",
+        "color",
+        "gender",
+        "address",
+        "about"
       ],
       "displayedAttributes": [
-        "title",
-        "tagline",
-        "id",
-        "overview",
-        "poster_path"
+        "name",
+        "age",
+        "gender"
       ],
       "stopWords": null,
       "synonyms": null,
-      "acceptNewFields": false,
     });
 
-    server.update_all_settings(config);
+    server.update_all_settings(config).await;
 
-    let query = "q=avangers&limit=3";
+    let query = "q=exercitation&limit=3";
     let expect = json!([
       {
-        "id": 299536,
-        "title": "Avengers: Infinity War",
-        "tagline": "An entire universe. Once and for all.",
-        "overview": "As the Avengers and their allies have continued to protect the world from threats too large for any one hero to handle, a new danger has emerged from the cosmic shadows: Thanos. A despot of intergalactic infamy, his goal is to collect all six Infinity Stones, artifacts of unimaginable power, and use them to inflict his twisted will on all of reality. Everything the Avengers have fought for has led up to this moment - the fate of Earth and existence itself has never been more uncertain.",
-        "poster_path": "https://image.tmdb.org/t/p/w500/7WsyChQLEftFiDOVTGkv3hFpyyt.jpg"
+        "age": 31,
+        "name": "Harper Carson",
+        "gender": "male"
       },
       {
-        "id": 299534,
-        "title": "Avengers: Endgame",
-        "tagline": "Part of the journey is the end.",
-        "overview": "After the devastating events of Avengers: Infinity War, the universe is in ruins due to the efforts of the Mad Titan, Thanos. With the help of remaining allies, the Avengers must assemble once more in order to undo Thanos' actions and restore order to the universe once and for all, no matter what consequences may be in store.",
-        "poster_path": "https://image.tmdb.org/t/p/w500/or06FN3Dka5tukK1e9sl16pB3iy.jpg"
+        "age": 27,
+        "name": "Cherry Orr",
+        "gender": "female"
       },
       {
-        "id": 100402,
-        "title": "Captain America: The Winter Soldier",
-        "tagline": "In heroes we trust.",
-        "overview": "After the cataclysmic events in New York with The Avengers, Steve Rogers, aka Captain America is living quietly in Washington, D.C. and trying to adjust to the modern world. But when a S.H.I.E.L.D. colleague comes under attack, Steve becomes embroiled in a web of intrigue that threatens to put the world at risk. Joining forces with the Black Widow, Captain America struggles to expose the ever-widening conspiracy while fighting off professional assassins sent to silence him at every turn. When the full scope of the villainous plot is revealed, Captain America and the Black Widow enlist the help of a new ally, the Falcon. However, they soon find themselves up against an unexpected and formidable enemy—the Winter Soldier.",
-        "poster_path": "https://image.tmdb.org/t/p/w500/5TQ6YDmymBpnF005OyoB7ohZps9.jpg"
+        "age": 28,
+        "name": "Maureen Dale",
+        "gender": "female"
       }
     ]);
 
-    let (response, _status_code) = server.search(query);
+    let (response, _status_code) = server.search_get(query).await;
     assert_json_eq!(expect, response["hits"].clone(), ordered: false);
 }
+
+// issue #798
+#[actix_rt::test]
+async fn distinct_attributes_returns_name_not_id() {
+    let mut server = common::Server::test_server().await;
+    let settings = json!({
+        "distinctAttribute": "color",
+    });
+    server.update_all_settings(settings).await;
+    let (response, _) = server.get_all_settings().await;
+    assert_eq!(response["distinctAttribute"], "color");
+    let (response, _) = server.get_distinct_attribute().await;
+    assert_eq!(response, "color");
+}

meilisearch-http/tests/settings.rs

@@ -1,14 +1,11 @@
 use assert_json_diff::assert_json_eq;
 use serde_json::json;
 use std::convert::Into;
-
 mod common;
 
-#[test]
-fn write_all_and_delete() {
-    let mut server = common::Server::with_uid("movies");
-    server.populate_movies();
-
+#[actix_rt::test]
+async fn write_all_and_delete() {
+    let mut server = common::Server::test_server().await;
     // 2 - Send the settings
 
     let body = json!({
@@ -19,53 +16,55 @@ fn write_all_and_delete() {
             "attribute",
             "wordsPosition",
             "exactness",
-            "desc(release_date)",
-            "desc(rank)",
+            "desc(registered)",
+            "desc(age)",
         ],
-        "distinctAttribute": "movie_id",
+        "distinctAttribute": "id",
         "searchableAttributes": [
             "id",
-            "movie_id",
-            "title",
-            "description",
-            "poster",
-            "release_date",
-            "rank",
+            "name",
+            "color",
+            "gender",
+            "email",
+            "phone",
+            "address",
+            "registered",
+            "about"
         ],
         "displayedAttributes": [
-            "title",
-            "description",
-            "poster",
-            "release_date",
-            "rank",
+            "name",
+            "gender",
+            "email",
+            "registered",
+            "age",
         ],
         "stopWords": [
-            "the",
-            "a",
-            "an",
+            "ad",
+            "in",
+            "ut",
         ],
         "synonyms": {
-            "wolverine": ["xmen", "logan"],
-            "logan": ["wolverine"],
+            "road": ["street", "avenue"],
+            "street": ["avenue"],
         },
-        "acceptNewFields": false,
+        "attributesForFaceting": ["name"],
     });
 
-    server.update_all_settings(body.clone());
+    server.update_all_settings(body.clone()).await;
 
     // 3 - Get all settings and compare to the previous one
 
-    let (response, _status_code) = server.get_all_settings();
+    let (response, _status_code) = server.get_all_settings().await;
 
     assert_json_eq!(body, response, ordered: false);
 
     // 4 - Delete all settings
 
-    server.delete_all_settings();
+    server.delete_all_settings().await;
 
     // 5 - Get all settings and check if they are set to default values
 
-    let (response, _status_code) = server.get_all_settings();
+    let (response, _status_code) = server.get_all_settings().await;
 
     let expect = json!({
         "rankingRules": [
@@ -77,58 +76,19 @@ fn write_all_and_delete() {
           "exactness"
         ],
         "distinctAttribute": null,
-        "searchableAttributes": [
-          "poster_path",
-          "director",
-          "id",
-          "production_companies",
-          "producer",
-          "poster",
-          "movie_id",
-          "vote_count",
-          "cast",
-          "release_date",
-          "vote_average",
-          "rank",
-          "genres",
-          "overview",
-          "description",
-          "tagline",
-          "popularity",
-          "title"
-        ],
-        "displayedAttributes": [
-          "poster_path",
-          "poster",
-          "vote_count",
-          "id",
-          "movie_id",
-          "title",
-          "rank",
-          "tagline",
-          "cast",
-          "producer",
-          "production_companies",
-          "description",
-          "director",
-          "genres",
-          "release_date",
-          "overview",
-          "vote_average",
-          "popularity"
-        ],
+        "searchableAttributes": ["*"],
+        "displayedAttributes": ["*"],
         "stopWords": [],
         "synonyms": {},
-        "acceptNewFields": true,
+        "attributesForFaceting": [],
     });
 
     assert_json_eq!(expect, response, ordered: false);
 }
 
-#[test]
-fn write_all_and_update() {
-    let mut server = common::Server::with_uid("movies");
-    server.populate_movies();
+#[actix_rt::test]
+async fn write_all_and_update() {
+    let mut server = common::Server::test_server().await;
 
     // 2 - Send the settings
 
@@ -140,43 +100,45 @@ fn write_all_and_update() {
             "attribute",
             "wordsPosition",
             "exactness",
-            "desc(release_date)",
-            "desc(rank)",
+            "desc(registered)",
+            "desc(age)",
         ],
-        "distinctAttribute": "movie_id",
+        "distinctAttribute": "id",
         "searchableAttributes": [
-            "uid",
-            "movie_id",
-            "title",
-            "description",
-            "poster",
-            "release_date",
-            "rank",
+            "id",
+            "name",
+            "color",
+            "gender",
+            "email",
+            "phone",
+            "address",
+            "registered",
+            "about"
         ],
         "displayedAttributes": [
-            "title",
-            "description",
-            "poster",
-            "release_date",
-            "rank",
+            "name",
+            "gender",
+            "email",
+            "registered",
+            "age",
         ],
         "stopWords": [
-            "the",
-            "a",
-            "an",
+            "ad",
+            "in",
+            "ut",
         ],
         "synonyms": {
-            "wolverine": ["xmen", "logan"],
-            "logan": ["wolverine"],
+            "road": ["street", "avenue"],
+            "street": ["avenue"],
         },
-        "acceptNewFields": false,
+        "attributesForFaceting": ["name"],
     });
 
-    server.update_all_settings(body.clone());
+    server.update_all_settings(body.clone()).await;
 
     // 3 - Get all settings and compare to the previous one
 
-    let (response, _status_code) = server.get_all_settings();
+    let (response, _status_code) = server.get_all_settings().await;
 
     assert_json_eq!(body, response, ordered: false);
 
@@ -190,34 +152,34 @@ fn write_all_and_update() {
             "attribute",
             "wordsPosition",
             "exactness",
-            "desc(release_date)",
+            "desc(age)",
         ],
         "distinctAttribute": null,
         "searchableAttributes": [
-            "title",
-            "description",
-            "uid",
+            "name",
+            "color",
+            "age",
         ],
         "displayedAttributes": [
-            "title",
-            "description",
-            "release_date",
-            "rank",
-            "poster",
+            "name",
+            "color",
+            "age",
+            "registered",
+            "picture",
         ],
         "stopWords": [],
         "synonyms": {
-            "wolverine": ["xmen", "logan"],
-            "logan": ["wolverine", "xmen"],
+            "road": ["street", "avenue"],
+            "street": ["avenue"],
         },
-        "acceptNewFields": false,
+        "attributesForFaceting": ["title"],
     });
 
-    server.update_all_settings(body);
+    server.update_all_settings(body).await;
 
     // 5 - Get all settings and check if the content is the same of (4)
 
-    let (response, _status_code) = server.get_all_settings();
+    let (response, _status_code) = server.get_all_settings().await;
 
     let expected = json!({
         "rankingRules": [
@@ -227,39 +189,39 @@ fn write_all_and_update() {
             "attribute",
             "wordsPosition",
             "exactness",
-            "desc(release_date)",
+            "desc(age)",
         ],
         "distinctAttribute": null,
         "searchableAttributes": [
-            "title",
-            "description",
-            "uid",
+            "name",
+            "color",
+            "age",
         ],
         "displayedAttributes": [
-            "title",
-            "description",
-            "release_date",
-            "rank",
-            "poster",
+            "name",
+            "color",
+            "age",
+            "registered",
+            "picture",
         ],
         "stopWords": [],
         "synonyms": {
-            "wolverine": ["xmen", "logan"],
-            "logan": ["wolverine", "xmen"],
+            "road": ["street", "avenue"],
+            "street": ["avenue"],
         },
-        "acceptNewFields": false
+        "attributesForFaceting": ["title"],
     });
 
     assert_json_eq!(expected, response, ordered: false);
 }
 
-#[test]
-fn test_default_settings() {
-    let mut server = common::Server::with_uid("movies");
+#[actix_rt::test]
+async fn test_default_settings() {
+    let mut server = common::Server::with_uid("test");
     let body = json!({
-        "uid": "movies",
+        "uid": "test",
     });
-    server.create_index(body);
+    server.create_index(body).await;
 
     // 1 - Get all settings and compare to the previous one
 
@@ -273,26 +235,26 @@ fn test_default_settings() {
             "exactness"
         ],
         "distinctAttribute": null,
-        "searchableAttributes": [],
-        "displayedAttributes": [],
+        "searchableAttributes": ["*"],
+        "displayedAttributes": ["*"],
         "stopWords": [],
         "synonyms": {},
-        "acceptNewFields": true,
+        "attributesForFaceting": [],
     });
 
-    let (response, _status_code) = server.get_all_settings();
+    let (response, _status_code) = server.get_all_settings().await;
 
     assert_json_eq!(body, response, ordered: false);
 }
 
-#[test]
-fn test_default_settings_2() {
-    let mut server = common::Server::with_uid("movies");
+#[actix_rt::test]
+async fn test_default_settings_2() {
+    let mut server = common::Server::with_uid("test");
     let body = json!({
-        "uid": "movies",
+        "uid": "test",
         "primaryKey": "id",
     });
-    server.create_index(body);
+    server.create_index(body).await;
 
     // 1 - Get all settings and compare to the previous one
 
@@ -306,53 +268,50 @@ fn test_default_settings_2() {
             "exactness"
         ],
         "distinctAttribute": null,
-        "searchableAttributes": [
-            "id"
-        ],
-        "displayedAttributes": [
-            "id"
-        ],
+        "searchableAttributes": ["*"],
+        "displayedAttributes": ["*"],
         "stopWords": [],
         "synonyms": {},
-        "acceptNewFields": true,
+        "attributesForFaceting": [],
     });
 
-    let (response, _status_code) = server.get_all_settings();
+    let (response, _status_code) = server.get_all_settings().await;
 
     assert_json_eq!(body, response, ordered: false);
 }
 
 // Test issue https://github.com/meilisearch/MeiliSearch/issues/516
-#[test]
-fn write_setting_and_update_partial() {
-    let mut server = common::Server::with_uid("movies");
+#[actix_rt::test]
+async fn write_setting_and_update_partial() {
+    let mut server = common::Server::with_uid("test");
     let body = json!({
-        "uid": "movies",
+        "uid": "test",
     });
-    server.create_index(body);
+    server.create_index(body).await;
 
     // 2 - Send the settings
 
     let body = json!({
         "searchableAttributes": [
-            "uid",
-            "movie_id",
-            "title",
-            "description",
-            "poster",
-            "release_date",
-            "rank",
+            "id",
+            "name",
+            "color",
+            "gender",
+            "email",
+            "phone",
+            "address",
+            "about"
         ],
         "displayedAttributes": [
-            "title",
-            "description",
-            "poster",
-            "release_date",
-            "rank",
+            "name",
+            "gender",
+            "email",
+            "registered",
+            "age",
         ]
     });
 
-    server.update_all_settings(body.clone());
+    server.update_all_settings(body.clone()).await;
 
     // 2 - Send the settings
 
@@ -364,23 +323,22 @@ fn write_setting_and_update_partial() {
             "attribute",
             "wordsPosition",
             "exactness",
-            "desc(release_date)",
-            "desc(rank)",
+            "desc(age)",
+            "desc(registered)",
         ],
-        "distinctAttribute": "movie_id",
+        "distinctAttribute": "id",
         "stopWords": [
-            "the",
-            "a",
-            "an",
+            "ad",
+            "in",
+            "ut",
         ],
         "synonyms": {
-            "wolverine": ["xmen", "logan"],
-            "logan": ["wolverine"],
+            "road": ["street", "avenue"],
+            "street": ["avenue"],
         },
-        "acceptNewFields": false,
     });
 
-    server.update_all_settings(body.clone());
+    server.update_all_settings(body.clone()).await;
 
     // 2 - Send the settings
 
@@ -392,39 +350,121 @@ fn write_setting_and_update_partial() {
             "attribute",
             "wordsPosition",
             "exactness",
-            "desc(release_date)",
-            "desc(rank)",
+            "desc(age)",
+            "desc(registered)",
         ],
-        "distinctAttribute": "movie_id",
+        "distinctAttribute": "id",
         "searchableAttributes": [
-            "uid",
-            "movie_id",
-            "title",
-            "description",
-            "poster",
-            "release_date",
-            "rank",
+            "id",
+            "name",
+            "color",
+            "gender",
+            "email",
+            "phone",
+            "address",
+            "about"
         ],
         "displayedAttributes": [
-            "title",
-            "description",
-            "poster",
-            "release_date",
-            "rank",
+            "name",
+            "gender",
+            "email",
+            "registered",
+            "age",
         ],
         "stopWords": [
-            "the",
-            "a",
-            "an",
+            "ad",
+            "in",
+            "ut",
         ],
         "synonyms": {
-            "wolverine": ["xmen", "logan"],
-            "logan": ["wolverine"],
+            "road": ["street", "avenue"],
+            "street": ["avenue"],
         },
-        "acceptNewFields": false,
+        "attributesForFaceting": [],
     });
 
-    let (response, _status_code) = server.get_all_settings();
+    let (response, _status_code) = server.get_all_settings().await;
 
     assert_json_eq!(expected, response, ordered: false);
 }
+
+#[actix_rt::test]
+async fn attributes_for_faceting_settings() {
+    let mut server = common::Server::test_server().await;
+    // initial attributes array should be empty
+    let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await;
+    assert_eq!(response, json!([]));
+    // add an attribute and test for its presence
+    let (_response, _status_code) = server.post_request_async(
+        "/indexes/test/settings/attributes-for-faceting",
+        json!(["foobar"])).await;
+    let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await;
+    assert_eq!(response, json!(["foobar"]));
+    // remove all attributes and test for emptiness
+    let (_response, _status_code) = server.delete_request_async(
+        "/indexes/test/settings/attributes-for-faceting").await;
+    let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await;
+    assert_eq!(response, json!([]));
+}
+
+#[actix_rt::test]
+async fn setting_ranking_rules_dont_mess_with_other_settings() {
+    let mut server = common::Server::test_server().await;
+    let body = json!({
+        "rankingRules": ["asc(foobar)"]
+    });
+    server.update_all_settings(body).await;
+    let (response, _) = server.get_all_settings().await;
+    assert_eq!(response["rankingRules"].as_array().unwrap().len(), 1);
+    assert_eq!(response["rankingRules"].as_array().unwrap().first().unwrap().as_str().unwrap(), "asc(foobar)");
+    assert!(!response["searchableAttributes"].as_array().unwrap().iter().any(|e| e.as_str().unwrap() == "foobar"));
+    assert!(!response["displayedAttributes"].as_array().unwrap().iter().any(|e| e.as_str().unwrap() == "foobar"));
+}
+
+#[actix_rt::test]
+async fn displayed_and_searchable_attributes_reset_to_wildcard() {
+    let mut server = common::Server::test_server().await;
+    server.update_all_settings(json!({ "searchableAttributes": ["color"], "displayedAttributes": ["color"] })).await;
+    let (response, _) = server.get_all_settings().await;
+
+    assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "color");
+    assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "color");
+
+    server.delete_searchable_attributes().await;
+    server.delete_displayed_attributes().await;
+
+    let (response, _) = server.get_all_settings().await;
+
+    assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1);
+    assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1);
+    assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*");
+    assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*");
+
+    let mut server = common::Server::test_server().await;
+    server.update_all_settings(json!({ "searchableAttributes": ["color"], "displayedAttributes": ["color"] })).await;
+    let (response, _) = server.get_all_settings().await;
+    assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "color");
+    assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "color");
+
+    server.update_all_settings(json!({ "searchableAttributes": [], "displayedAttributes": [] })).await;
+
+    let (response, _) = server.get_all_settings().await;
+
+    assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1);
+    assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1);
+    assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*");
+    assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*");
+}
+
+#[actix_rt::test]
+async fn settings_that_contains_wildcard_is_wildcard() {
+    let mut server = common::Server::test_server().await;
+    server.update_all_settings(json!({ "searchableAttributes": ["color", "*"], "displayedAttributes": ["color", "*"] })).await;
+
+    let (response, _) = server.get_all_settings().await;
+
+    assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1);
+    assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1);
+    assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*");
+    assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*");
+}

meilisearch-http/tests/settings_accept_new_fields.rs

@@ -1,349 +0,0 @@
-use assert_json_diff::assert_json_eq;
-use serde_json::json;
-
-mod common;
-
-#[test]
-fn index_new_fields_default() {
-    let mut server = common::Server::with_uid("movies");
-    let body = json!({
-        "uid": "movies",
-        "primaryKey": "id",
-    });
-    server.create_index(body);
-
-    // 1 - Add a document
-
-    let body = json!([{
-        "id": 1,
-        "title": "I'm a legend",
-    }]);
-
-    server.add_or_replace_multiple_documents(body);
-
-    // 2 - Get the complete document
-
-    let expected = json!({
-        "id": 1,
-        "title": "I'm a legend",
-    });
-
-    let (response, status_code) = server.get_document(1);
-    assert_eq!(status_code, 200);
-    assert_json_eq!(response, expected);
-
-    // 3 - Add a document with more fields
-
-    let body = json!([{
-        "id": 2,
-        "title": "I'm not a legend",
-        "description": "A bad copy of the original movie I'm a lengend"
-    }]);
-
-    server.add_or_replace_multiple_documents(body);
-
-    // 4 - Get the complete document
-
-    let expected = json!({
-        "id": 2,
-        "title": "I'm not a legend",
-        "description": "A bad copy of the original movie I'm a lengend"
-    });
-
-    let (response, status_code) = server.get_document(2);
-    assert_eq!(status_code, 200);
-    assert_json_eq!(response, expected);
-}
-
-#[test]
-fn index_new_fields_true() {
-    let mut server = common::Server::with_uid("movies");
-    let body = json!({
-        "uid": "movies",
-        "primaryKey": "id",
-    });
-    server.create_index(body);
-
-    // 1 - Set indexNewFields = true
-
-    server.update_accept_new_fields(json!(true));
-
-    // 2 - Add a document
-
-    let body = json!([{
-        "id": 1,
-        "title": "I'm a legend",
-    }]);
-
-    server.add_or_replace_multiple_documents(body);
-
-    // 3 - Get the complete document
-
-    let expected = json!({
-        "id": 1,
-        "title": "I'm a legend",
-    });
-
-    let (response, status_code) = server.get_document(1);
-    assert_eq!(status_code, 200);
-    assert_json_eq!(response, expected);
-
-    // 4 - Add a document with more fields
-
-    let body = json!([{
-        "id": 2,
-        "title": "I'm not a legend",
-        "description": "A bad copy of the original movie I'm a lengend"
-    }]);
-
-    server.add_or_replace_multiple_documents(body);
-
-    // 5 - Get the complete document
-
-    let expected = json!({
-        "id": 2,
-        "title": "I'm not a legend",
-        "description": "A bad copy of the original movie I'm a lengend"
-    });
-
-    let (response, status_code) = server.get_document(2);
-    assert_eq!(status_code, 200);
-    assert_json_eq!(response, expected);
-}
-
-#[test]
-fn index_new_fields_false() {
-    let mut server = common::Server::with_uid("movies");
-    let body = json!({
-        "uid": "movies",
-        "primaryKey": "id",
-    });
-    server.create_index(body);
-
-    // 1 - Set indexNewFields = false
-
-    server.update_accept_new_fields(json!(false));
-
-    // 2 - Add a document
-
-    let body = json!([{
-        "id": 1,
-        "title": "I'm a legend",
-    }]);
-
-    server.add_or_replace_multiple_documents(body);
-
-    // 3 - Get the complete document
-
-    let expected = json!({
-        "id": 1,
-    });
-
-    let (response, status_code) = server.get_document(1);
-    assert_eq!(status_code, 200);
-    assert_json_eq!(response, expected);
-
-    // 4 - Add a document with more fields
-
-    let body = json!([{
-        "id": 2,
-        "title": "I'm not a legend",
-        "description": "A bad copy of the original movie I'm a lengend"
-    }]);
-
-    server.add_or_replace_multiple_documents(body);
-
-    // 5 - Get the complete document
-
-    let expected = json!({
-        "id": 2,
-    });
-
-    let (response, status_code) = server.get_document(2);
-    assert_eq!(status_code, 200);
-    assert_json_eq!(response, expected);
-}
-
-#[test]
-fn index_new_fields_true_then_false() {
-    let mut server = common::Server::with_uid("movies");
-    let body = json!({
-        "uid": "movies",
-        "primaryKey": "id",
-    });
-    server.create_index(body);
-
-    // 1 - Set indexNewFields = true
-
-    server.update_accept_new_fields(json!(true));
-
-    // 2 - Add a document
-
-    let body = json!([{
-        "id": 1,
-        "title": "I'm a legend",
-    }]);
-
-    server.add_or_replace_multiple_documents(body);
-
-    // 3 - Get the complete document
-
-    let expected = json!({
-        "id": 1,
-        "title": "I'm a legend",
-    });
-
-    let (response, status_code) = server.get_document(1);
-    assert_eq!(status_code, 200);
-    assert_json_eq!(response, expected);
-
-    // 4 - Set indexNewFields = false
-
-    server.update_accept_new_fields(json!(false));
-
-    // 5 - Add a document with more fields
-
-    let body = json!([{
-        "id": 2,
-        "title": "I'm not a legend",
-        "description": "A bad copy of the original movie I'm a lengend"
-    }]);
-
-    server.add_or_replace_multiple_documents(body);
-
-    // 6 - Get the complete document
-
-    let expected = json!({
-        "id": 2,
-        "title": "I'm not a legend",
-    });
-
-    let (response, status_code) = server.get_document(2);
-    assert_eq!(status_code, 200);
-    assert_json_eq!(response, expected);
-}
-
-#[test]
-fn index_new_fields_false_then_true() {
-    let mut server = common::Server::with_uid("movies");
-    let body = json!({
-        "uid": "movies",
-        "primaryKey": "id",
-    });
-    server.create_index(body);
-
-    // 1 - Set indexNewFields = false
-
-    server.update_accept_new_fields(json!(false));
-
-    // 2 - Add a document
-
-    let body = json!([{
-        "id": 1,
-        "title": "I'm a legend",
-    }]);
-
-    server.add_or_replace_multiple_documents(body);
-
-    // 3 - Get the complete document
-
-    let expected = json!({
-        "id": 1,
-    });
-
-    let (response, status_code) = server.get_document(1);
-    assert_eq!(status_code, 200);
-    assert_json_eq!(response, expected);
-
-    // 4 - Set indexNewFields = false
-
-    server.update_accept_new_fields(json!(true));
-
-    // 5 - Add a document with more fields
-
-    let body = json!([{
-        "id": 2,
-        "title": "I'm not a legend",
-        "description": "A bad copy of the original movie I'm a lengend"
-    }]);
-
-    server.add_or_replace_multiple_documents(body);
-
-    // 6 - Get the complete document
-
-    let expected = json!({
-        "id": 1,
-    });
-
-    let (response, status_code) = server.get_document(1);
-    assert_eq!(status_code, 200);
-    assert_json_eq!(response, expected);
-
-    let expected = json!({
-        "id": 2,
-        "description": "A bad copy of the original movie I'm a lengend"
-    });
-
-    let (response, status_code) = server.get_document(2);
-    assert_eq!(status_code, 200);
-    assert_json_eq!(response, expected);
-}
-
-
-// Fix issue https://github.com/meilisearch/MeiliSearch/issues/518
-#[test]
-fn accept_new_fields_does_not_take_into_account_the_primary_key () {
-    let mut server = common::Server::with_uid("movies");
-
-    // 1 - Create an index with no primary-key
-
-    let body = json!({
-        "uid": "movies",
-    });
-    let (response, status_code) = server.create_index(body);
-    assert_eq!(status_code, 201);
-    assert_eq!(response["primaryKey"], json!(null));
-
-    // 2 - Add searchable and displayed attributes as: ["title"] & Set acceptNewFields to false
-
-    let body = json!({
-        "searchableAttributes": ["title"],
-        "displayedAttributes": ["title"],
-        "acceptNewFields": false,
-    });
-
-    server.update_all_settings(body);
-
-    // 4 - Add a document
-
-    let body = json!([{
-      "id": 1,
-      "title": "Test",
-      "comment": "comment test"
-    }]);
-
-    server.add_or_replace_multiple_documents(body);
-
-    // 5 - Get settings, they should not changed
-
-    let (response, _status_code) = server.get_all_settings();
-
-    let expected = json!({
-        "rankingRules": [
-            "typo",
-            "words",
-            "proximity",
-            "attribute",
-            "wordsPosition",
-            "exactness",
-        ],
-        "distinctAttribute": null,
-        "searchableAttributes": ["title"],
-        "displayedAttributes": ["title"],
-        "stopWords": [],
-        "synonyms": {},
-        "acceptNewFields": false,
-    });
-
-    assert_json_eq!(response, expected, ordered: false);
-}

meilisearch-http/tests/settings_ranking_rules.rs

@@ -3,10 +3,9 @@ use serde_json::json;
 
 mod common;
 
-#[test]
-fn write_all_and_delete() {
-    let mut server = common::Server::with_uid("movies");
-    server.populate_movies();
+#[actix_rt::test]
+async fn write_all_and_delete() {
+    let mut server = common::Server::test_server().await;
 
     // 2 - Send the settings
 
@@ -17,25 +16,25 @@ fn write_all_and_delete() {
         "attribute",
         "wordsPosition",
         "exactness",
-        "desc(release_date)",
-        "desc(rank)",
+        "desc(registered)",
+        "desc(age)",
     ]);
 
-    server.update_ranking_rules(body.clone());
+    server.update_ranking_rules(body.clone()).await;
 
     // 3 - Get all settings and compare to the previous one
 
-    let (response, _status_code) = server.get_ranking_rules();
+    let (response, _status_code) = server.get_ranking_rules().await;
 
     assert_json_eq!(body, response, ordered: false);
 
     // 4 - Delete all settings
 
-    server.delete_ranking_rules();
+    server.delete_ranking_rules().await;
 
     // 5 - Get all settings and check if they are empty
 
-    let (response, _status_code) = server.get_ranking_rules();
+    let (response, _status_code) = server.get_ranking_rules().await;
 
     let expected = json!([
         "typo",
@@ -49,10 +48,9 @@ fn write_all_and_delete() {
     assert_json_eq!(expected, response, ordered: false);
 }
 
-#[test]
-fn write_all_and_update() {
-    let mut server = common::Server::with_uid("movies");
-    server.populate_movies();
+#[actix_rt::test]
+async fn write_all_and_update() {
+    let mut server = common::Server::test_server().await;
 
     // 2 - Send the settings
 
@@ -63,15 +61,15 @@ fn write_all_and_update() {
         "attribute",
         "wordsPosition",
         "exactness",
-        "desc(release_date)",
-        "desc(rank)",
+        "desc(registered)",
+        "desc(age)",
     ]);
 
-    server.update_ranking_rules(body.clone());
+    server.update_ranking_rules(body.clone()).await;
 
     // 3 - Get all settings and compare to the previous one
 
-    let (response, _status_code) = server.get_ranking_rules();
+    let (response, _status_code) = server.get_ranking_rules().await;
 
     assert_json_eq!(body, response, ordered: false);
 
@@ -84,14 +82,14 @@ fn write_all_and_update() {
         "attribute",
         "wordsPosition",
         "exactness",
-        "desc(release_date)",
+        "desc(registered)",
     ]);
 
-    server.update_ranking_rules(body);
+    server.update_ranking_rules(body).await;
 
     // 5 - Get all settings and check if the content is the same of (4)
 
-    let (response, _status_code) = server.get_ranking_rules();
+    let (response, _status_code) = server.get_ranking_rules().await;
 
     let expected = json!([
         "typo",
@@ -100,89 +98,85 @@ fn write_all_and_update() {
         "attribute",
         "wordsPosition",
         "exactness",
-        "desc(release_date)",
+        "desc(registered)",
     ]);
 
     assert_json_eq!(expected, response, ordered: false);
 }
 
-#[test]
-fn send_undefined_rule() {
-    let mut server = common::Server::with_uid("movies");
+#[actix_rt::test]
+async fn send_undefined_rule() {
+    let mut server = common::Server::with_uid("test");
     let body = json!({
-        "uid": "movies",
+        "uid": "test",
         "primaryKey": "id",
     });
-    server.create_index(body);
+    server.create_index(body).await;
 
     let body = json!(["typos",]);
 
-    let (_response, status_code) = server.update_ranking_rules_sync(body);
+    let (_response, status_code) = server.update_ranking_rules_sync(body).await;
     assert_eq!(status_code, 400);
 }
 
-#[test]
-fn send_malformed_custom_rule() {
-    let mut server = common::Server::with_uid("movies");
+#[actix_rt::test]
+async fn send_malformed_custom_rule() {
+    let mut server = common::Server::with_uid("test");
     let body = json!({
-        "uid": "movies",
+        "uid": "test",
         "primaryKey": "id",
     });
-    server.create_index(body);
+    server.create_index(body).await;
 
     let body = json!(["dsc(truc)",]);
 
-    let (_response, status_code) = server.update_ranking_rules_sync(body);
+    let (_response, status_code) = server.update_ranking_rules_sync(body).await;
     assert_eq!(status_code, 400);
 }
 
 // Test issue https://github.com/meilisearch/MeiliSearch/issues/521
-#[test]
-fn write_custom_ranking_and_index_documents() {
-    let mut server = common::Server::with_uid("movies");
+#[actix_rt::test]
+async fn write_custom_ranking_and_index_documents() {
+    let mut server = common::Server::with_uid("test");
     let body = json!({
-        "uid": "movies",
+        "uid": "test",
         "primaryKey": "id",
     });
-    server.create_index(body);
+    server.create_index(body).await;
 
     // 1 - Add ranking rules with one custom ranking on a string
 
-    let body = json!([
-        "asc(title)",
-        "typo"
-    ]);
+    let body = json!(["asc(name)", "typo"]);
 
-    server.update_ranking_rules(body);
+    server.update_ranking_rules(body).await;
 
     // 2 - Add documents
 
     let body = json!([
       {
         "id": 1,
-        "title": "Le Petit Prince",
-        "author": "Exupéry"
+        "name": "Cherry Orr",
+        "color": "green"
       },
       {
         "id": 2,
-        "title": "Pride and Prejudice",
-        "author": "Jane Austen"
+        "name": "Lucas Hess",
+        "color": "yellow"
       }
     ]);
 
-    server.add_or_replace_multiple_documents(body);
+    server.add_or_replace_multiple_documents(body).await;
 
     // 3 - Get the first document and compare
 
     let expected = json!({
         "id": 1,
-        "title": "Le Petit Prince",
-        "author": "Exupéry"
+        "name": "Cherry Orr",
+        "color": "green"
     });
 
-    let (response, status_code) = server.get_document(1);
+    let (response, status_code) = server.get_document(1).await;
     assert_eq!(status_code, 200);
 
     assert_json_eq!(response, expected, ordered: false);
-
 }

meilisearch-http/tests/settings_stop_words.rs

@@ -3,54 +3,52 @@ use serde_json::json;
 
 mod common;
 
-#[test]
-fn update_stop_words() {
-    let mut server = common::Server::with_uid("movies");
-    server.populate_movies();
+#[actix_rt::test]
+async fn update_stop_words() {
+    let mut server = common::Server::test_server().await;
 
     // 1 - Get stop words
 
-    let (response, _status_code) = server.get_stop_words();
+    let (response, _status_code) = server.get_stop_words().await;
     assert_eq!(response.as_array().unwrap().is_empty(), true);
 
     // 2 - Update stop words
 
-    let body = json!(["the", "a"]);
-    server.update_stop_words(body.clone());
+    let body = json!(["ut", "ea"]);
+    server.update_stop_words(body.clone()).await;
 
     // 3 - Get all stop words and compare to the previous one
 
-    let (response, _status_code) = server.get_stop_words();
+    let (response, _status_code) = server.get_stop_words().await;
     assert_json_eq!(body, response, ordered: false);
 
     // 4 - Delete all stop words
 
-    server.delete_stop_words();
+    server.delete_stop_words().await;
 
     // 5 - Get all stop words and check if they are empty
 
-    let (response, _status_code) = server.get_stop_words();
+    let (response, _status_code) = server.get_stop_words().await;
     assert_eq!(response.as_array().unwrap().is_empty(), true);
 }
 
-#[test]
-fn add_documents_and_stop_words() {
-    let mut server = common::Server::with_uid("movies");
-    server.populate_movies();
+#[actix_rt::test]
+async fn add_documents_and_stop_words() {
+    let mut server = common::Server::test_server().await;
 
     // 2 - Update stop words
 
-    let body = json!(["the", "of"]);
-    server.update_stop_words(body.clone());
+    let body = json!(["ad", "in"]);
+    server.update_stop_words(body.clone()).await;
 
     // 3 - Search for a document with stop words
 
-    let (response, _status_code) = server.search("q=the%20mask");
+    let (response, _status_code) = server.search_get("q=in%20exercitation").await;
     assert!(!response["hits"].as_array().unwrap().is_empty());
 
     // 4 - Search for documents with *only* stop words
 
-    let (response, _status_code) = server.search("q=the%20of");
+    let (response, _status_code) = server.search_get("q=ad%20in").await;
     assert!(response["hits"].as_array().unwrap().is_empty());
 
     // 5 - Delete all stop words