Merge #1218

1218: bump meilisearch version 0.19.0 r=LegendreM a=MarinPostma



Co-authored-by: mpostma <postma.marin@protonmail.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/tracking-issue.md

@@ -0,0 +1,40 @@
+---
+name: Tracking issue
+about: Template for a tracking issue
+title: ''
+labels: tracking-issue
+assignees: ''
+
+---
+
+# Summary
+
+One paragraph to explain the feature.
+
+# Motivations
+
+Why are we doing this? What use cases does it support? What is the expected outcome?
+
+# Explanation
+
+Explain the proposal like it was the final documentation of this proposal.
+
+- What is changing for end-users.
+- How it works.
+- What is breaking?
+- Examples.
+
+# Implementation
+
+Explain the technical specificities that will need to be known or done in order to implement this proposal.
+
+## Steps
+
+Describe each step to create the feature with it's associated issue/PR.
+
+# Related
+
+- [ ] Validated by the team (@people needed)
+- [ ] Test added
+- [ ] [Documentation](https://github.com/meilisearch/documentation/issues/#xxx) //Change xxx or remove the line
+- [ ] [SDK/Integrations](https://github.com/meilisearch/integration-guides/issues/#xxx) //Change xxx or remove the line

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "cargo"
+    directory: "/"
+    schedule:
+      interval: "monthly"

.github/is-latest-release.sh

@@ -0,0 +1,132 @@
+#!/bin/sh
+
+# Checks if the current tag should be the latest (in terms of semver and not of release date).
+# Ex: previous tag -> v0.10.1
+#     new tag -> v0.8.12
+#     The new tag should not be the latest
+#     So it returns "false", the CI should not run for the release v0.8.2
+
+# Used in GHA in publish-docker-latest.yml
+# Returns "true" or "false" (as a string) to be used in the `if` in GHA
+
+# GLOBAL
+GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number]
+
+# FUNCTIONS
+
+# semverParseInto and semverLT from https://github.com/cloudflare/semver_bash/blob/master/semver.sh
+
+# usage: semverParseInto version major minor patch special
+# version: the string version
+# major, minor, patch, special: will be assigned by the function
+semverParseInto() {
+    local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)'
+    #MAJOR
+    eval $2=`echo $1 | sed -e "s#$RE#\1#"`
+    #MINOR
+    eval $3=`echo $1 | sed -e "s#$RE#\2#"`
+    #MINOR
+    eval $4=`echo $1 | sed -e "s#$RE#\3#"`
+    #SPECIAL
+    eval $5=`echo $1 | sed -e "s#$RE#\4#"`
+}
+
+# usage: semverLT version1 version2
+semverLT() {
+    local MAJOR_A=0
+    local MINOR_A=0
+    local PATCH_A=0
+    local SPECIAL_A=0
+
+    local MAJOR_B=0
+    local MINOR_B=0
+    local PATCH_B=0
+    local SPECIAL_B=0
+
+    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
+    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
+
+    if [ $MAJOR_A -lt $MAJOR_B ]; then
+        return 0
+    fi
+    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -lt $MINOR_B ]; then
+        return 0
+    fi
+    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then
+        return 0
+    fi
+    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+        return 1
+    fi
+    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  != "_" ] ; then
+        return 1
+    fi
+    if [ "_$SPECIAL_A"  != "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+        return 0
+    fi
+    if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then
+        return 0
+    fi
+
+    return 1
+}
+
+# Returns the tag of the latest stable release (in terms of semver and not of release date)
+get_latest() {
+    temp_file='temp_file' # temp_file needed because the grep would start before the download is over
+    curl -s 'https://api.github.com/repos/meilisearch/MeiliSearch/releases' > "$temp_file"
+    releases=$(cat "$temp_file" | \
+        grep -E "tag_name|draft|prerelease" \
+        | tr -d ',"' | cut -d ':' -f2 | tr -d ' ')
+        # Returns a list of [tag_name draft_boolean prerelease_boolean ...]
+        # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false...
+
+    i=0
+    latest=""
+    current_tag=""
+    for release_info in $releases; do
+        if [ $i -eq 0 ]; then # Cheking tag_name
+            if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release
+                current_tag=$release_info
+            else
+                current_tag=""
+            fi
+            i=1
+        elif [ $i -eq 1 ]; then # Checking draft boolean
+            if [ "$release_info" = "true" ]; then
+                current_tag=""
+            fi
+            i=2
+        elif [ $i -eq 2 ]; then # Checking prerelease boolean
+            if [ "$release_info" = "true" ]; then
+                current_tag=""
+            fi
+            i=0
+            if [ "$current_tag" != "" ]; then # If the current_tag is valid
+                if [ "$latest" = "" ]; then # If there is no latest yet
+                    latest="$current_tag"
+                else
+                    semverLT $current_tag $latest # Comparing latest and the current tag
+                    if [ $? -eq 1 ]; then
+                        latest="$current_tag"
+                    fi
+                fi
+            fi
+        fi
+    done
+
+    rm -f "$temp_file"
+    echo $latest
+}
+
+# MAIN
+current_tag="$(echo $GITHUB_REF | tr -d 'refs/tags/')"
+latest="$(get_latest)"
+
+if [ "$current_tag" != "$latest" ]; then
+    # The current release tag is not the latest
+    echo "false"
+else
+    # The current release tag is the latest
+    echo "true"
+fi

.github/workflows/README.md

@@ -1,4 +1,4 @@
-# GitHub actions workflow for MeiliDB
+# GitHub Actions Workflow for MeiliSearch
 
 > **Note:**
 
@@ -6,12 +6,14 @@
 
 ## Workflow
 
-- On each pull request, we are triggering `cargo test`.
-- On each commit on master, we are building the latest docker image.
-- On each tag, we are building:
-    - the tagged docker image
+- On each pull request, we trigger `cargo test`.
+- On each tag, we build:
+    - the tagged Docker image and publish it to Docker Hub
     - the binaries for MacOS, Ubuntu, and Windows
-    - the debian package
+    - the Debian package
+- On each stable release (`v*.*.*` tag):
+    - we build the `latest` Docker image and publish it to Docker Hub
+    - we publish the binary to Hombrew and Gemfury
 
 ## Problems
 

.github/workflows/coverage.yml

@@ -0,0 +1,34 @@
+---
+on:
+  pull_request:
+    types: [review_requested, ready_for_review]
+
+name: Execute code coverage
+
+jobs:
+  nightly-coverage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: clean
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --all-features --no-fail-fast
+        env:
+          CARGO_INCREMENTAL: "0"
+          RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=unwind -Zpanic_abort_tests"
+      - uses: actions-rs/grcov@v0.1
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ${{ steps.coverage.outputs.report }}
+          yml: ./codecov.yml
+          fail_ci_if_error: true

.github/workflows/publish-binaries.yml

@@ -1,9 +1,8 @@
-name: Publish binaries to GitHub release
-
 on:
-  push:
-    tags:
-      - '*'
+  release:
+    types: [published]
+
+name: Publish binaries to release
 
 jobs:
   publish:
@@ -20,8 +19,8 @@ jobs:
             artifact_name: meilisearch
             asset_name: meilisearch-macos-amd64
           - os: windows-latest
-            artifact_name: meilisearch
-            asset_name: meilisearch-windows-amd64
+            artifact_name: meilisearch.exe
+            asset_name: meilisearch-windows-amd64.exe
 
     steps:
     - uses: hecrj/setup-rust-action@master
@@ -33,7 +32,55 @@ jobs:
     - name: Upload binaries to release
       uses: svenstaro/upload-release-action@v1-release
       with:
-        repo_token: ${{ secrets.GITHUB_TOKEN }}
+        repo_token: ${{ secrets.PUBLISH_TOKEN }}
         file: target/release/${{ matrix.artifact_name }}
         asset_name: ${{ matrix.asset_name }}
         tag: ${{ github.ref }}
+
+  publish-armv7:
+    name: Publish for ARMv7
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v1.0.0
+      - uses: uraimo/run-on-arch-action@v1.0.7
+        id: runcmd
+        with:
+          architecture: armv7
+          distribution: ubuntu18.04
+          run: |
+            apt update
+            apt install -y curl gcc make
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
+            source $HOME/.cargo/env
+            cargo build --release --locked
+      - name: Upload the binary to release
+        uses: svenstaro/upload-release-action@v1-release
+        with:
+          repo_token: ${{ secrets.PUBLISH_TOKEN }}
+          file: target/release/meilisearch
+          asset_name: meilisearch-linux-armv7
+          tag: ${{ github.ref }}
+
+  publish-armv8:
+    name: Publish for ARMv8
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v1.0.0
+      - uses: uraimo/run-on-arch-action@v1.0.7
+        id: runcmd
+        with:
+          architecture: aarch64 # aka ARMv8
+          distribution: ubuntu18.04
+          run: |
+            apt update
+            apt install -y curl gcc make
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
+            source $HOME/.cargo/env
+            cargo build --release --locked
+      - name: Upload the binary to release
+        uses: svenstaro/upload-release-action@v1-release
+        with:
+          repo_token: ${{ secrets.PUBLISH_TOKEN }}
+          file: target/release/meilisearch
+          asset_name: meilisearch-linux-armv8
+          tag: ${{ github.ref }}

.github/workflows/publish-deb-brew-pkg.yml

@@ -1,15 +1,13 @@
-name: Publish deb pkg to GitHub release & apt repository
+name: Publish deb pkg to GitHub release & APT repository & Homebrew
 
 on:
-  push:
-    tags:
-      - '*'
+  release:
+    types: [released]
 
 jobs:
-  publish:
+  debian:
     name: Publish debian packagge
     runs-on: ubuntu-latest
-
     steps:
     - uses: hecrj/setup-rust-action@master
       with:
@@ -28,3 +26,14 @@ jobs:
         tag: ${{ github.ref }}
     - name: Upload debian pkg to apt repository
       run: curl -F package=@target/debian/meilisearch.deb https://${{ secrets.GEMFURY_PUSH_TOKEN }}@push.fury.io/meilisearch/
+
+  homebrew:
+    name: Bump Homebrew formula
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create PR to Homebrew
+        uses: mislav/bump-homebrew-formula-action@v1
+        with:
+          formula-name: meilisearch
+        env:
+          COMMITTER_TOKEN: ${{ secrets.HOMEBREW_COMMITTER_TOKEN }}

.github/workflows/publish-docker-latest.yml

@@ -1,8 +1,7 @@
 ---
 on:
-  push:
-    branches:
-      - master
+  release:
+    types: [released]
 
 name: Publish latest image to Docker Hub
 
@@ -10,8 +9,12 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+      - name: Check if current release is latest
+        run: echo "##[set-output name=is_latest;]$(sh .github/is-latest-release.sh)"
+        id: release
       - name: Publish to Registry
+        if: steps.release.outputs.is_latest == 'true'
         uses: elgohr/Publish-Docker-Github-Action@master
         with:
           name: getmeili/meilisearch

.github/workflows/test.yml

@@ -1,5 +1,12 @@
 ---
-on: [pull_request]
+on:
+  push:
+    branches:
+      - release-v*
+      - trying
+      - staging
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+' # this only concerns tags on stable
 
 name: Test binaries with cargo test
 
@@ -10,7 +17,6 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-
     steps:
     - uses: actions/checkout@v1
     - uses: actions-rs/toolchain@v1
@@ -18,8 +24,70 @@ jobs:
         profile: minimal
         toolchain: stable
         override: true
+        components: clippy
     - name: Run cargo test
       uses: actions-rs/cargo@v1
       with:
         command: test
-        args: --locked
+        args: --locked --release
+    - name: Run cargo clippy
+      uses: actions-rs/cargo@v1
+      with:
+        command: clippy
+
+  build-image:
+    name: Test the build of Docker image
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - run: docker build . --file Dockerfile -t meilisearch
+      name: Docker build
+
+  ## A push occurred on a release branch, a prerelease is created and assets are generated
+  prerelease:
+    name: create prerelease
+    needs: [check, build-image]
+    if: ${{ contains(github.ref, 'release-') && github.event_name == 'push' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Get version number
+        id: version-number
+        run: echo "##[set-output name=number;]$(echo ${{ github.ref }} | sed 's/.*\(v.*\)/\1/')"
+      - name: Get commit count
+        id: commit-count
+        run: echo "##[set-output name=count;]$(git rev-list remotes/origin/master..remotes/origin/release-${{ steps.version-number.outputs.number }} --count)"
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.PUBLISH_TOKEN }} # Personal Access Token
+        with:
+          tag_name: ${{ steps.version-number.outputs.number }}rc${{ steps.commit-count.outputs.count }}
+          release_name: Pre-release ${{ steps.version-number.outputs.number }}-rc${{ steps.commit-count.outputs.count }}
+          prerelease: true
+
+  ## If a tag is pushed, a release is created for this tag, and assets will be generated
+  release:
+    name: create release
+    needs: [check, build-image]
+    if:  ${{ contains(github.ref, 'tags/v') }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Get version number
+        id: version-number
+        run: echo "##[set-output name=number;]$(echo ${{ github.ref }} | sed 's/.*\(v.*\)/\1/')" 
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.PUBLISH_TOKEN }} # PAT
+        with:
+          tag_name: ${{ steps.version-number.outputs.number }}
+          release_name: Meilisearch ${{ steps.version-number.outputs.number }}
+          prerelease: false

CHANGELOG.md

@@ -0,0 +1,119 @@
+## v0.19.0
+
+  - The snapshots are now created and then renamed in atomically (#1172)
+  - Fix a race condition when an update and a document addition are processed immediately one after the other (#1176)
+  - Latin synonyms are normalized during indexation (#1174)
+
+## v0.18.1
+
+  - Fix unexpected CORS error (#1185)
+
+## v0.18.0
+
+  - Integration with the new tokenizer (#1091)
+  - Fix setting consistency bug (#1128)
+  - Fix attributes to retrieve bug (#1131)
+  - Increase default payload size (#1147)
+  - Improvements to code quality (#1167, #1165, #1126, #1161)
+
+## v0.17.0
+  - Fix corrupted data during placeholder search (#1089)
+  - Remove maintenance error from http (#1082)
+  - Disable frontend in production (#1097)
+  - Update nbHits count with filtered documents (#849)
+  - Remove update changelog ci check (#1090)
+  - Add deploy on Platform.sh option to README (#1087)
+  - Change movie gifs in README (#1077)
+  - Remove some clippy warnings (#1100)
+  - Improve script `download-latest.sh` (#1054)
+  - Bump dependencies version (#1056, #1057, #1059)
+
+## v0.16.0
+
+  - Automatically create index on document push if index doesn't exist (#914)
+  - Sort displayedAttributes and facetDistribution (#946)
+
+## v0.15.0
+
+  - Update actix-web dependency to 3.0.0 (#963)
+  - Consider an empty query to be a placeholder search (#916)
+
+## v0.14.1
+
+  - Fix version mismatch in snapshot importation (#959)
+
+## v0.14.0
+
+  - Sort displayedAttributes (#943)
+  - Fix facet distribution case (#797)
+  - Snapshotting (#839)
+  - Fix bucket-sort unwrap bug (#915)
+
+## v0.13.0
+
+  - placeholder search (#771)
+  - Add database version mismatch check (#794)
+  - Displayed and searchable attributes wildcard (#846)
+  - Remove sys-info route (#810)
+  - Check database version mismatch (#794)
+  - Fix unique docid bug (#841)
+  - Error codes in updates (#792)
+  - Sentry disable argument (#813)
+  - Log analytics if enabled (#825)
+  - Fix default values displayed on web interface (#874)
+
+## v0.12.0
+
+  - Fix long documents not being indexed completely bug (#816)
+  - Fix distinct attribute returning id instead of name (#800)
+  - error code rename (#805)
+
+## v0.11.1
+
+  - Fix facet cache on document update (#789)
+  - Improvements on settings consistency (#778)
+
+## v0.11.0
+
+  - Change the HTTP framework, moving from tide to actix-web (#601)
+  - Bump sentry version to 0.18.1 (#690)
+  - Enable max payload size override (#684)
+  - Disable sentry in debug (#681)
+  - Better terminal greeting (#680)
+  - Fix highlight misalignment (#679)
+  - Add support for facet count (#676)
+  - Add support for faceted search (#631)
+  - Add support for configuring the lmdb map size (#646, #647)
+  - Add exposed port for Dockerfile (#654)
+  - Add sentry probe (#664)
+  - Fix url trailing slash and double slash issues (#659)
+  - Fix accept all Content-Type by default (#653)
+  - Return the error message from Serde when a deserialization error is encountered (#661)
+  - Fix NormalizePath middleware to make the dashboard accessible (#695)
+  - Update sentry features to remove openssl (#702)
+  - Add SSL support (#669)
+  - Rename fieldsFrequency into fieldsDistribution in stats (#719)
+  - Add support for error code reporting (#703)
+  - Allow the dashboard to query private servers (#732)
+  - Add telemetry (#720)
+  - Add post route for search (#735)
+
+## v0.10.1
+
+  - Add support for floating points in filters (#640)
+  - Add '@' character as tokenizer separator (#607)
+  - Add support for filtering on arrays of strings (#611)
+
+## v0.10.0
+
+  - Refined filtering (#592)
+  - Add the number of hits in search result (#541)
+  - Add support for aligned crop in search result (#543)
+  - Sanitize the content displayed in the web interface (#539)
+  - Add support of nested null, boolean and seq values (#571 and #568, #574)
+  - Fixed the core benchmark (#576)
+  - Publish an ARMv7 and ARMv8 binaries on releases (#540 and #581)
+  - Fixed a bug where the result of the update status after the first update was empty (#542)
+  - Fixed a bug where stop words were not handled correctly (#594)
+  - Fix CORS issues (#602)
+  - Support wildcard on attributes to retrieve, highlight, and crop (#549, #565, and #598)

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at bonjour@meilisearch.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

CONTRIBUTING.md

@@ -0,0 +1,112 @@
+# Contributing
+
+First, thank you for contributing to MeiliSearch! The goal of this document is to
+provide everything you need to start contributing to MeiliSearch. The
+following TOC is sorted progressively, starting with the basics and
+expanding into more specifics.
+
+<!-- MarkdownTOC autolink="true" style="ordered" indent="   " -->
+
+1. [Assumptions](#assumptions)
+1. [Your First Contribution](#your-first-contribution)
+1. [Change Control](#change-control)
+   1. [Git Branches](#git-branches)
+   1. [Git Commits](#git-commits)
+      1. [Style](#style)
+   1. [Github Pull Requests](#github-pull-requests)
+      1. [Reviews & Approvals](#reviews--approvals)
+      1. [Merge Style](#merge-style)
+   1. [CI](#ci)
+1. [Development](#development)
+   1. [Setup](#setup)
+   1. [Testing](#testing)
+   1. [Benchmarking](#benchmarking--profiling)
+1. [Humans](#humans)
+   1. [Documentation](#documentation)
+   1. [Changelog](#changelog)
+
+<!-- /MarkdownTOC -->
+
+## Assumptions
+
+1. **You're familiar with [Github](https://github.com) and the [pull request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests)
+   workflow.**
+2. **You've read the MeiliSearch [docs](https://docs.meilisearch.com).**
+3. **You know about the [MeiliSearch community](https://docs.meilisearch.com/resources/contact.html).
+   Please use this for help.**
+
+## Your First Contribution
+
+1. Ensure your change has an issue! Find an
+   [existing issue](https://github.com/meilisearch/meilisearch/issues/) or [open a new issue](https://github.com/meilisearch/meilisearch/issues/new).
+   * This is where you can get a feel if the change will be accepted or not.
+2. Once approved, [fork the MeiliSearch repository](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) in your own
+   Github account.
+3. [Create a new Git branch](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-and-deleting-branches-within-your-repository)
+4. Review the MeiliSearch [workflow](#workflow) and [development](#development).
+5. Make your changes.
+6. [Submit the branch as a pull request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request-from-a-fork) to the main MeiliSearch
+   repo. A MeiliSearch team member should comment and/or review your pull request
+   with a few days. Although, depending on the circumstances, it may take
+   longer.
+
+## Change Control
+
+### Git Branches
+
+_All_ changes must be made in a branch and submitted as [pull requests](#pull-requests).
+MeiliSearch does not adopt any type of branch naming style, but please use something
+descriptive of your changes.
+
+### Git Commits
+
+#### Style
+
+Please ensure your commits are small and focused; they should tell a story of
+your change. This helps reviewers to follow your changes, especially for more
+complex changes.
+
+Familiarise yourself with [How to Write a Git Commit Message](https://chris.beams.io/posts/git-commit/).
+
+### Github Pull Requests
+
+Once your changes are ready you must submit your branch as a pull request.
+
+#### Reviews & Approvals
+
+All pull requests must be reviewed and approved by at least one MeiliSearch team
+member.
+
+#### Merge Style
+
+All pull requests are squashed and merged. We generally discourage large pull
+requests that are over 300-500 lines of diff. If you would like to propose
+a change that is larger we suggest coming onto our chat channel and
+discuss it with one of our engineers. This way we can talk through the
+solution and discuss if a change that large is even needed! This overall
+will produce a quicker response to the change and likely produce code that
+aligns better with our process.
+
+## Development
+
+### Setup
+
+See the [MeiliSearch Docs](https://docs.meilisearch.com/guides/advanced_guides/installation.html) for how to set up a development environment.
+
+### Benchmarking & Profiling
+
+We do not yet do any benchmarking, nor have we formalised our profiling. If you'd like to work on this please get in touch!
+
+## Humans
+
+After making your change, you'll want to prepare it for MeiliSearch users (mostly humans). This usually entails updating documentation and announcing your feature.
+
+### Documentation
+
+Documentation is very important to MeiliSearch. All contributions that
+alter user-facing behavior MUST include documentation changes. Please see
+[GitHub.com/meilisearch/documentation](https://github.com/meilisearch/documentation) for more info.
+
+### Changelog
+
+Until we have guidelines in place, updating the [`Changelog`](/CHANGELOG.md) is solely the responsibility of MeiliSearch team members.

Cargo.toml

@@ -3,7 +3,6 @@ members = [
     "meilisearch-core",
     "meilisearch-http",
     "meilisearch-schema",
-    "meilisearch-tokenizer",
     "meilisearch-types",
 ]
 

Dockerfile

@@ -18,10 +18,12 @@ RUN     $HOME/.cargo/bin/cargo build --release
 # Run
 FROM    alpine:3.10
 
-RUN     apk update --quiet
-RUN     apk add libgcc
+RUN     apk add -q --no-cache libgcc tini
 
 COPY    --from=compiler /meilisearch/target/release/meilisearch .
 
 ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
+EXPOSE  7700/tcp
+
+ENTRYPOINT ["tini", "--"]
 CMD     ./meilisearch

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019-2020 Meili SAS
+Copyright (c) 2019-2021 Meili SAS
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Procfile

@@ -1 +0,0 @@
-web: ./target/release/meilisearch --http-addr=0.0.0.0:$PORT

README.md

@@ -1,39 +1,81 @@
-# MeiliSearch
+<p align="center">
+  <img src="assets/logo.svg" alt="MeiliSearch" width="200" height="200" />
+</p>
 
-[![Build Status](https://github.com/meilisearch/MeiliSearch/workflows/Cargo%20test/badge.svg)](https://github.com/meilisearch/MeiliSearch/actions)
-[![dependency status](https://deps.rs/repo/github/meilisearch/MeiliSearch/status.svg)](https://deps.rs/repo/github/meilisearch/MeiliSearch)
-[![License](https://img.shields.io/badge/license-MIT-informational)](https://github.com/meilisearch/MeiliSearch/blob/master/LICENSE)
+<h1 align="center">MeiliSearch</h1>
 
-⚡ Ultra relevant and instant full-text search API 🔍
+<h4 align="center">
+  <a href="https://www.meilisearch.com">Website</a> |
+  <a href="https://roadmap.meilisearch.com/tabs/1-under-consideration">Roadmap</a> |
+  <a href="https://blog.meilisearch.com">Blog</a> |
+  <a href="https://fr.linkedin.com/company/meilisearch">LinkedIn</a> |
+  <a href="https://twitter.com/meilisearch">Twitter</a> |
+  <a href="https://docs.meilisearch.com">Documentation</a> |
+  <a href="https://docs.meilisearch.com/faq/">FAQ</a>
+</h4>
 
-MeiliSearch is a powerful, fast, open-source, easy to use, and deploy search engine. The search and indexation are fully customizable and handles features like typo-tolerance, filters, and synonyms.
-For more [details about those features, go to our documentation](https://docs.meilisearch.com/).
+<p align="center">
+  <a href="https://github.com/meilisearch/MeiliSearch/actions"><img src="https://github.com/meilisearch/MeiliSearch/workflows/Cargo%20test/badge.svg" alt="Build Status"></a>
+  <a href="https://deps.rs/repo/github/meilisearch/MeiliSearch"><img src="https://deps.rs/repo/github/meilisearch/MeiliSearch/status.svg" alt="Dependency status"></a>
+  <a href="https://github.com/meilisearch/MeiliSearch/blob/master/LICENSE"><img src="https://img.shields.io/badge/license-MIT-informational" alt="License"></a>
+  <a href="https://slack.meilisearch.com"><img src="https://img.shields.io/badge/slack-MeiliSearch-blue.svg?logo=slack" alt="Slack"></a>
+  <a href="https://github.com/meilisearch/MeiliSearch/discussions" alt="Discussions"><img src="https://img.shields.io/badge/github-discussions-red" /></a>
+  <a href="https://app.bors.tech/repositories/26457"><img src="https://bors.tech/images/badge_small.svg" alt="Bors enabled"></a>
+</p>
 
-[![crates.io demo gif](misc/crates-io-demo.gif)](https://crates.meilisearch.com)
-> Meili helps the Rust community find crates on [crates.meilisearch.com](https://crates.meilisearch.com)
+<p align="center">⚡ Lightning Fast, Ultra Relevant, and Typo-Tolerant Search Engine 🔍</p>
 
-## Features
-* Search as-you-type experience (answers < 50ms)
+**MeiliSearch** is a powerful, fast, open-source, easy to use and deploy search engine. Both searching and indexing are highly customizable. Features such as typo-tolerance, filters, and synonyms are provided out-of-the-box.
+For more information about features go to [our documentation](https://docs.meilisearch.com/).
+
+<p align="center">
+  <img src="assets/trumen_quick_loop.gif" alt="Web interface gif" />
+</p>
+
+## ✨ Features
+* Search as-you-type experience (answers < 50 milliseconds)
 * Full-text search
-* Typo tolerant (understands typos and spelling mistakes)
-* Supports Kanji
+* Typo tolerant (understands typos and miss-spelling)
+* Faceted search and filters
+* Supports Kanji characters
 * Supports Synonym
 * Easy to install, deploy, and maintain
-* Whole documents returned
+* Whole documents are returned
 * Highly customizable
-* RESTfull API
+* RESTful API
 
-## Quick Start
+## Getting started
 
 ### Deploy the Server
 
-#### Run it using Docker
+#### Homebrew (Mac OS)
 
 ```bash
-docker run -it -p 7700:7700 --rm getmeili/meilisearch
+brew update && brew install meilisearch
+meilisearch
 ```
 
-#### Installation using APT
+#### Docker
+
+```bash
+docker run -p 7700:7700 -v $(pwd)/data.ms:/data.ms getmeili/meilisearch
+```
+
+#### Try MeiliSearch in our Sandbox
+
+Create a MeiliSearch instance in [MeiliSearch Sandbox](https://sandbox.meilisearch.com/). This instance is free, and will be active for 72 hours.
+
+#### Run on Digital Ocean
+
+[![DigitalOcean Marketplace](assets/do-btn-blue.svg)](https://marketplace.digitalocean.com/apps/meilisearch?action=deploy&refcode=7c67bd97e101)
+
+#### Deploy on Platform.sh
+
+<a href="https://console.platform.sh/projects/create-project?template=https://raw.githubusercontent.com/platformsh/template-builder/master/templates/meilisearch/.platform.template.yaml&utm_content=meilisearch&utm_source=github&utm_medium=button&utm_campaign=deploy_on_platform">
+    <img src="https://platform.sh/images/deploy/lg-blue.svg" alt="Deploy on Platform.sh" width="180px" />
+</a>
+
+#### APT (Debian & Ubuntu)
 
 ```bash
 echo "deb [trusted=yes] https://apt.fury.io/meilisearch/ /" > /etc/apt/sources.list.d/fury.list
@@ -41,44 +83,46 @@ apt update && apt install meilisearch-http
 meilisearch
 ```
 
-#### Download the binary
+#### Download the binary (Linux & Mac OS)
 
 ```bash
 curl -L https://install.meilisearch.com | sh
 ./meilisearch
 ```
 
-#### Run it on heroku
-
-[![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy?template=https://github.com/meilisearch/MeiliSearch)
-
 #### Compile and run it from sources
 
-If you have the Rust toolchain already installed, you can compile from the source
+If you have the latest stable Rust toolchain installed on your local system, clone the repository and change it to your working directory.
 
 ```bash
 git clone https://github.com/meilisearch/MeiliSearch.git
 cd MeiliSearch
+```
+
+In the cloned repository, compile MeiliSearch.
+
+```bash
+rustup override set stable
+rustup update stable
 cargo run --release
 ```
 
 ### Create an Index and Upload Some Documents
 
-We provide a movie dataset that you can use for testing purposes.
+Let's create an index! If you need a sample dataset, use [this movie database](https://www.notion.so/meilisearch/A-movies-dataset-to-test-Meili-1cbf7c9cfa4247249c40edfa22d7ca87#b5ae399b81834705ba5420ac70358a65). You can also find it in the `datasets/` directory.
 
 ```bash
 curl -L 'https://bit.ly/2PAcw9l' -o movies.json
 ```
 
-MeiliSearch can serve multiple indexes, with different kinds of documents,
-therefore, it is required to create the index before sending documents to it.
+MeiliSearch can serve multiple indexes, with different kinds of documents.
+It is required to create an index before sending documents to it.
 
 ```bash
 curl -i -X POST 'http://127.0.0.1:7700/indexes' --data '{ "name": "Movies", "uid": "movies" }'
 ```
 
-Now that the server knows about our brand new index, we can send it data.
-We provided you a small dataset that is available in the `datasets/` directory.
+Now that the server knows about your brand new index, you're ready to send it some data.
 
 ```bash
 curl -i -X POST 'http://127.0.0.1:7700/indexes/movies/documents' \
@@ -90,8 +134,9 @@ curl -i -X POST 'http://127.0.0.1:7700/indexes/movies/documents' \
 
 #### In command line
 
-The search engine is now aware of our documents and can serve those via our HTTP server again.
-The [`jq` command-line tool](https://stedolan.github.io/jq/) can significantly help you read the server responses.
+The search engine is now aware of your documents and can serve those via an HTTP server.
+
+The [`jq` command-line tool](https://stedolan.github.io/jq/) can greatly help you read the server responses.
 
 ```bash
 curl 'http://127.0.0.1:7700/indexes/movies/search?q=botman+robin&limit=2' | jq
@@ -122,67 +167,37 @@ curl 'http://127.0.0.1:7700/indexes/movies/search?q=botman+robin&limit=2' | jq
 }
 ```
 
-#### With the Web Interface
+#### Use the Web Interface
 
-MeiliSearch provides a simple web interface containing a search bar in order to quickly test the instant search experience with a given set of documents.
+We also deliver an **out-of-the-box web interface** in which you can test MeiliSearch interactively.
 
-This web interface is available in your browser at the root of the server. The default URL is [http://127.0.0.1:7700](http://127.0.0.1:7700).
+You can access the web interface in your web browser at the root of the server. The default URL is [http://127.0.0.1:7700](http://127.0.0.1:7700). All you need to do is open your web browser and enter MeiliSearch’s address to visit it. This will lead you to a web page with a search bar that will allow you to search in the selected index.
 
-### Documentation
+| [See the gif above](#demo)
 
-Now, that you have a running MeiliSearch, you can learn more and tune your search engine using [the documentation](https://docs.meilisearch.com).
+## Documentation
 
-## How it works
-
-MeiliSearch uses [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database) as the internal key-value store. The key-value store allows us to handle updates and queries with small memory and CPU overheads. The whole ranking system is [data oriented](https://github.com/meilisearch/MeiliSearch/issues/82) and provides great performances.
-
-You can [read the deep dive](deep-dive.md) if you want more information on the engine; it describes the whole process of generating updates and handling queries. Also, you can take a look at the [typos and ranking rules](typos-ranking-rules.md) if you want to know the default rules used to sort the documents.
-
-### Technical features
-
-- Provides [6 default ranking criteria](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/criterion/mod.rs#L106-L111) used to [bucket sort](https://en.wikipedia.org/wiki/Bucket_sort) documents
-- Accepts [custom criteria](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/criterion/mod.rs#L20-L29) and can apply them in any custom order
-- Support [ranged queries](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/query_builder.rs#L342), useful for paginating results
-- Can [distinct](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/query_builder.rs#L324-L329) and [filter](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/query_builder.rs#L313-L318) returned documents based on context defined rules
-- Searches for [concatenated](https://github.com/meilisearch/MeiliSearch/pull/164) and [splitted query words](https://github.com/meilisearch/MeiliSearch/pull/232) to improve the search quality.
-- Can store complete documents or only [user schema specified fields](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/datasets/movies/schema.toml)
-- The [default tokenizer](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-tokenizer/src/lib.rs) can index latin and kanji based languages
-- Returns [the matching text areas](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-types/src/lib.rs#L49-L65), useful to highlight matched words in results
-- Accepts query time search config like the [searchable attributes](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/query_builder.rs#L331-L336)
-- Supports [runtime incremental indexing](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/store/mod.rs#L143-L212)
-
-## Performances
-
-With a dataset composed of _100 353_ documents with _352_ attributes each and _3_ of them indexed.
-So more than _300 000_ fields indexed for _35 million_ stored we can handle more than _2.8k req/sec_ with an average response time of _9 ms_ on an Intel i7-7700 (8) @ 4.2GHz.
-
-Requests are made using [wrk](https://github.com/wg/wrk) and scripted to simulate real users' queries.
-
-```
-Running 10s test @ http://localhost:2230
-  2 threads and 25 connections
-  Thread Stats   Avg      Stdev     Max   +/- Stdev
-    Latency     9.52ms    7.61ms  99.25ms   84.58%
-    Req/Sec     1.41k   119.11     1.78k    64.50%
-  28080 requests in 10.01s, 7.42MB read
-Requests/sec:   2806.46
-Transfer/sec:    759.17KB
-```
-
-We also indexed a dataset containing something like _12 millions_ cities names in _24 minutes_ on a machine with _8 cores_, _64 GB of RAM_, and a _300 GB NMVe_ SSD.<br/>
-The resulting database was _16 GB_ and search results were between _30 ms_ and _4 seconds_ for short prefix queries.
-
-### Notes
-
-With Rust 1.32 the allocator has been [changed to use the system allocator](https://blog.rust-lang.org/2019/01/17/Rust-1.32.0.html#jemalloc-is-removed-by-default).
-We have seen much better performances when [using jemalloc as the global allocator](https://github.com/alexcrichton/jemallocator#documentation).
+Now that your MeiliSearch server is up and running, you can learn more about how to tune your search engine in [the documentation](https://docs.meilisearch.com).
 
 ## Contributing
 
-We will be glad if you submit issues and pull requests. You can help to grow this project and start contributing by checking [issues tagged "good-first-issue"](https://github.com/meilisearch/MeiliSearch/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22). It is a good start!
+Hey! We're glad you're thinking about contributing to MeiliSearch! If you think something is missing or could be improved, please open issues and pull requests. If you'd like to help this project grow, we'd love to have you! To start contributing, checking [issues tagged as "good-first-issue"](https://github.com/meilisearch/MeiliSearch/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) is a good start!
 
-### Analytic Events
+## Telemetry
 
-We send events to our Amplitude instance to be aware of the number of people who use MeiliSearch.<br/>
-We only send the platform on which the server runs once by day. No other information is sent.<br/>
-If you do not want us to send events, you can disable these analytics by using the `MEILI_NO_ANALYTICS` env variable.
+MeiliSearch collects anonymous data regarding general usage.
+This helps us better understand developers usage of MeiliSearch features.<br/>
+To see what information we're retrieving, please see the complete list [on the dedicated issue](https://github.com/meilisearch/MeiliSearch/issues/720).<br/>
+We also use Sentry to make us crash and error reports. If you want to know more about what Sentry collects, please visit their [privacy policy website](https://sentry.io/privacy/).<br/>
+This program is optional, you can disable these analytics by using the `MEILI_NO_ANALYTICS` env variable.
+
+## 💌 Contact
+
+Feel free to contact us about any questions you may have:
+* At [bonjour@meilisearch.com](mailto:bonjour@meilisearch.com)
+* Via the chat box available on every page of [our documentation](https://docs.meilisearch.com/) and on [our landing page](https://www.meilisearch.com/).
+* 🆕 Join our [GitHub Discussions forum](https://github.com/meilisearch/MeiliSearch/discussions)
+* Join our [Slack community](https://slack.meilisearch.com/).
+* By opening an issue.
+
+MeiliSearch is developed by [Meili](https://www.meilisearch.com), a young company. To know more about us, you can [read our blog](https://blog.meilisearch.com). Any suggestion or feedback is highly appreciated. Thank you for your support!

app.json

@@ -1,16 +0,0 @@
-{
-  "name": "MeiliSearch",
-  "description": "Ultra relevant, instant and typo-tolerant full-text search API",
-  "keywords": [
-    "search-engine",
-    "instant search",
-    "search API"
-  ],
-  "website": "https://docs.meilisearch.com/",
-  "repository": "https://github.com/meilisearch/MeiliSearch",
-  "buildpacks": [
-    {
-      "url": "https://github.com/emk/heroku-buildpack-rust"
-    }
-  ]
-}

assets/do-btn-blue.svg

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="200px" height="42px" viewBox="0 0 200 42" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 52.5 (67469) - http://www.bohemiancoding.com/sketch -->
+    <title>do-btn-blue</title>
+    <desc>Created with Sketch.</desc>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="Partner-welcome-kit-Copy-3" transform="translate(-651.000000, -762.000000)">
+            <g id="do-btn-blue" transform="translate(651.000000, 763.000000)">
+                <rect id="Rectangle-Copy" fill="#0069FF" x="0" y="0" width="200" height="40" rx="6"></rect>
+                <path d="M45,0 L45,40" id="Line-2" stroke="#FFFFFF" stroke-linecap="square"></path>
+                <g id="DO_Logo_horizontal_blue-Copy" transform="translate(13.000000, 11.000000)" fill="#FFFFFF">
+                    <path d="M10.0098493,20 L10.0098493,16.1262429 C14.12457,16.1262429 17.2897398,12.0548452 15.7269372,7.74627862 C15.1334679,6.14538921 13.8674,4.86072487 12.2650328,4.28756693 C7.952489,2.72620566 3.87733294,5.88845634 3.87733294,9.99938223 C3.87733294,9.99938223 3.87733294,9.99938223 3.87733294,9.99938223 L0,9.99938223 C0,3.45747613 6.3303395,-1.64165309 13.1948014,0.492866119 C16.2017127,1.42177726 18.57559,3.81322933 19.5053586,6.79760341 C21.6418482,13.6754986 16.5577943,20 10.0098493,20 Z" id="XMLID_49_"></path>
+                    <polygon id="XMLID_47_" points="9.56521739 15.6521739 6.08695652 15.6521739 6.08695652 12.173913 6.08695652 12.173913 9.56521739 12.173913 9.56521739 12.173913"></polygon>
+                    <polygon id="XMLID_46_" points="6.08695652 19.1304348 3.47826087 19.1304348 3.47826087 19.1304348 3.47826087 16.5217391 6.08695652 16.5217391"></polygon>
+                    <polygon id="XMLID_45_" points="3.47826087 16.5217391 0.869565217 16.5217391 0.869565217 16.5217391 0.869565217 13.9130435 0.869565217 13.9130435 3.47826087 13.9130435 3.47826087 13.9130435"></polygon>
+                </g>
+                <text id="Create-a-Droplet-Copy" font-family="Sailec-Medium, Sailec" font-size="16" font-weight="400" fill="#FFFFFF">
+                    <tspan x="58" y="26">Create a Droplet</tspan>
+                </text>
+            </g>
+        </g>
+    </g>
+</svg>

assets/logo.svg

@@ -0,0 +1,17 @@
+<svg width="360" height="360" viewBox="0 0 360 360" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g id="logo_main">
+<rect id="Rectangle" x="107.333" y="0.150146" width="274.315" height="274.315" rx="98.8334" transform="rotate(23 107.333 0.150146)" fill="url(#paint0_linear)"/>
+<path id="Rectangle_2" fill-rule="evenodd" clip-rule="evenodd" d="M61.3296 230.199C46.2224 194.608 38.6688 176.813 38.208 160.329C37.5286 136.025 47.0175 112.539 64.3891 95.5282C76.1718 83.9904 93.9669 76.4368 129.557 61.3296C165.147 46.2224 182.943 38.6688 199.427 38.208C223.731 37.5286 247.217 47.0175 264.228 64.3891C275.766 76.1718 283.319 93.9669 298.426 129.557C313.534 165.147 321.087 182.943 321.548 199.427C322.227 223.731 312.738 247.217 295.367 264.228C283.584 275.766 265.789 283.319 230.199 298.426C194.608 313.534 176.813 321.087 160.329 321.548C136.025 322.227 112.539 312.738 95.5282 295.367C83.9903 283.584 76.4368 265.789 61.3296 230.199Z" fill="url(#paint1_linear)"/>
+<path id="m" fill-rule="evenodd" clip-rule="evenodd" d="M219.568 130.748C242.363 130.748 259.263 147.451 259.263 174.569V229.001H227.232V179.678C227.232 166.119 220.747 159.634 210.136 159.634C205.223 159.634 200.311 161.796 195.595 167.494C195.791 169.852 195.988 172.21 195.988 174.569V229.001H164.154V179.678C164.154 166.119 157.472 159.634 147.057 159.634C142.145 159.634 137.429 161.992 132.712 168.084V229.001H100.878V133.695H132.712V139.394C139.197 133.892 145.878 130.748 156.49 130.748C168.477 130.748 178.695 135.267 185.769 143.52C195.791 134.678 205.42 130.748 219.568 130.748Z" fill="white"/>
+</g>
+<defs>
+<linearGradient id="paint0_linear" x1="-13.6248" y1="129.208" x2="244.49" y2="403.522" gradientUnits="userSpaceOnUse">
+<stop stop-color="#E41359"/>
+<stop offset="1" stop-color="#F23C79"/>
+</linearGradient>
+<linearGradient id="paint1_linear" x1="11.0088" y1="111.65" x2="111.65" y2="348.747" gradientUnits="userSpaceOnUse">
+<stop stop-color="#24222F"/>
+<stop offset="1" stop-color="#2B2937"/>
+</linearGradient>
+</defs>
+</svg>

bors.toml

@@ -0,0 +1,3 @@
+status = ["Test on macos-latest", "Test on ubuntu-latest"]
+# 4 hours timeout
+timeout-sec = 14400

bump.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/bash
+
+NEW_VERSION=$1
+
+if [ -z "$NEW_VERSION" ]
+then
+	echo "error: a version number must be provided"
+	exit 1
+fi
+
+# find current version
+CURRENT_VERSION=$(cat **/*.toml | grep meilisearch | grep version | sed 's/.*\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/' | sed "1q;d")
+
+# bump all version in .toml
+echo "bumping from version $CURRENT_VERSION to version $NEW_VERSION"
+while true
+do
+	read -r -p "Continue (y/n)?" choice
+	case "$choice" in
+		y|Y ) break;;
+		n|N ) echo "aborting bump" && exit 0;;
+		* ) echo "invalid choice";;
+	esac
+done
+# update all crate version
+sed -i "s/version = \"$CURRENT_VERSION\"/version = \"$NEW_VERSION\"/" **/*.toml
+
+printf "running cargo check: "
+
+CARGO_CHECK=$(cargo check 2>&1)
+
+if [ $? != "0" ]
+then
+	printf "\033[31;1m FAIL \033[0m\n"
+	printf "$CARGO_CHECK"
+	exit 1
+fi
+printf "\033[32;1m OK \033[0m\n"

datasets/movies/README.md

@@ -1 +1 @@
-_datas in movies.csv are from https://www.themoviedb.org/_
+_datas in movies.json are from https://www.themoviedb.org/_

datasets/movies/schema.toml

@@ -1,21 +0,0 @@
-# This schema has been generated ...
-# The order in which the attributes are declared is important,
-# it specify the attribute xxx...
-identifier = "id"
-
-[attributes.id]
-displayed = true
-
-[attributes.title]
-displayed = true
-indexed = true
-
-[attributes.overview]
-displayed = true
-indexed = true
-
-[attributes.release_date]
-displayed = true
-
-[attributes.poster]
-displayed = true

deep-dive.md

@@ -1,95 +0,0 @@
-# A deep dive in MeiliSearch
-
-On the 15 of May 2019.
-
-MeiliSearch is a full text search engine based on a final state transducer named [fst](https://github.com/BurntSushi/fst) and a key-value store named [sled](https://github.com/spacejam/sled). The goal of a search engine is to store data and to respond to queries as accurate and fast as possible. To achieve this it must save the matching words in an [inverted index](https://en.wikipedia.org/wiki/Inverted_index).
-
-<!-- MarkdownTOC autolink="true" -->
-
-- [Where is the data stored?](#where-is-the-data-stored)
-- [What does the key-value store contains?](#what-does-the-key-value-store-contains)
-    - [The inverted word index](#the-inverted-word-index)
-        - [A final state transducer](#a-final-state-transducer)
-        - [Document indexes](#document-indexes)
-    - [The schema](#the-schema)
-    - [Document attributes](#document-attributes)
-- [How is a request processed?](#how-is-a-request-processed)
-    - [Query lexemes](#query-lexemes)
-    - [Automatons and query index](#automatons-and-query-index)
-    - [Sort by criteria](#sort-by-criteria)
-
-<!-- /MarkdownTOC -->
-
-## Where is the data stored?
-
-MeiliSearch is entirely backed by a key-value store like any good database (i.e. Postgres, MySQL). This brings a great flexibility in the way documents can be stored and updates handled along time.
-
-[sled will brings some](https://github.com/spacejam/sled/tree/434533332a3f485e6d2e467023be0a0b55d3a1af#plans) of the [A.C.I.D. properties](https://en.wikipedia.org/wiki/ACID_(computer_science)) to help us be sure the saved data is consistent.
-
-
-
-## What does the key-value store contains?
-
-It contain the inverted word index, the schema and the documents fields.
-
-### The inverted word index
-
-[The inverted word index](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/words_index.rs) is a sled Tree dedicated to store and give access to all documents that contains a specific word. The information stored under the word is simply a big ordered array of where in the document the word has been found. In other word, a big list of [`DocIndex`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/lib.rs#L35-L51).
-
-#### A final state transducer
-
-_...also abbreviated fst_
-
-This is the first entry point of the engine, you can read more about how it work with the beautiful blog post of @BurntSushi, [Index 1,600,000,000 Keys with Automata and Rust](https://blog.burntsushi.net/transducers/).
-
-To make it short it is a powerful way to store all the words that are present in the indexed documents. You construct it by giving it all the words you want to index. When you want to search in it you can provide any automaton you want, in MeiliSearch [a custom levenshtein automaton](https://github.com/tantivy-search/levenshtein-automata/) is used.
-
-#### Document indexes
-
-The `fst` will only return the words that match with the search automaton but the goal of the search engine is to retrieve all matches in all the documents when a query is made. You want it to return some sort of position in an attribute in a document, an information about where the given word matched.
-
-To make it possible we retrieve all of the `DocIndex` corresponding to all the matching words in the fst, we use the [`WordsIndex`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/words_index.rs#L11-L21) Tree to get the `DocIndexes` corresponding the words.
-
-### The schema
-
-The schema is a data structure that represents which documents attributes should be stored and which should be indexed. It is stored under a the [`MainIndex`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/main_index.rs#L12) Tree and given to MeiliSearch only at the creation of an index.
-
-Each document attribute is associated to a unique 16 bit number named [`SchemaAttr`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/schema.rs#L186).
-
-In the future, this schema type could be given along with updates, the database could be able to handled a new schema and reindex the database according to the new one.
-
-### Document attributes
-
-When the engine handle a query the result that the requester want is a document, not only the [`Matches`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/lib.rs#L62-L88) associated to it, fields of the original document must be returned too.
-
-So MeiliSearch again uses the power of the underlying key-value store and save the documents attributes marked as _STORE_ in the schema. The dedicated Tree for this information is the [`DocumentsIndex`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/documents_index.rs#L11).
-
-When a document field is saved in the key-value store its value is binary encoded using [message pack](https://github.com/3Hren/msgpack-rust), so a document must be serializable using serde.
-
-
-
-## How is a request processed?
-
-Now that we have our inverted index we are able to return results based on a query. In the MeiliSearch universe a query is a simple string containing words.
-
-### Query lexemes
-
-The first step to be able to call the underlying structures is to split the query in words, for that we use a [custom tokenizer](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-tokenizer/src/lib.rs#L82-L84). Note that a tokenizer is specialized for a human language, this is the hard part.
-
-### Automatons and query index
-
-So to query the fst we need an automaton, in MeiliSearch we use a [levenshtein automaton](https://en.wikipedia.org/wiki/Levenshtein_automaton), this automaton is constructed using a string and a maximum distance. According to the [Algolia's blog post](https://blog.algolia.com/inside-the-algolia-engine-part-3-query-processing/#algolia%e2%80%99s-way-of-searching-for-alternatives) we [created the DFAs](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/automaton.rs#L59-L78) with different settings.
-
-Thanks to the power of the fst library [it is possible to union multiple automatons](https://docs.rs/fst/0.3.2/fst/map/struct.OpBuilder.html#method.union) on the same fst set. The `Stream` is able to return all the matching words. We use these words to find the whole list of `DocIndexes` associated.
-
-With all these informations it is possible [to reconstruct a list of all the `DocIndexes` associated](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/query_builder.rs#L103-L130) with the words queried.
-
-### Sort by criteria
-
-Now that we are able to get a big list of [DocIndexes](https://github.com/Kerollmops/MeiliSearch/blob/550dc1e99224e386516877450320f694947332d4/src/lib.rs#L21-L36) it is not enough to sort them by criteria, we need more informations like the levenshtein distance or the fact that a query word match exactly the word stored in the fst. So [we stuff it a little bit](https://github.com/Kerollmops/MeiliSearch/blob/550dc1e99224e386516877450320f694947332d4/src/rank/query_builder.rs#L86-L93), and aggregate all these [Matches](https://github.com/Kerollmops/MeiliSearch/blob/550dc1e99224e386516877450320f694947332d4/src/lib.rs#L47-L74) for each document. This way it will be easy to sort a simple vector of document using a bunch of functions.
-
-With this big list of documents and associated matches [we are able to sort only the part of the slice that we want](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/query_builder.rs#L160-L188) using bucket sorting. [Each criterion](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/criterion/mod.rs#L95-L101) is evaluated on each subslice without copy, thanks to [GroupByMut](https://docs.rs/slice-group-by/0.2.4/slice_group_by/) which, I hope [will soon be merged](https://github.com/rust-lang/rfcs/pull/2477).
-
-Note that it is possible to customize the criteria used by using the `QueryBuilder::with_criteria` constructor, this way you can implement some custom ranking based on the document attributes using the appropriate structure and the [`document` method](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/index.rs#L86).
-
-At this point, MeiliSearch work is over 🎉

download-latest.sh

@@ -6,8 +6,10 @@ GREEN='\033[32m'
 DEFAULT='\033[0m'
 
 # GLOBALS
-GREP_SEMVER_REGEXP='\"v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\"' # i.e. "v[number].[number].[number]"
 BINARY_NAME='meilisearch'
+GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number]
+
+# FUNCTIONS
 
 # semverParseInto and semverLT from https://github.com/cloudflare/semver_bash/blob/master/semver.sh
 
@@ -66,6 +68,88 @@ semverLT() {
     return 1
 }
 
+# Returns the tag of the latest stable release (in terms of semver and not of release date)
+get_latest() {
+    temp_file='temp_file' # temp_file needed because the grep would start before the download is over
+    curl -s 'https://api.github.com/repos/meilisearch/MeiliSearch/releases' > "$temp_file" || return 1
+    releases=$(cat "$temp_file" | \
+        grep -E "tag_name|draft|prerelease" \
+        | tr -d ',"' | cut -d ':' -f2 | tr -d ' ')
+        # Returns a list of [tag_name draft_boolean prerelease_boolean ...]
+        # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false...
+
+    i=0
+    latest=""
+    current_tag=""
+    for release_info in $releases; do
+        if [ $i -eq 0 ]; then # Cheking tag_name
+            if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release
+                current_tag=$release_info
+            else
+                current_tag=""
+            fi
+            i=1
+        elif [ $i -eq 1 ]; then # Checking draft boolean
+            if [ "$release_info" = "true" ]; then
+                current_tag=""
+            fi
+            i=2
+        elif [ $i -eq 2 ]; then # Checking prerelease boolean
+            if [ "$release_info" = "true" ]; then
+                current_tag=""
+            fi
+            i=0
+            if [ "$current_tag" != "" ]; then # If the current_tag is valid
+                if [ "$latest" = "" ]; then # If there is no latest yet
+                    latest="$current_tag"
+                else
+                    semverLT $current_tag $latest # Comparing latest and the current tag
+                    if [ $? -eq 1 ]; then
+                        latest="$current_tag"
+                    fi
+                fi
+            fi
+        fi
+    done
+
+    rm -f "$temp_file"
+    echo $latest
+}
+
+# Gets the OS by setting the $os variable
+# Returns 0 in case of success, 1 otherwise.
+get_os() {
+    os_name=$(uname -s)
+    case "$os_name" in
+    'Darwin')
+        os='macos'
+        ;;
+    'Linux')
+        os='linux'
+        ;;
+    *)
+        return 1
+    esac
+    return 0
+}
+
+# Gets the architecture by setting the $archi variable
+# Returns 0 in case of success, 1 otherwise.
+get_archi() {
+    architecture=$(uname -m)
+    case "$architecture" in
+    'x86_64' | 'amd64')
+        archi='amd64'
+        ;;
+    'aarch64')
+        archi='armv8'
+        ;;
+    *)
+        return 1
+    esac
+    return 0
+}
+
 success_usage() {
     printf "$GREEN%s\n$DEFAULT" "MeiliSearch binary successfully downloaded as '$BINARY_NAME' file."
     echo ''
@@ -76,53 +160,28 @@ success_usage() {
 }
 
 failure_usage() {
-    printf "$RED%s\n$DEFAULT" 'ERROR: MeiliSearch binary is not available for your OS distribution yet.'
+    printf "$RED%s\n$DEFAULT" 'ERROR: MeiliSearch binary is not available for your OS distribution or your architecture yet.'
     echo ''
     echo 'However, you can easily compile the binary from the source files.'
-    echo 'Follow the steps on the docs: https://docs.meilisearch.com/advanced_guides/binary.html#how-to-compile-meilisearch'
+    echo 'Follow the steps at the page ("Source" tab): https://docs.meilisearch.com/guides/advanced_guides/installation.html'
 }
 
-# OS DETECTION
-echo 'Detecting OS distribution...'
-os_name=$(uname -s)
-if [ "$os_name" != "Darwin" ]; then
-    os_name=$(cat /etc/os-release | grep '^ID=' | tr -d '"' | cut -d '=' -f 2)
-fi
-echo "OS distribution detected: $os_name"
-case "$os_name" in
-'Darwin')
-    os='macos'
-    ;;
-'ubuntu' | 'debian')
-    os='linux'
-    ;;
-*)
+# MAIN
+latest="$(get_latest)"
+
+if ! get_os; then
     failure_usage
     exit 1
-esac
+fi
 
-# GET LATEST VERSION
-tags=$(curl -s 'https://api.github.com/repos/meilisearch/MeiliSearch/tags' \
-    | grep "$GREP_SEMVER_REGEXP" \
-    | grep 'name' \
-    | tr -d '"' | tr -d ',' | cut -d 'v' -f 2)
+if ! get_archi; then
+    failure_usage
+    exit 1
+fi
 
-latest=""
-for tag in $tags; do
-    if [ "$latest" = "" ]; then
-        latest="$tag"
-    else
-        semverLT $tag $latest
-        if [ $? -eq 1 ]; then
-            latest="$tag"
-        fi
-    fi
-done
-
-# DOWNLOAD THE LATEST
-echo "Downloading MeiliSearch binary v$latest for $os..."
-release_file="meilisearch-$os-amd64"
-link="https://github.com/meilisearch/MeiliSearch/releases/download/v$latest/$release_file"
+echo "Downloading MeiliSearch binary $latest for $os, architecture $archi..."
+release_file="meilisearch-$os-$archi"
+link="https://github.com/meilisearch/MeiliSearch/releases/download/$latest/$release_file"
 curl -OL "$link"
 mv "$release_file" "$BINARY_NAME"
 chmod 744 "$BINARY_NAME"

meilisearch-core/Cargo.toml

@@ -1,54 +1,53 @@
 [package]
 name = "meilisearch-core"
-version = "0.8.4"
+version = "0.19.0"
 license = "MIT"
 authors = ["Kerollmops <clement@meilisearch.com>"]
 edition = "2018"
 
 [dependencies]
-arc-swap = "0.4.3"
-bincode = "1.1.4"
-byteorder = "1.3.2"
-chrono = { version = "0.4.9", features = ["serde"] }
-compact_arena = "0.4.0"
-crossbeam-channel = "0.4.0"
-deunicode = "1.0.0"
-env_logger = "0.7.0"
-fst = { version = "0.3.5", default-features = false }
-hashbrown = { version = "0.6.0", features = ["serde"] }
-heed = "0.6.1"
-intervaltree = "0.2.4"
-itertools = "0.8.2" # kill me please
-levenshtein_automata = { version = "0.1.1", features = ["fst_automaton"] }
-log = "0.4.8"
-meilisearch-schema = { path = "../meilisearch-schema", version = "0.8.4" }
-meilisearch-tokenizer = { path = "../meilisearch-tokenizer", version = "0.8.4" }
-meilisearch-types = { path = "../meilisearch-types", version = "0.8.4" }
-once_cell = "1.2.0"
-ordered-float = { version = "1.0.2", features = ["serde"] }
-serde = { version = "1.0.101", features = ["derive"] }
-serde_json = "1.0.41"
-siphasher = "0.3.1"
+arc-swap = "1.2.0"
+bincode = "1.3.1"
+byteorder = "1.3.4"
+chrono = { version = "0.4.19", features = ["serde"] }
+compact_arena = "0.4.1"
+cow-utils = "0.1.2"
+crossbeam-channel = "0.5.0"
+deunicode = "1.1.1"
+either = "1.6.1"
+env_logger = "0.8.2"
+fst = "0.4.5"
+hashbrown = { version = "0.9.1", features = ["serde"] }
+heed = "0.10.6"
+indexmap = { version = "1.6.1", features = ["serde-1"] }
+intervaltree = "0.2.6"
+itertools = "0.10.0"
+levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }
+log = "0.4.11"
+meilisearch-error = { path = "../meilisearch-error", version = "0.19.0" }
+meilisearch-schema = { path = "../meilisearch-schema", version = "0.19.0" }
+meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.1.3" }
+meilisearch-types = { path = "../meilisearch-types", version = "0.19.0" }
+once_cell = "1.5.2"
+ordered-float = { version = "2.0.1", features = ["serde"] }
+pest = { git = "https://github.com/pest-parser/pest.git", rev = "51fd1d49f1041f7839975664ef71fe15c7dcaf67" }
+pest_derive = "2.1.0"
+regex = "1.4.2"
+sdset = "0.4.0"
+serde = { version = "1.0.118", features = ["derive"] }
+serde_json = { version = "1.0.61", features = ["preserve_order"] }
 slice-group-by = "0.2.6"
-zerocopy = "0.2.8"
-
-[dependencies.sdset]
-# version = "0.3.6"
-git = "https://github.com/Kerollmops/sdset"
-branch = "typed-algorithms"
+unicase = "2.6.0"
+zerocopy = "0.3.0"
 
 [dev-dependencies]
-assert_matches = "1.3"
-criterion = "0.3"
-csv = "1.0.7"
-indexmap = { version = "1.2.0", features = ["serde-1"] }
-jemallocator = "0.3.2"
-rustyline = { version = "5.0.0", default-features = false }
-structopt = "0.3.2"
+assert_matches = "1.4.0"
+criterion = "0.3.3"
+csv = "1.1.5"
+rustyline = { version = "7.1.0", default-features = false }
+structopt = "0.3.21"
 tempfile = "3.1.0"
-termcolor = "1.0.4"
-toml = "0.5.3"
+termcolor = "1.1.2"
 
-[[bench]]
-name = "search_benchmark"
-harness = false
+[target.'cfg(unix)'.dev-dependencies]
+jemallocator = "0.3.2"

meilisearch-core/benches/search_benchmark.rs

@@ -1,95 +0,0 @@
-#[cfg(test)]
-#[macro_use]
-extern crate assert_matches;
-
-use std::sync::mpsc;
-use std::path::Path;
-use std::fs;
-use std::iter;
-
-use meilisearch_core::Database;
-use meilisearch_core::{ProcessedUpdateResult, UpdateStatus};
-use serde_json::Value;
-
-use criterion::{criterion_group, criterion_main, Criterion, BenchmarkId};
-
-fn prepare_database(path: &Path) -> Database {
-    let database = Database::open_or_create(path).unwrap();
-    let db = &database;
-
-    let (sender, receiver) = mpsc::sync_channel(100);
-    let update_fn = move |_name: &str, update: ProcessedUpdateResult| {
-        sender.send(update.update_id).unwrap()
-    };
-    let index = database.create_index("bench").unwrap();
-
-    database.set_update_callback(Box::new(update_fn));
-
-    let schema = {
-        let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../datasets/movies/schema.toml");
-        let string = fs::read_to_string(path).expect("find schema");
-        toml::from_str(&string).unwrap()
-    };
-
-    let mut update_writer = db.update_write_txn().unwrap();
-    let _update_id = index.schema_update(&mut update_writer, schema).unwrap();
-    update_writer.commit().unwrap();
-
-    let mut additions = index.documents_addition();
-
-    let json: Value = {
-        let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../datasets/movies/movies.json");
-        let movies_file = fs::File::open(path).expect("find movies");
-        serde_json::from_reader(movies_file).unwrap()
-    };
-
-    let documents = json.as_array().unwrap();
-
-    for document in documents {
-        additions.update_document(document);
-    }
-
-    let mut update_writer = db.update_write_txn().unwrap();
-    let update_id = additions.finalize(&mut update_writer).unwrap();
-    update_writer.commit().unwrap();
-
-    // block until the transaction is processed
-    let _ = receiver.into_iter().find(|id| *id == update_id);
-
-    let update_reader = db.update_read_txn().unwrap();
-    let result = index.update_status(&update_reader, update_id).unwrap();
-    assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-
-    database
-}
-
-pub fn criterion_benchmark(c: &mut Criterion) {
-    let dir = tempfile::tempdir().unwrap();
-    let database = prepare_database(dir.path());
-
-    let reader = database.main_read_txn().unwrap();
-    let index = database.open_index("bench").unwrap();
-
-    let mut count = 0;
-    let query = "I love paris ";
-
-    let iter = iter::from_fn(|| {
-        count += 1;
-        query.get(0..count)
-    });
-
-    let mut group = c.benchmark_group("searching in movies (19654 docs)");
-    group.sample_size(10);
-
-    for query in iter {
-        let bench_name = BenchmarkId::from_parameter(format!("{:?}", query));
-        group.bench_with_input(bench_name, &query, |b, query| b.iter(|| {
-            let builder = index.query_builder();
-            builder.query(&reader, query, 0..20).unwrap();
-        }));
-    }
-    group.finish();
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

meilisearch-core/examples/from_file.rs

@@ -12,10 +12,11 @@ use serde::{Deserialize, Serialize};
 use structopt::StructOpt;
 use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
 
-use meilisearch_core::{Database, Highlight, ProcessedUpdateResult};
-use meilisearch_schema::SchemaAttr;
+use meilisearch_core::{Database, DatabaseOptions, Highlight, ProcessedUpdateResult};
+use meilisearch_core::settings::Settings;
+use meilisearch_schema::FieldId;
 
-// #[cfg(target_os = "linux")]
+#[cfg(target_os = "linux")]
 #[global_allocator]
 static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
 
@@ -32,9 +33,9 @@ struct IndexCommand {
     #[structopt(parse(from_os_str))]
     csv_data_path: PathBuf,
 
-    /// The path to the schema.
+    /// The path to the settings.
     #[structopt(long, parse(from_os_str))]
-    schema: PathBuf,
+    settings: PathBuf,
 
     #[structopt(long)]
     update_group_size: Option<usize>,
@@ -119,25 +120,13 @@ fn index_command(command: IndexCommand, database: Database) -> Result<(), Box<dy
 
     let db = &database;
 
-    let schema = {
-        let string = fs::read_to_string(&command.schema)?;
-        toml::from_str(&string).unwrap()
+    let settings = {
+        let string = fs::read_to_string(&command.settings)?;
+        let settings: Settings = serde_json::from_str(&string).unwrap();
+        settings.to_update().unwrap()
     };
 
-    let reader = db.main_read_txn().unwrap();
-    let mut update_writer = db.update_write_txn().unwrap();
-    match index.main.schema(&reader)? {
-        Some(current_schema) => {
-            if current_schema != schema {
-                return Err(meilisearch_core::Error::SchemaDiffer.into());
-            }
-            update_writer.abort();
-        }
-        None => {
-            index.schema_update(&mut update_writer, schema)?;
-            update_writer.commit().unwrap();
-        }
-    }
+    db.update_write(|w| index.settings_update(w, settings))?;
 
     let mut rdr = if command.csv_data_path.as_os_str() == "-" {
         csv::Reader::from_reader(Box::new(io::stdin()) as Box<dyn Read>)
@@ -184,10 +173,9 @@ fn index_command(command: IndexCommand, database: Database) -> Result<(), Box<dy
 
         println!();
 
-        let mut update_writer = db.update_write_txn().unwrap();
+        let update_id = db.update_write(|w| additions.finalize(w))?;
+
         println!("committing update...");
-        let update_id = additions.finalize(&mut update_writer)?;
-        update_writer.commit().unwrap();
         max_update_id = max_update_id.max(update_id);
         println!("committed update {}", update_id);
     }
@@ -334,7 +322,7 @@ fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<
 
     let reader = db.main_read_txn().unwrap();
     let schema = index.main.schema(&reader)?;
-    reader.abort();
+    reader.abort().unwrap();
 
     let schema = schema.ok_or(meilisearch_core::Error::SchemaMissing)?;
 
@@ -361,14 +349,14 @@ fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<
 
                 if let Some(ref filter) = command.filter {
                     let filter = filter.as_str();
-                    let (positive, filter) = if filter.chars().next() == Some('!') {
-                        (false, &filter[1..])
+                    let (positive, filter) = if let Some(stripped) = filter.strip_prefix('!') {
+                        (false, stripped)
                     } else {
                         (true, filter)
                     };
 
                     let attr = schema
-                        .attribute(&filter)
+                        .id(filter)
                         .expect("Could not find filtered attribute");
 
                     builder.with_filter(move |document_id| {
@@ -380,12 +368,12 @@ fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<
                     });
                 }
 
-                let documents = builder.query(ref_reader, &query, 0..command.number_results)?;
+                let result = builder.query(ref_reader, Some(&query), 0..command.number_results)?;
 
                 let mut retrieve_duration = Duration::default();
 
-                let number_of_documents = documents.len();
-                for mut doc in documents {
+                let number_of_documents = result.documents.len();
+                for mut doc in result.documents {
                     doc.highlights
                         .sort_unstable_by_key(|m| (m.char_index, m.char_length));
 
@@ -399,11 +387,11 @@ fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<
                             for (name, text) in document.0 {
                                 print!("{}: ", name);
 
-                                let attr = schema.attribute(&name).unwrap();
+                                let attr = schema.id(&name).unwrap();
                                 let highlights = doc
                                     .highlights
                                     .iter()
-                                    .filter(|m| SchemaAttr::new(m.attribute) == attr)
+                                    .filter(|m| FieldId::new(m.attribute) == attr)
                                     .cloned();
                                 let (text, highlights) =
                                     crop_text(&text, highlights, command.char_context);
@@ -418,8 +406,8 @@ fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<
 
                     let mut matching_attributes = HashSet::new();
                     for highlight in doc.highlights {
-                        let attr = SchemaAttr::new(highlight.attribute);
-                        let name = schema.attribute_name(attr);
+                        let attr = FieldId::new(highlight.attribute);
+                        let name = schema.name(attr);
                         matching_attributes.insert(name);
                     }
 
@@ -463,7 +451,7 @@ fn show_updates_command(
     let reader = db.update_read_txn().unwrap();
     let updates = index.all_updates_status(&reader)?;
     println!("{:#?}", updates);
-    reader.abort();
+    reader.abort().unwrap();
 
     Ok(())
 }
@@ -472,7 +460,7 @@ fn main() -> Result<(), Box<dyn Error>> {
     env_logger::init();
 
     let opt = Command::from_args();
-    let database = Database::open_or_create(opt.path())?;
+    let database = Database::open_or_create(opt.path(), DatabaseOptions::default())?;
 
     match opt {
         Command::Index(command) => index_command(command, database),

meilisearch-core/src/automaton/mod.rs

@@ -1,15 +1,4 @@
 mod dfa;
 
-use meilisearch_tokenizer::is_cjk;
-
 pub use self::dfa::{build_dfa, build_prefix_dfa, build_exact_dfa};
 
-pub fn normalize_str(string: &str) -> String {
-    let mut string = string.to_lowercase();
-
-    if !string.contains(is_cjk) {
-        string = deunicode::deunicode_with_tofu(&string, "");
-    }
-
-    string
-}

meilisearch-core/src/bucket_sort.rs

@@ -9,34 +9,42 @@ use std::time::Instant;
 use std::fmt;
 
 use compact_arena::{SmallArena, Idx32, mk_arena};
-use log::debug;
-use meilisearch_types::DocIndex;
-use sdset::{Set, SetBuf, exponential_search};
+use log::{debug, error};
+use sdset::{Set, SetBuf, exponential_search, SetOperation, Counter, duo::OpBuilder};
 use slice_group_by::{GroupBy, GroupByMut};
 
+use meilisearch_types::DocIndex;
+
 use crate::criterion::{Criteria, Context, ContextMut};
 use crate::distinct_map::{BufferedDistinctMap, DistinctMap};
 use crate::raw_document::RawDocument;
 use crate::{database::MainT, reordered_attrs::ReorderedAttrs};
-use crate::{store, Document, DocumentId, MResult};
+use crate::{store, Document, DocumentId, MResult, Index, RankedMap, MainReader, Error};
 use crate::query_tree::{create_query_tree, traverse_query_tree};
 use crate::query_tree::{Operation, QueryResult, QueryKind, QueryId, PostingsKey};
 use crate::query_tree::Context as QTContext;
 
+#[derive(Debug, Default)]
+pub struct SortResult {
+    pub documents: Vec<Document>,
+    pub nb_hits: usize,
+    pub exhaustive_nb_hit: bool,
+    pub facets: Option<HashMap<String, HashMap<String, usize>>>,
+    pub exhaustive_facets_count: Option<bool>,
+}
+
+#[allow(clippy::too_many_arguments)]
 pub fn bucket_sort<'c, FI>(
     reader: &heed::RoTxn<MainT>,
     query: &str,
     range: Range<usize>,
+    facets_docids: Option<SetBuf<DocumentId>>,
+    facet_count_docids: Option<HashMap<String, HashMap<String, (&str, Cow<Set<DocumentId>>)>>>,
     filter: Option<FI>,
     criteria: Criteria<'c>,
     searchable_attrs: Option<ReorderedAttrs>,
-    main_store: store::Main,
-    postings_lists_store: store::PostingsLists,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    synonyms_store: store::Synonyms,
-    prefix_documents_cache_store: store::PrefixDocumentsCache,
-    prefix_postings_lists_cache_store: store::PrefixPostingsListsCache,
-) -> MResult<Vec<Document>>
+    index: &Index,
+) -> MResult<SortResult>
 where
     FI: Fn(DocumentId) -> bool,
 {
@@ -49,30 +57,28 @@ where
             reader,
             query,
             range,
+            facets_docids,
+            facet_count_docids,
             filter,
             distinct,
             distinct_size,
             criteria,
             searchable_attrs,
-            main_store,
-            postings_lists_store,
-            documents_fields_counts_store,
-            synonyms_store,
-            prefix_documents_cache_store,
-            prefix_postings_lists_cache_store,
+            index,
         );
     }
 
-    let words_set = match unsafe { main_store.static_words_fst(reader)? } {
-        Some(words) => words,
-        None => return Ok(Vec::new()),
-    };
+    let mut result = SortResult::default();
+
+    let words_set = index.main.words_fst(reader)?;
+    let stop_words = index.main.stop_words_fst(reader)?;
 
     let context = QTContext {
         words_set,
-        synonyms: synonyms_store,
-        postings_lists: postings_lists_store,
-        prefix_postings_lists: prefix_postings_lists_cache_store,
+        stop_words,
+        synonyms: index.synonyms,
+        postings_lists: index.postings_lists,
+        prefix_postings_lists: index.prefix_postings_lists_cache,
     };
 
     let (operation, mapping) = create_query_tree(reader, &context, query)?;
@@ -90,14 +96,27 @@ where
     let mut queries_kinds = HashMap::new();
     recurs_operation(&mut queries_kinds, &operation);
 
-    let QueryResult { docids, queries } = traverse_query_tree(reader, &context, &operation)?;
-    println!("found {} documents", docids.len());
-    println!("number of postings {:?}", queries.len());
+    let QueryResult { mut docids, queries } = traverse_query_tree(reader, &context, &operation)?;
+    debug!("found {} documents", docids.len());
+    debug!("number of postings {:?}", queries.len());
+
+    if let Some(facets_docids) = facets_docids {
+        let intersection = sdset::duo::OpBuilder::new(docids.as_ref(), facets_docids.as_set())
+            .intersection()
+            .into_set_buf();
+        docids = Cow::Owned(intersection);
+    }
+
+    if let Some(f) = facet_count_docids {
+        // hardcoded value, until approximation optimization
+        result.exhaustive_facets_count = Some(true);
+        result.facets = Some(facet_count(f, &docids));
+    }
 
     let before = Instant::now();
     mk_arena!(arena);
     let mut bare_matches = cleanup_bare_matches(&mut arena, &docids, queries);
-    println!("matches cleaned in {:.02?}", before.elapsed());
+    debug!("matches cleaned in {:.02?}", before.elapsed());
 
     let before_bucket_sort = Instant::now();
 
@@ -128,7 +147,7 @@ where
                 reader,
                 postings_lists: &mut arena,
                 query_mapping: &mapping,
-                documents_fields_counts_store,
+                documents_fields_counts_store: index.documents_fields_counts,
             };
 
             criterion.prepare(ctx, &mut group)?;
@@ -161,45 +180,49 @@ where
     debug!("criterion loop took {:.02?}", before_criterion_loop.elapsed());
     debug!("proximity evaluation called {} times", proximity_count.load(Ordering::Relaxed));
 
+    let schema = index.main.schema(reader)?.ok_or(Error::SchemaMissing)?;
     let iter = raw_documents.into_iter().skip(range.start).take(range.len());
-    let iter = iter.map(|rd| Document::from_raw(rd, &queries_kinds, &arena, searchable_attrs.as_ref()));
+    let iter = iter.map(|rd| Document::from_raw(rd, &queries_kinds, &arena, searchable_attrs.as_ref(), &schema));
     let documents = iter.collect();
 
     debug!("bucket sort took {:.02?}", before_bucket_sort.elapsed());
 
-    Ok(documents)
+    result.documents = documents;
+    result.nb_hits = docids.len();
+
+    Ok(result)
 }
 
+#[allow(clippy::too_many_arguments)]
 pub fn bucket_sort_with_distinct<'c, FI, FD>(
     reader: &heed::RoTxn<MainT>,
     query: &str,
     range: Range<usize>,
+    facets_docids: Option<SetBuf<DocumentId>>,
+    facet_count_docids: Option<HashMap<String, HashMap<String, (&str, Cow<Set<DocumentId>>)>>>,
     filter: Option<FI>,
     distinct: FD,
     distinct_size: usize,
     criteria: Criteria<'c>,
     searchable_attrs: Option<ReorderedAttrs>,
-    main_store: store::Main,
-    postings_lists_store: store::PostingsLists,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    synonyms_store: store::Synonyms,
-    _prefix_documents_cache_store: store::PrefixDocumentsCache,
-    prefix_postings_lists_cache_store: store::PrefixPostingsListsCache,
-) -> MResult<Vec<Document>>
+    index: &Index,
+) -> MResult<SortResult>
 where
     FI: Fn(DocumentId) -> bool,
     FD: Fn(DocumentId) -> Option<u64>,
 {
-    let words_set = match unsafe { main_store.static_words_fst(reader)? } {
-        Some(words) => words,
-        None => return Ok(Vec::new()),
-    };
+    let mut result = SortResult::default();
+    let mut filtered_count = 0;
+
+    let words_set = index.main.words_fst(reader)?;
+    let stop_words = index.main.stop_words_fst(reader)?;
 
     let context = QTContext {
         words_set,
-        synonyms: synonyms_store,
-        postings_lists: postings_lists_store,
-        prefix_postings_lists: prefix_postings_lists_cache_store,
+        stop_words,
+        synonyms: index.synonyms,
+        postings_lists: index.postings_lists,
+        prefix_postings_lists: index.prefix_postings_lists_cache,
     };
 
     let (operation, mapping) = create_query_tree(reader, &context, query)?;
@@ -217,14 +240,27 @@ where
     let mut queries_kinds = HashMap::new();
     recurs_operation(&mut queries_kinds, &operation);
 
-    let QueryResult { docids, queries } = traverse_query_tree(reader, &context, &operation)?;
-    println!("found {} documents", docids.len());
-    println!("number of postings {:?}", queries.len());
+    let QueryResult { mut docids, queries } = traverse_query_tree(reader, &context, &operation)?;
+    debug!("found {} documents", docids.len());
+    debug!("number of postings {:?}", queries.len());
+
+    if let Some(facets_docids) = facets_docids {
+        let intersection = OpBuilder::new(docids.as_ref(), facets_docids.as_set())
+            .intersection()
+            .into_set_buf();
+        docids = Cow::Owned(intersection);
+    }
+
+    if let Some(f) = facet_count_docids {
+        // hardcoded value, until approximation optimization
+        result.exhaustive_facets_count = Some(true);
+        result.facets = Some(facet_count(f, &docids));
+    }
 
     let before = Instant::now();
     mk_arena!(arena);
     let mut bare_matches = cleanup_bare_matches(&mut arena, &docids, queries);
-    println!("matches cleaned in {:.02?}", before.elapsed());
+    debug!("matches cleaned in {:.02?}", before.elapsed());
 
     let before_raw_documents_building = Instant::now();
     let mut raw_documents = Vec::new();
@@ -265,7 +301,7 @@ where
                 reader,
                 postings_lists: &mut arena,
                 query_mapping: &mapping,
-                documents_fields_counts_store,
+                documents_fields_counts_store: index.documents_fields_counts,
             };
 
             let before_criterion_preparation = Instant::now();
@@ -287,19 +323,36 @@ where
                     let filter_accepted = match &filter {
                         Some(filter) => {
                             let entry = filter_map.entry(document.id);
-                            *entry.or_insert_with(|| (filter)(document.id))
+                            *entry.or_insert_with(|| {
+                                let accepted = (filter)(document.id);
+                                // we only want to count it out the first time we see it
+                                if !accepted {
+                                    filtered_count += 1;
+                                }
+                                accepted
+                            })
                         }
                         None => true,
                     };
 
                     if filter_accepted {
                         let entry = key_cache.entry(document.id);
-                        let key = entry.or_insert_with(|| (distinct)(document.id).map(Rc::new));
+                        let mut seen = true;
+                        let key = entry.or_insert_with(|| {
+                            seen = false;
+                            (distinct)(document.id).map(Rc::new)
+                        });
 
-                        match key.clone() {
+                        let distinct = match key.clone() {
                             Some(key) => buf_distinct.register(key),
                             None => buf_distinct.register_without_key(),
                         };
+
+                        // we only want to count the document if it is the first time we see it and
+                        // if it wasn't accepted by distinct
+                        if !seen && !distinct {
+                            filtered_count += 1;
+                        }
                     }
 
                     // the requested range end is reached: stop computing distinct
@@ -330,31 +383,40 @@ where
     // once we classified the documents related to the current
     // automatons we save that as the next valid result
     let mut seen = BufferedDistinctMap::new(&mut distinct_map);
+    let schema = index.main.schema(reader)?.ok_or(Error::SchemaMissing)?;
 
     let mut documents = Vec::with_capacity(range.len());
     for raw_document in raw_documents.into_iter().skip(distinct_raw_offset) {
         let filter_accepted = match &filter {
-            Some(_) => filter_map.remove(&raw_document.id).unwrap(),
+            Some(_) => filter_map.remove(&raw_document.id).unwrap_or_else(|| {
+                error!("error during filtering: expected value for document id {}", &raw_document.id.0);
+                Default::default()
+            }),
             None => true,
         };
 
         if filter_accepted {
-            let key = key_cache.remove(&raw_document.id).unwrap();
+            let key = key_cache.remove(&raw_document.id).unwrap_or_else(|| {
+                error!("error during distinct: expected value for document id {}", &raw_document.id.0);
+                Default::default()
+            });
             let distinct_accepted = match key {
                 Some(key) => seen.register(key),
                 None => seen.register_without_key(),
             };
 
             if distinct_accepted && seen.len() > range.start {
-                documents.push(Document::from_raw(raw_document, &queries_kinds, &arena, searchable_attrs.as_ref()));
+                documents.push(Document::from_raw(raw_document, &queries_kinds, &arena, searchable_attrs.as_ref(), &schema));
                 if documents.len() == range.len() {
                     break;
                 }
             }
         }
     }
+    result.documents = documents;
+    result.nb_hits = docids.len() - filtered_count;
 
-    Ok(documents)
+    Ok(result)
 }
 
 fn cleanup_bare_matches<'tag, 'txn>(
@@ -549,3 +611,69 @@ impl Deref for PostingsListView<'_> {
         }
     }
 }
+
+/// sorts documents ids according to user defined ranking rules.
+pub fn placeholder_document_sort(
+    document_ids: &mut [DocumentId],
+    index: &store::Index,
+    reader: &MainReader,
+    ranked_map: &RankedMap
+) -> MResult<()> {
+    use crate::settings::RankingRule;
+    use std::cmp::Ordering;
+
+    enum SortOrder {
+        Asc,
+        Desc,
+    }
+
+    if let Some(ranking_rules) = index.main.ranking_rules(reader)? {
+        let schema = index.main.schema(reader)?
+            .ok_or(Error::SchemaMissing)?;
+
+        // Select custom rules from ranking rules, and map them to custom rules
+        // containing a field_id
+        let ranking_rules = ranking_rules.iter().filter_map(|r|
+            match r {
+                RankingRule::Asc(name) => schema.id(name).map(|f| (f, SortOrder::Asc)),
+                RankingRule::Desc(name) => schema.id(name).map(|f| (f, SortOrder::Desc)),
+                _ => None,
+            }).collect::<Vec<_>>();
+
+        document_ids.sort_unstable_by(|a, b| {
+            for (field_id, order) in &ranking_rules {
+                let a_value = ranked_map.get(*a, *field_id);
+                let b_value = ranked_map.get(*b, *field_id);
+                let (a, b) = match order {
+                    SortOrder::Asc => (a_value, b_value),
+                    SortOrder::Desc => (b_value, a_value),
+                };
+                match a.cmp(&b) {
+                    Ordering::Equal => continue,
+                    ordering => return ordering,
+                }
+            }
+            Ordering::Equal
+        });
+    }
+    Ok(())
+}
+
+/// For each entry in facet_docids, calculates the number of documents in the intersection with candidate_docids.
+pub fn facet_count(
+    facet_docids: HashMap<String, HashMap<String, (&str, Cow<Set<DocumentId>>)>>,
+    candidate_docids: &Set<DocumentId>,
+) -> HashMap<String, HashMap<String, usize>> {
+    let mut facets_counts = HashMap::with_capacity(facet_docids.len());
+    for (key, doc_map) in facet_docids {
+        let mut count_map = HashMap::with_capacity(doc_map.len());
+        for (_, (value, docids)) in doc_map {
+            let mut counter = Counter::new();
+            let op = OpBuilder::new(docids.as_ref(), candidate_docids).intersection();
+            SetOperation::<DocumentId>::extend_collection(op, &mut counter);
+            count_map.insert(value.to_string(), counter.0);
+        }
+        facets_counts.insert(key, count_map);
+    }
+    facets_counts
+}

meilisearch-core/src/criterion/exactness.rs

@@ -1,15 +1,15 @@
 use std::cmp::{Ordering, Reverse};
 use std::collections::hash_map::{HashMap, Entry};
-use meilisearch_schema::SchemaAttr;
+use meilisearch_schema::IndexedPos;
 use slice_group_by::GroupBy;
 use crate::{RawDocument, MResult};
 use crate::bucket_sort::BareMatch;
 use super::{Criterion, Context, ContextMut};
 
-pub struct Exact;
+pub struct Exactness;
 
-impl Criterion for Exact {
-    fn name(&self) -> &str { "exact" }
+impl Criterion for Exactness {
+    fn name(&self) -> &str { "exactness" }
 
     fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>(
         &self,
@@ -32,7 +32,7 @@ impl Criterion for Exact {
                     for bm in group {
                         for di in ctx.postings_lists[bm.postings_list].as_ref() {
 
-                            let attr = SchemaAttr(di.attribute);
+                            let attr = IndexedPos(di.attribute);
                             let count = match fields_counts.entry(attr) {
                                 Entry::Occupied(entry) => *entry.get(),
                                 Entry::Vacant(entry) => {

meilisearch-core/src/criterion/mod.rs

@@ -16,7 +16,7 @@ mod words;
 mod proximity;
 mod attribute;
 mod words_position;
-mod exact;
+mod exactness;
 mod document_id;
 mod sort_by_attr;
 
@@ -25,7 +25,7 @@ pub use self::words::Words;
 pub use self::proximity::Proximity;
 pub use self::attribute::Attribute;
 pub use self::words_position::WordsPosition;
-pub use self::exact::Exact;
+pub use self::exactness::Exactness;
 pub use self::document_id::DocumentId;
 pub use self::sort_by_attr::SortByAttr;
 
@@ -61,7 +61,7 @@ pub trait Criterion {
 }
 
 pub struct ContextMut<'h, 'p, 'tag, 'txn, 'q> {
-    pub reader: &'h heed::RoTxn<MainT>,
+    pub reader: &'h heed::RoTxn<'h, MainT>,
     pub postings_lists: &'p mut SmallArena<'tag, PostingsListView<'txn>>,
     pub query_mapping: &'q HashMap<QueryId, Range<usize>>,
     pub documents_fields_counts_store: store::DocumentsFieldsCounts,
@@ -92,6 +92,7 @@ impl<'a> CriteriaBuilder<'a> {
         self.inner.reserve(additional)
     }
 
+    #[allow(clippy::should_implement_trait)]
     pub fn add<C: 'a>(mut self, criterion: C) -> CriteriaBuilder<'a>
     where
         C: Criterion,
@@ -124,7 +125,7 @@ impl<'a> Default for Criteria<'a> {
             .add(Proximity)
             .add(Attribute)
             .add(WordsPosition)
-            .add(Exact)
+            .add(Exactness)
             .add(DocumentId)
             .build()
     }

meilisearch-core/src/criterion/sort_by_attr.rs

@@ -1,7 +1,7 @@
 use std::cmp::Ordering;
 use std::error::Error;
 use std::fmt;
-use meilisearch_schema::{Schema, SchemaAttr};
+use meilisearch_schema::{Schema, FieldId};
 use crate::{RankedMap, RawDocument};
 use super::{Criterion, Context};
 
@@ -32,7 +32,7 @@ use super::{Criterion, Context};
 ///        .add(Proximity)
 ///        .add(Attribute)
 ///        .add(WordsPosition)
-///        .add(Exact)
+///        .add(Exactness)
 ///        .add(custom_ranking)
 ///        .add(DocumentId);
 ///
@@ -41,7 +41,7 @@ use super::{Criterion, Context};
 /// ```
 pub struct SortByAttr<'a> {
     ranked_map: &'a RankedMap,
-    attr: SchemaAttr,
+    field_id: FieldId,
     reversed: bool,
 }
 
@@ -68,18 +68,18 @@ impl<'a> SortByAttr<'a> {
         attr_name: &str,
         reversed: bool,
     ) -> Result<SortByAttr<'a>, SortByAttrError> {
-        let attr = match schema.attribute(attr_name) {
-            Some(attr) => attr,
+        let field_id = match schema.id(attr_name) {
+            Some(field_id) => field_id,
             None => return Err(SortByAttrError::AttributeNotFound),
         };
 
-        if !schema.props(attr).is_ranked() {
+        if !schema.is_ranked(field_id) {
             return Err(SortByAttrError::AttributeNotRegisteredForRanking);
         }
 
         Ok(SortByAttr {
             ranked_map,
-            attr,
+            field_id,
             reversed,
         })
     }
@@ -91,8 +91,8 @@ impl Criterion for SortByAttr<'_> {
     }
 
     fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = self.ranked_map.get(lhs.id, self.attr);
-        let rhs = self.ranked_map.get(rhs.id, self.attr);
+        let lhs = self.ranked_map.get(lhs.id, self.field_id);
+        let rhs = self.ranked_map.get(rhs.id, self.field_id);
 
         match (lhs, rhs) {
             (Some(lhs), Some(rhs)) => {

meilisearch-core/src/criterion/typo.rs

@@ -22,6 +22,7 @@ impl Criterion for Typo {
         // It is safe to panic on input number higher than 3,
         // the number of typos is never bigger than that.
         #[inline]
+        #[allow(clippy::approx_constant)]
         fn custom_log10(n: u8) -> f32 {
             match n {
                 0 => 0.0,     // log(1)

meilisearch-core/src/database.rs

@@ -1,19 +1,35 @@
 use std::collections::hash_map::{Entry, HashMap};
+use std::collections::BTreeMap;
 use std::fs::File;
 use std::path::Path;
 use std::sync::{Arc, RwLock};
 use std::{fs, thread};
+use std::io::{Read, Write, ErrorKind};
 
+use chrono::{DateTime, Utc};
 use crossbeam_channel::{Receiver, Sender};
-use heed::types::{Str, Unit};
-use heed::{CompactionOption, Result as ZResult};
-use log::debug;
+use heed::CompactionOption;
+use heed::types::{Str, Unit, SerdeBincode};
+use log::{debug, error};
+use meilisearch_schema::Schema;
+use regex::Regex;
 
-use crate::{store, update, Index, MResult};
+use crate::{store, update, Index, MResult, Error};
 
 pub type BoxUpdateFn = Box<dyn Fn(&str, update::ProcessedUpdateResult) + Send + Sync + 'static>;
+
 type ArcSwapFn = arc_swap::ArcSwapOption<BoxUpdateFn>;
 
+type SerdeDatetime = SerdeBincode<DateTime<Utc>>;
+
+pub type MainWriter<'a, 'b> = heed::RwTxn<'a, 'b, MainT>;
+pub type MainReader<'a, 'b> = heed::RoTxn<'a, MainT>;
+
+pub type UpdateWriter<'a, 'b> = heed::RwTxn<'a, 'b, UpdateT>;
+pub type UpdateReader<'a> = heed::RoTxn<'a, UpdateT>;
+
+const LAST_UPDATE_KEY: &str = "last-update";
+
 pub struct MainT;
 pub struct UpdateT;
 
@@ -24,6 +40,21 @@ pub struct Database {
     indexes_store: heed::Database<Str, Unit>,
     indexes: RwLock<HashMap<String, (Index, thread::JoinHandle<MResult<()>>)>>,
     update_fn: Arc<ArcSwapFn>,
+    database_version: (u32, u32, u32),
+}
+
+pub struct DatabaseOptions {
+    pub main_map_size: usize,
+    pub update_map_size: usize,
+}
+
+impl Default for DatabaseOptions {
+    fn default() -> DatabaseOptions {
+        DatabaseOptions {
+            main_map_size: 100 * 1024 * 1024 * 1024, //100Gb
+            update_map_size: 100 * 1024 * 1024 * 1024, //100Gb
+        }
+    }
 }
 
 macro_rules! r#break_try {
@@ -54,8 +85,7 @@ fn update_awaiter(
     update_fn: Arc<ArcSwapFn>,
     index: Index,
 ) -> MResult<()> {
-    let mut receiver = receiver.into_iter();
-    while let Some(event) = receiver.next() {
+    for event in receiver {
 
         // if we receive a *MustClear* event, clear the index and break the loop
         if let UpdateEvent::MustClear = event {
@@ -89,7 +119,7 @@ fn update_awaiter(
             };
 
             // do not keep the reader for too long
-            update_reader.abort();
+            break_try!(update_reader.abort(), "aborting update transaction failed");
 
             // instantiate a transaction to touch to the main env
             let result = env.typed_write_txn::<MainT>();
@@ -103,7 +133,7 @@ fn update_awaiter(
             if status.error.is_none() {
                 break_try!(main_writer.commit(), "commit nested transaction failed");
             } else {
-                main_writer.abort()
+                break_try!(main_writer.abort(), "abborting nested transaction failed");
             }
 
             // now that the update has been processed we can instantiate
@@ -134,20 +164,90 @@ fn update_awaiter(
     Ok(())
 }
 
+/// Ensures Meilisearch version is compatible with the database, returns an error versions mismatch.
+/// If create is set to true, a VERSION file is created with the current version.
+fn version_guard(path: &Path, create: bool) -> MResult<(u32, u32, u32)> {
+    let current_version_major = env!("CARGO_PKG_VERSION_MAJOR");
+    let current_version_minor = env!("CARGO_PKG_VERSION_MINOR");
+    let current_version_patch = env!("CARGO_PKG_VERSION_PATCH");
+    let version_path = path.join("VERSION");
+
+    match File::open(&version_path) {
+        Ok(mut file) => {
+            let mut version = String::new();
+            file.read_to_string(&mut version)?;
+            // Matches strings like XX.XX.XX
+            let re = Regex::new(r"(\d+).(\d+).(\d+)").unwrap();
+
+            // Make sure there is a result
+            let version = re
+                .captures_iter(&version)
+                .next()
+                .ok_or_else(|| Error::VersionMismatch("bad VERSION file".to_string()))?;
+            // the first is always the complete match, safe to unwrap because we have a match
+            let version_major = version.get(1).unwrap().as_str();
+            let version_minor = version.get(2).unwrap().as_str();
+            let version_patch = version.get(3).unwrap().as_str();
+
+            if version_major != current_version_major || version_minor != current_version_minor {
+                Err(Error::VersionMismatch(format!("{}.{}.XX", version_major, version_minor)))
+            } else {
+                Ok((
+                    version_major.parse().map_err(|e| Error::VersionMismatch(format!("error parsing database version: {}", e)))?,
+                    version_minor.parse().map_err(|e| Error::VersionMismatch(format!("error parsing database version: {}", e)))?,
+                    version_patch.parse().map_err(|e| Error::VersionMismatch(format!("error parsing database version: {}", e)))?
+                ))
+            }
+        }
+        Err(error) => {
+            match error.kind() {
+                ErrorKind::NotFound => {
+                    if create {
+                        // when no version file is found, and we've been told to create one,
+                        // create a new file with the current version in it.
+                        let mut version_file = File::create(&version_path)?;
+                        version_file.write_all(format!("{}.{}.{}",
+                                current_version_major,
+                                current_version_minor,
+                                current_version_patch).as_bytes())?;
+
+                        Ok((
+                            current_version_major.parse().map_err(|e| Error::VersionMismatch(format!("error parsing database version: {}", e)))?,
+                            current_version_minor.parse().map_err(|e| Error::VersionMismatch(format!("error parsing database version: {}", e)))?,
+                            current_version_patch.parse().map_err(|e| Error::VersionMismatch(format!("error parsing database version: {}", e)))?
+                        ))
+                    } else {
+                        // when no version file is found and we were not told to create one, this
+                        // means that the version is inferior to the one this feature was added in.
+                        Err(Error::VersionMismatch("<0.12.0".to_string()))
+                    }
+                }
+                _ => Err(error.into())
+            }
+        }
+    }
+}
+
 impl Database {
-    pub fn open_or_create(path: impl AsRef<Path>) -> MResult<Database> {
+    pub fn open_or_create(path: impl AsRef<Path>, options: DatabaseOptions) -> MResult<Database> {
         let main_path = path.as_ref().join("main");
         let update_path = path.as_ref().join("update");
 
+        //create db directory
+        fs::create_dir_all(&path)?;
+
+        // create file only if main db wasn't created before (first run)
+        let database_version = version_guard(path.as_ref(), !main_path.exists() && !update_path.exists())?;
+
         fs::create_dir_all(&main_path)?;
         let env = heed::EnvOpenOptions::new()
-            .map_size(100 * 1024 * 1024 * 1024) // 100GB
+            .map_size(options.main_map_size)
             .max_dbs(3000)
             .open(main_path)?;
 
         fs::create_dir_all(&update_path)?;
         let update_env = heed::EnvOpenOptions::new()
-            .map_size(100 * 1024 * 1024 * 1024) // 100GB
+            .map_size(options.update_map_size)
             .max_dbs(3000)
             .open(update_path)?;
 
@@ -163,7 +263,7 @@ impl Database {
             must_open.push(index_uid.to_owned());
         }
 
-        reader.abort();
+        reader.abort()?;
 
         // open the previously aggregated indexes
         let mut indexes = HashMap::new();
@@ -215,6 +315,7 @@ impl Database {
             indexes_store,
             indexes: RwLock::new(indexes),
             update_fn,
+            database_version,
         })
     }
 
@@ -226,6 +327,13 @@ impl Database {
         }
     }
 
+    pub fn is_indexing(&self, reader: &UpdateReader, index: &str) -> MResult<Option<bool>> {
+        match self.open_index(&index) {
+            Some(index) => index.current_update_id(&reader).map(|u| Some(u.is_some())),
+            None => Ok(None),
+        }
+    }
+
     pub fn create_index(&self, name: impl AsRef<str>) -> MResult<Index> {
         let name = name.as_ref();
         let mut indexes_lock = self.indexes.write().unwrap();
@@ -242,6 +350,7 @@ impl Database {
                 index.main.put_name(&mut writer, name)?;
                 index.main.put_created_at(&mut writer)?;
                 index.main.put_updated_at(&mut writer)?;
+                index.main.put_schema(&mut writer, &Schema::default())?;
 
                 let env_clone = self.env.clone();
                 let update_env_clone = self.update_env.clone();
@@ -303,24 +412,101 @@ impl Database {
         self.update_fn.swap(None);
     }
 
-    pub fn main_read_txn(&self) -> heed::Result<heed::RoTxn<MainT>> {
-        self.env.typed_read_txn::<MainT>()
+    pub fn main_read_txn(&self) -> MResult<MainReader> {
+        Ok(self.env.typed_read_txn::<MainT>()?)
     }
 
-    pub fn main_write_txn(&self) -> heed::Result<heed::RwTxn<MainT>> {
-        self.env.typed_write_txn::<MainT>()
+    pub(crate) fn main_write_txn(&self) -> MResult<MainWriter> {
+        Ok(self.env.typed_write_txn::<MainT>()?)
     }
 
-    pub fn update_read_txn(&self) -> heed::Result<heed::RoTxn<UpdateT>> {
-        self.update_env.typed_read_txn::<UpdateT>()
+    /// Calls f providing it with a writer to the main database. After f is called, makes sure the
+    /// transaction is commited. Returns whatever result f returns.
+    pub fn main_write<F, R, E>(&self, f: F) -> Result<R, E>
+    where
+        F: FnOnce(&mut MainWriter) -> Result<R, E>,
+        E: From<Error>,
+    {
+        let mut writer = self.main_write_txn()?;
+        let result = f(&mut writer)?;
+        writer.commit().map_err(Error::Heed)?;
+        Ok(result)
     }
 
-    pub fn update_write_txn(&self) -> heed::Result<heed::RwTxn<UpdateT>> {
-        self.update_env.typed_write_txn::<UpdateT>()
+    /// provides a context with a reader to the main database. experimental.
+    pub fn main_read<F, R, E>(&self, f: F) -> Result<R, E>
+    where
+        F: FnOnce(&MainReader) -> Result<R, E>,
+        E: From<Error>,
+    {
+        let reader = self.main_read_txn()?;
+        let result = f(&reader)?;
+        reader.abort().map_err(Error::Heed)?;
+        Ok(result)
     }
 
-    pub fn copy_and_compact_to_path<P: AsRef<Path>>(&self, path: P) -> ZResult<File> {
-        self.env.copy_to_path(path, CompactionOption::Enabled)
+    pub fn update_read_txn(&self) -> MResult<UpdateReader> {
+        Ok(self.update_env.typed_read_txn::<UpdateT>()?)
+    }
+
+    pub(crate) fn update_write_txn(&self) -> MResult<heed::RwTxn<UpdateT>> {
+        Ok(self.update_env.typed_write_txn::<UpdateT>()?)
+    }
+
+    /// Calls f providing it with a writer to the main database. After f is called, makes sure the
+    /// transaction is commited. Returns whatever result f returns.
+    pub fn update_write<F, R, E>(&self, f: F) -> Result<R, E>
+    where
+        F: FnOnce(&mut UpdateWriter) -> Result<R, E>,
+        E: From<Error>,
+    {
+        let mut writer = self.update_write_txn()?;
+        let result = f(&mut writer)?;
+        writer.commit().map_err(Error::Heed)?;
+        Ok(result)
+    }
+
+    /// provides a context with a reader to the update database. experimental.
+    pub fn update_read<F, R, E>(&self, f: F) -> Result<R, E>
+    where
+        F: FnOnce(&UpdateReader) -> Result<R, E>,
+        E: From<Error>,
+    {
+        let reader = self.update_read_txn()?;
+        let result = f(&reader)?;
+        reader.abort().map_err(Error::Heed)?;
+        Ok(result)
+    }
+
+    pub fn copy_and_compact_to_path<P: AsRef<Path>>(&self, path: P) -> MResult<(File, File)> {
+        let path = path.as_ref();
+
+        let env_path = path.join("main");
+        let env_update_path = path.join("update");
+        let env_version_path = path.join("VERSION");
+
+        fs::create_dir(&env_path)?;
+        fs::create_dir(&env_update_path)?;
+    
+        // write Database Version
+        let (current_version_major, current_version_minor, current_version_patch) = self.database_version;
+        let mut version_file = File::create(&env_version_path)?;
+        version_file.write_all(format!("{}.{}.{}",
+                current_version_major,
+                current_version_minor,
+                current_version_patch).as_bytes())?;
+
+        let env_path = env_path.join("data.mdb");
+        let env_file = self.env.copy_to_path(&env_path, CompactionOption::Enabled)?;
+
+        let env_update_path = env_update_path.join("data.mdb");
+        match self.update_env.copy_to_path(env_update_path, CompactionOption::Enabled) {
+            Ok(update_env_file) => Ok((env_file, update_env_file)),
+            Err(e) => {
+                fs::remove_file(env_path)?;
+                Err(e.into())
+            },
+        }
     }
 
     pub fn indexes_uids(&self) -> Vec<String> {
@@ -328,17 +514,73 @@ impl Database {
         indexes.keys().cloned().collect()
     }
 
-    pub fn common_store(&self) -> heed::PolyDatabase {
+    pub(crate) fn common_store(&self) -> heed::PolyDatabase {
         self.common_store
     }
+
+    pub fn last_update(&self, reader: &heed::RoTxn<MainT>) -> MResult<Option<DateTime<Utc>>> {
+        match self.common_store()
+            .get::<_, Str, SerdeDatetime>(reader, LAST_UPDATE_KEY)? {
+                Some(datetime) => Ok(Some(datetime)),
+                None => Ok(None),
+            }
+    }
+
+    pub fn set_last_update(&self, writer: &mut heed::RwTxn<MainT>, time: &DateTime<Utc>) -> MResult<()> {
+        self.common_store()
+            .put::<_, Str, SerdeDatetime>(writer, LAST_UPDATE_KEY, time)?;
+        Ok(())
+    }
+
+    pub fn compute_stats(&self, writer: &mut MainWriter, index_uid: &str) -> MResult<()> {
+        let index = match self.open_index(&index_uid) {
+            Some(index) => index,
+            None => {
+                error!("Impossible to retrieve index {}", index_uid);
+                return Ok(());
+            }
+        };
+
+        let schema = match index.main.schema(&writer)? {
+            Some(schema) => schema,
+            None => return Ok(()),
+        };
+
+        let all_documents_fields = index
+            .documents_fields_counts
+            .all_documents_fields_counts(&writer)?;
+
+        // count fields frequencies
+        let mut fields_frequency = HashMap::<_, usize>::new();
+        for result in all_documents_fields {
+            let (_, attr, _) = result?;
+            if let Some(field_id) = schema.indexed_pos_to_field_id(attr) {
+                *fields_frequency.entry(field_id).or_default() += 1;
+            }
+        }
+
+        // convert attributes to their names
+        let frequency: BTreeMap<_, _> = fields_frequency
+            .into_iter()
+            .filter_map(|(a, c)| schema.name(a).map(|name| (name.to_string(), c)))
+            .collect();
+
+        index
+            .main
+            .put_fields_distribution(writer, &frequency)
+    }
+
+    pub fn version(&self) -> (u32, u32, u32) { self.database_version }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    use crate::bucket_sort::SortResult;
     use crate::criterion::{self, CriteriaBuilder};
     use crate::update::{ProcessedUpdateResult, UpdateStatus};
+    use crate::settings::Settings;
     use crate::{Document, DocumentId};
     use serde::de::IgnoredAny;
     use std::sync::mpsc;
@@ -347,7 +589,7 @@ mod tests {
     fn valid_updates() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -358,23 +600,25 @@ mod tests {
 
         database.set_update_callback(Box::new(update_fn));
 
-        let schema = {
+        let mut writer = db.main_write_txn().unwrap();
+        index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
+        writer.commit().unwrap();
+
+        // block until the transaction is processed
+
+        let settings = {
             let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
+                {
+                    "searchableAttributes": ["name", "description"],
+                    "displayedAttributes": ["name", "description"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.to_update().unwrap()
         };
 
         let mut update_writer = db.update_write_txn().unwrap();
-        let _update_id = index.schema_update(&mut update_writer, schema).unwrap();
+        let _update_id = index.settings_update(&mut update_writer, settings).unwrap();
         update_writer.commit().unwrap();
 
         let mut additions = index.documents_addition();
@@ -410,7 +654,7 @@ mod tests {
     fn invalid_updates() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -421,23 +665,23 @@ mod tests {
 
         database.set_update_callback(Box::new(update_fn));
 
-        let schema = {
+        let mut writer = db.main_write_txn().unwrap();
+        index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
+        writer.commit().unwrap();
+
+        let settings = {
             let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
+                {
+                    "searchableAttributes": ["name", "description"],
+                    "displayedAttributes": ["name", "description"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.to_update().unwrap()
         };
 
         let mut update_writer = db.update_write_txn().unwrap();
-        let _update_id = index.schema_update(&mut update_writer, schema).unwrap();
+        let _update_id = index.settings_update(&mut update_writer, settings).unwrap();
         update_writer.commit().unwrap();
 
         let mut additions = index.documents_addition();
@@ -472,7 +716,7 @@ mod tests {
     fn ignored_words_too_long() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -483,19 +727,23 @@ mod tests {
 
         database.set_update_callback(Box::new(update_fn));
 
-        let schema = {
-            let data = r#"
-                identifier = "id"
+        let mut writer = db.main_write_txn().unwrap();
+        index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
+        writer.commit().unwrap();
 
-                [attributes."name"]
-                displayed = true
-                indexed = true
+        let settings = {
+            let data = r#"
+                {
+                    "searchableAttributes": ["name"],
+                    "displayedAttributes": ["name"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.to_update().unwrap()
         };
 
         let mut update_writer = db.update_write_txn().unwrap();
-        let _update_id = index.schema_update(&mut update_writer, schema).unwrap();
+        let _update_id = index.settings_update(&mut update_writer, settings).unwrap();
         update_writer.commit().unwrap();
 
         let mut additions = index.documents_addition();
@@ -523,7 +771,7 @@ mod tests {
     fn add_schema_attributes_at_end() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -534,23 +782,23 @@ mod tests {
 
         database.set_update_callback(Box::new(update_fn));
 
-        let schema = {
+        let mut writer = db.main_write_txn().unwrap();
+        index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
+        writer.commit().unwrap();
+
+        let settings = {
             let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
+                {
+                    "searchableAttributes": ["name", "description"],
+                    "displayedAttributes": ["name", "description"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.to_update().unwrap()
         };
 
         let mut update_writer = db.update_write_txn().unwrap();
-        let _update_id = index.schema_update(&mut update_writer, schema).unwrap();
+        let _update_id = index.settings_update(&mut update_writer, settings).unwrap();
         update_writer.commit().unwrap();
 
         let mut additions = index.documents_addition();
@@ -574,31 +822,19 @@ mod tests {
         let _update_id = additions.finalize(&mut update_writer).unwrap();
         update_writer.commit().unwrap();
 
-        let schema = {
+        let settings = {
             let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
-
-                [attributes."age"]
-                displayed = true
-                indexed = true
-
-                [attributes."sex"]
-                displayed = true
-                indexed = true
+                {
+                    "searchableAttributes": ["name", "description", "age", "sex"],
+                    "displayedAttributes": ["name", "description", "age", "sex"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.to_update().unwrap()
         };
 
         let mut writer = db.update_write_txn().unwrap();
-        let update_id = index.schema_update(&mut writer, schema).unwrap();
+        let update_id = index.settings_update(&mut writer, settings).unwrap();
         writer.commit().unwrap();
 
         // block until the transaction is processed
@@ -608,7 +844,7 @@ mod tests {
         let update_reader = db.update_read_txn().unwrap();
         let result = index.update_status(&update_reader, update_id).unwrap();
         assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-        update_reader.abort();
+        update_reader.abort().unwrap();
 
         let mut additions = index.documents_addition();
 
@@ -642,61 +878,44 @@ mod tests {
         let update_reader = db.update_read_txn().unwrap();
         let result = index.update_status(&update_reader, update_id).unwrap();
         assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-        update_reader.abort();
+        update_reader.abort().unwrap();
 
         // even try to search for a document
         let reader = db.main_read_txn().unwrap();
-        let results = index.query_builder().query(&reader, "21 ", 0..20).unwrap();
-        assert_matches!(results.len(), 1);
+        let SortResult {documents, .. } = index.query_builder().query(&reader, Some("21 "), 0..20).unwrap();
+        assert_matches!(documents.len(), 1);
 
-        reader.abort();
+        reader.abort().unwrap();
 
         // try to introduce attributes in the middle of the schema
-        let schema = {
+        let settings = {
             let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
-
-                [attributes."city"]
-                displayed = true
-                indexed = true
-
-                [attributes."age"]
-                displayed = true
-                indexed = true
-
-                [attributes."sex"]
-                displayed = true
-                indexed = true
+                {
+                    "searchableAttributes": ["name", "description", "city", "age", "sex"],
+                    "displayedAttributes": ["name", "description", "city", "age", "sex"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.to_update().unwrap()
         };
 
         let mut writer = db.update_write_txn().unwrap();
-        let update_id = index.schema_update(&mut writer, schema).unwrap();
+        let update_id = index.settings_update(&mut writer, settings).unwrap();
         writer.commit().unwrap();
 
         // block until the transaction is processed
         let _ = receiver.iter().find(|id| *id == update_id);
-
         // check if it has been accepted
         let update_reader = db.update_read_txn().unwrap();
         let result = index.update_status(&update_reader, update_id).unwrap();
-        assert_matches!(result, Some(UpdateStatus::Failed { content }) if content.error.is_some());
+        assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
     }
 
     #[test]
     fn deserialize_documents() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -707,23 +926,23 @@ mod tests {
 
         database.set_update_callback(Box::new(update_fn));
 
-        let schema = {
+        let mut writer = db.main_write_txn().unwrap();
+        index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
+        writer.commit().unwrap();
+
+        let settings = {
             let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
+                {
+                    "searchableAttributes": ["name", "description"],
+                    "displayedAttributes": ["name", "description"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.to_update().unwrap()
         };
 
         let mut writer = db.update_write_txn().unwrap();
-        let _update_id = index.schema_update(&mut writer, schema).unwrap();
+        let _update_id = index.settings_update(&mut writer, settings).unwrap();
         writer.commit().unwrap();
 
         let mut additions = index.documents_addition();
@@ -755,19 +974,19 @@ mod tests {
         let update_reader = db.update_read_txn().unwrap();
         let result = index.update_status(&update_reader, update_id).unwrap();
         assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-        update_reader.abort();
+        update_reader.abort().unwrap();
 
         let reader = db.main_read_txn().unwrap();
         let document: Option<IgnoredAny> = index.document(&reader, None, DocumentId(25)).unwrap();
         assert!(document.is_none());
 
         let document: Option<IgnoredAny> = index
-            .document(&reader, None, DocumentId(7900334843754999545))
+            .document(&reader, None, DocumentId(0))
             .unwrap();
         assert!(document.is_some());
 
         let document: Option<IgnoredAny> = index
-            .document(&reader, None, DocumentId(8367468610878465872))
+            .document(&reader, None, DocumentId(1))
             .unwrap();
         assert!(document.is_some());
     }
@@ -776,7 +995,7 @@ mod tests {
     fn partial_document_update() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -787,26 +1006,23 @@ mod tests {
 
         database.set_update_callback(Box::new(update_fn));
 
-        let schema = {
+        let mut writer = db.main_write_txn().unwrap();
+        index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
+        writer.commit().unwrap();
+
+        let settings = {
             let data = r#"
-                identifier = "id"
-
-                [attributes."id"]
-                displayed = true
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
+                {
+                    "searchableAttributes": ["name", "description"],
+                    "displayedAttributes": ["name", "description", "id"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.to_update().unwrap()
         };
 
         let mut writer = db.update_write_txn().unwrap();
-        let _update_id = index.schema_update(&mut writer, schema).unwrap();
+        let _update_id = index.settings_update(&mut writer, settings).unwrap();
         writer.commit().unwrap();
 
         let mut additions = index.documents_addition();
@@ -838,23 +1054,23 @@ mod tests {
         let update_reader = db.update_read_txn().unwrap();
         let result = index.update_status(&update_reader, update_id).unwrap();
         assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-        update_reader.abort();
+        update_reader.abort().unwrap();
 
         let reader = db.main_read_txn().unwrap();
         let document: Option<IgnoredAny> = index.document(&reader, None, DocumentId(25)).unwrap();
         assert!(document.is_none());
 
         let document: Option<IgnoredAny> = index
-            .document(&reader, None, DocumentId(7900334843754999545))
+            .document(&reader, None, DocumentId(0))
             .unwrap();
         assert!(document.is_some());
 
         let document: Option<IgnoredAny> = index
-            .document(&reader, None, DocumentId(8367468610878465872))
+            .document(&reader, None, DocumentId(1))
             .unwrap();
         assert!(document.is_some());
 
-        reader.abort();
+        reader.abort().unwrap();
 
         let mut partial_additions = index.documents_partial_addition();
 
@@ -883,11 +1099,11 @@ mod tests {
         let update_reader = db.update_read_txn().unwrap();
         let result = index.update_status(&update_reader, update_id).unwrap();
         assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-        update_reader.abort();
+        update_reader.abort().unwrap();
 
         let reader = db.main_read_txn().unwrap();
         let document: Option<serde_json::Value> = index
-            .document(&reader, None, DocumentId(7900334843754999545))
+            .document(&reader, None, DocumentId(0))
             .unwrap();
 
         let new_doc1 = serde_json::json!({
@@ -898,7 +1114,7 @@ mod tests {
         assert_eq!(document, Some(new_doc1));
 
         let document: Option<serde_json::Value> = index
-            .document(&reader, None, DocumentId(8367468610878465872))
+            .document(&reader, None, DocumentId(1))
             .unwrap();
 
         let new_doc2 = serde_json::json!({
@@ -913,7 +1129,7 @@ mod tests {
     fn delete_index() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Arc::new(Database::open_or_create(dir.path()).unwrap());
+        let database = Arc::new(Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap());
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -929,24 +1145,23 @@ mod tests {
 
         database.set_update_callback(Box::new(update_fn));
 
-        let schema = {
+        let mut writer = db.main_write_txn().unwrap();
+        index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
+        writer.commit().unwrap();
+
+        let settings = {
             let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
+                {
+                    "searchableAttributes": ["name", "description"],
+                    "displayedAttributes": ["name", "description"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.to_update().unwrap()
         };
 
-        // add a schema to the index
         let mut writer = db.update_write_txn().unwrap();
-        let _update_id = index.schema_update(&mut writer, schema).unwrap();
+        let _update_id = index.settings_update(&mut writer, settings).unwrap();
         writer.commit().unwrap();
 
         // add documents to the index
@@ -986,7 +1201,7 @@ mod tests {
     fn check_number_ordering() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -997,23 +1212,32 @@ mod tests {
 
         database.set_update_callback(Box::new(update_fn));
 
-        let schema = {
+        let mut writer = db.main_write_txn().unwrap();
+        index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
+        writer.commit().unwrap();
+
+        let settings = {
             let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."release_date"]
-                displayed = true
-                ranked = true
+                {
+                    "rankingRules": [
+                        "typo",
+                        "words",
+                        "proximity",
+                        "attribute",
+                        "wordsPosition",
+                        "exactness",
+                        "desc(release_date)"
+                    ],
+                    "searchableAttributes": ["name", "release_date"],
+                    "displayedAttributes": ["name", "release_date"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.to_update().unwrap()
         };
 
         let mut writer = db.update_write_txn().unwrap();
-        let _update_id = index.schema_update(&mut writer, schema).unwrap();
+        let _update_id = index.settings_update(&mut writer, settings).unwrap();
         writer.commit().unwrap();
 
         let mut additions = index.documents_addition();
@@ -1056,20 +1280,20 @@ mod tests {
 
         let builder = index.query_builder_with_criteria(criteria);
 
-        let results = builder.query(&reader, "Kevin", 0..20).unwrap();
-        let mut iter = results.into_iter();
+        let SortResult {documents, .. } = builder.query(&reader, Some("Kevin"), 0..20).unwrap();
+        let mut iter = documents.into_iter();
 
         assert_matches!(
             iter.next(),
             Some(Document {
-                id: DocumentId(7900334843754999545),
+                id: DocumentId(0),
                 ..
             })
         );
         assert_matches!(
             iter.next(),
             Some(Document {
-                id: DocumentId(8367468610878465872),
+                id: DocumentId(1),
                 ..
             })
         );

meilisearch-core/src/error.rs

@@ -1,24 +1,63 @@
 use crate::serde::{DeserializerError, SerializerError};
 use serde_json::Error as SerdeJsonError;
+use pest::error::Error as PestError;
+use crate::filters::Rule;
 use std::{error, fmt, io};
 
+pub use bincode::Error as BincodeError;
+pub use fst::Error as FstError;
+pub use heed::Error as HeedError;
+pub use pest::error as pest_error;
+
+use meilisearch_error::{ErrorCode, Code};
+
 pub type MResult<T> = Result<T, Error>;
 
 #[derive(Debug)]
 pub enum Error {
-    Io(io::Error),
-    IndexAlreadyExists,
-    SchemaDiffer,
-    SchemaMissing,
-    WordIndexMissing,
-    MissingDocumentId,
-    Zlmdb(heed::Error),
-    Fst(fst::Error),
-    SerdeJson(SerdeJsonError),
     Bincode(bincode::Error),
-    Serializer(SerializerError),
     Deserializer(DeserializerError),
-    UnsupportedOperation(UnsupportedOperation),
+    FacetError(FacetError),
+    FilterParseError(PestError<Rule>),
+    Fst(fst::Error),
+    Heed(heed::Error),
+    IndexAlreadyExists,
+    Io(io::Error),
+    MaxFieldsLimitExceeded,
+    MissingDocumentId,
+    MissingPrimaryKey,
+    Schema(meilisearch_schema::Error),
+    SchemaMissing,
+    SerdeJson(SerdeJsonError),
+    Serializer(SerializerError),
+    VersionMismatch(String),
+    WordIndexMissing,
+}
+
+impl ErrorCode for Error {
+    fn error_code(&self) -> Code {
+        use Error::*;
+
+        match self {
+            FacetError(_) => Code::Facet,
+            FilterParseError(_) => Code::Filter,
+            IndexAlreadyExists => Code::IndexAlreadyExists,
+            MissingPrimaryKey => Code::MissingPrimaryKey,
+            MissingDocumentId => Code::MissingDocumentId,
+            MaxFieldsLimitExceeded => Code::MaxFieldsLimitExceeded,
+            Schema(s) =>  s.error_code(),
+            WordIndexMissing
+            | SchemaMissing => Code::InvalidState,
+            Heed(_)
+            | Fst(_)
+            | SerdeJson(_)
+            | Bincode(_)
+            | Serializer(_)
+            | Deserializer(_)
+            | VersionMismatch(_)
+            | Io(_) => Code::Internal,
+        }
+    }
 }
 
 impl From<io::Error> for Error {
@@ -27,14 +66,48 @@ impl From<io::Error> for Error {
     }
 }
 
-impl From<heed::Error> for Error {
-    fn from(error: heed::Error) -> Error {
-        Error::Zlmdb(error)
+impl From<PestError<Rule>> for Error {
+    fn from(error: PestError<Rule>) -> Error {
+        Error::FilterParseError(error.renamed_rules(|r| {
+            let s  = match r {
+                Rule::or => "OR",
+                Rule::and => "AND",
+                Rule::not => "NOT",
+                Rule::string => "string",
+                Rule::word => "word",
+                Rule::greater => "field > value",
+                Rule::less => "field < value",
+                Rule::eq => "field = value",
+                Rule::leq => "field <= value",
+                Rule::geq => "field >= value",
+                Rule::key => "key",
+                _ => "other",
+            };
+            s.to_string()
+        }))
     }
 }
 
-impl From<fst::Error> for Error {
-    fn from(error: fst::Error) -> Error {
+impl From<FacetError> for Error {
+    fn from(error: FacetError) -> Error {
+        Error::FacetError(error)
+    }
+}
+
+impl From<meilisearch_schema::Error> for Error {
+    fn from(error: meilisearch_schema::Error) -> Error {
+        Error::Schema(error)
+    }
+}
+
+impl From<HeedError> for Error {
+    fn from(error: HeedError) -> Error {
+        Error::Heed(error)
+    }
+}
+
+impl From<FstError> for Error {
+    fn from(error: FstError) -> Error {
         Error::Fst(error)
     }
 }
@@ -45,15 +118,18 @@ impl From<SerdeJsonError> for Error {
     }
 }
 
-impl From<bincode::Error> for Error {
-    fn from(error: bincode::Error) -> Error {
+impl From<BincodeError> for Error {
+    fn from(error: BincodeError) -> Error {
         Error::Bincode(error)
     }
 }
 
 impl From<SerializerError> for Error {
     fn from(error: SerializerError) -> Error {
-        Error::Serializer(error)
+        match error {
+            SerializerError::DocumentIdNotFound => Error::MissingDocumentId,
+            e => Error::Serializer(e),
+        }
     }
 }
 
@@ -63,55 +139,86 @@ impl From<DeserializerError> for Error {
     }
 }
 
-impl From<UnsupportedOperation> for Error {
-    fn from(op: UnsupportedOperation) -> Error {
-        Error::UnsupportedOperation(op)
-    }
-}
-
 impl fmt::Display for Error {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         use self::Error::*;
         match self {
-            Io(e) => write!(f, "{}", e),
-            IndexAlreadyExists => write!(f, "index already exists"),
-            SchemaDiffer => write!(f, "schemas differ"),
-            SchemaMissing => write!(f, "this index does not have a schema"),
-            WordIndexMissing => write!(f, "this index does not have a word index"),
-            MissingDocumentId => write!(f, "document id is missing"),
-            Zlmdb(e) => write!(f, "heed error; {}", e),
-            Fst(e) => write!(f, "fst error; {}", e),
-            SerdeJson(e) => write!(f, "serde json error; {}", e),
             Bincode(e) => write!(f, "bincode error; {}", e),
-            Serializer(e) => write!(f, "serializer error; {}", e),
             Deserializer(e) => write!(f, "deserializer error; {}", e),
-            UnsupportedOperation(op) => write!(f, "unsupported operation; {}", op),
+            FacetError(e) => write!(f, "error processing facet filter: {}", e),
+            FilterParseError(e) => write!(f, "error parsing filter; {}", e),
+            Fst(e) => write!(f, "fst error; {}", e),
+            Heed(e) => write!(f, "heed error; {}", e),
+            IndexAlreadyExists => write!(f, "index already exists"),
+            Io(e) => write!(f, "{}", e),
+            MaxFieldsLimitExceeded => write!(f, "maximum number of fields in a document exceeded"),
+            MissingDocumentId => write!(f, "document id is missing"),
+            MissingPrimaryKey => write!(f, "schema cannot be built without a primary key"),
+            Schema(e) => write!(f, "schema error; {}", e),
+            SchemaMissing => write!(f, "this index does not have a schema"),
+            SerdeJson(e) => write!(f, "serde json error; {}", e),
+            Serializer(e) => write!(f, "serializer error; {}", e),
+            VersionMismatch(version) => write!(f, "Cannot open database, expected MeiliSearch engine version: {}, current engine version: {}.{}.{}",
+                version,
+                env!("CARGO_PKG_VERSION_MAJOR"),
+                env!("CARGO_PKG_VERSION_MINOR"),
+                env!("CARGO_PKG_VERSION_PATCH")),
+            WordIndexMissing => write!(f, "this index does not have a word index"),
         }
     }
 }
 
 impl error::Error for Error {}
 
-#[derive(Debug)]
-pub enum UnsupportedOperation {
-    SchemaAlreadyExists,
-    CannotUpdateSchemaIdentifier,
-    CannotReorderSchemaAttribute,
-    CanOnlyIntroduceNewSchemaAttributesAtEnd,
-    CannotRemoveSchemaAttribute,
+struct FilterParseError(PestError<Rule>);
+
+impl fmt::Display for FilterParseError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use crate::pest_error::LineColLocation::*;
+
+        let (line, column) = match self.0.line_col {
+            Span((line, _), (column, _)) => (line, column),
+            Pos((line, column)) => (line, column),
+        };
+        write!(f, "parsing error on line {} at column {}: {}", line, column, self.0.variant.message())
+    }
 }
 
-impl fmt::Display for UnsupportedOperation {
+#[derive(Debug)]
+pub enum FacetError {
+    EmptyArray,
+    ParsingError(String),
+    UnexpectedToken { expected: &'static [&'static str], found: String },
+    InvalidFormat(String),
+    AttributeNotFound(String),
+    AttributeNotSet { expected: Vec<String>, found: String },
+    InvalidDocumentAttribute(String),
+    NoAttributesForFaceting,
+}
+
+impl FacetError {
+    pub fn unexpected_token(expected: &'static [&'static str], found: impl ToString) -> FacetError {
+        FacetError::UnexpectedToken{ expected, found: found.to_string() }
+    }
+
+    pub fn attribute_not_set(expected: Vec<String>, found: impl ToString) -> FacetError {
+        FacetError::AttributeNotSet{ expected, found: found.to_string() }
+    }
+}
+
+impl fmt::Display for FacetError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        use self::UnsupportedOperation::*;
+        use FacetError::*;
+
         match self {
-            SchemaAlreadyExists => write!(f, "Cannot update index which already have a schema"),
-            CannotUpdateSchemaIdentifier => write!(f, "Cannot update the identifier of a schema"),
-            CannotReorderSchemaAttribute => write!(f, "Cannot reorder the attributes of a schema"),
-            CanOnlyIntroduceNewSchemaAttributesAtEnd => {
-                write!(f, "Can only introduce new attributes at end of a schema")
-            }
-            CannotRemoveSchemaAttribute => write!(f, "Cannot remove attributes from a schema"),
+            EmptyArray => write!(f, "empty array in facet filter is unspecified behavior"),
+            ParsingError(msg) => write!(f, "parsing error: {}", msg),
+            UnexpectedToken { expected, found } => write!(f, "unexpected token {}, expected {}", found, expected.join("or")),
+            InvalidFormat(found) => write!(f, "invalid facet: {}, facets should be \"facetName:facetValue\"", found),
+            AttributeNotFound(attr) => write!(f, "unknown {:?} attribute", attr),
+            AttributeNotSet { found, expected } => write!(f, "`{}` is not set as a faceted attribute. available facet attributes: {}", found, expected.join(", ")),
+            InvalidDocumentAttribute(attr) => write!(f, "invalid document attribute {}, accepted types: String and [String]", attr),
+            NoAttributesForFaceting => write!(f, "impossible to perform faceted search, no attributes for faceting are set"),
         }
     }
 }

meilisearch-core/src/facets.rs

@@ -0,0 +1,357 @@
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::hash::Hash;
+use std::ops::Deref;
+
+use cow_utils::CowUtils;
+use either::Either;
+use heed::types::{Str, OwnedType};
+use indexmap::IndexMap;
+use serde_json::Value;
+
+use meilisearch_schema::{FieldId, Schema};
+use meilisearch_types::DocumentId;
+
+use crate::database::MainT;
+use crate::error::{FacetError, MResult};
+use crate::store::BEU16;
+
+/// Data structure used to represent a boolean expression in the form of nested arrays.
+/// Values in the outer array are and-ed together, values in the inner arrays are or-ed together.
+#[derive(Debug, PartialEq)]
+pub struct FacetFilter(Vec<Either<Vec<FacetKey>, FacetKey>>);
+
+impl Deref for FacetFilter {
+    type Target = Vec<Either<Vec<FacetKey>, FacetKey>>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl FacetFilter {
+    pub fn from_str(
+        s: &str,
+        schema: &Schema,
+        attributes_for_faceting: &[FieldId],
+    ) -> MResult<FacetFilter> {
+        if attributes_for_faceting.is_empty() {
+            return Err(FacetError::NoAttributesForFaceting.into());
+        }
+        let parsed = serde_json::from_str::<Value>(s).map_err(|e| FacetError::ParsingError(e.to_string()))?;
+        let mut filter = Vec::new();
+        match parsed {
+            Value::Array(and_exprs) => {
+                if and_exprs.is_empty() {
+                    return Err(FacetError::EmptyArray.into());
+                }
+                for expr in and_exprs {
+                    match expr {
+                        Value::String(s) => {
+                            let key = FacetKey::from_str( &s, schema, attributes_for_faceting)?;
+                            filter.push(Either::Right(key));
+                        }
+                        Value::Array(or_exprs) => {
+                            if or_exprs.is_empty() {
+                                return Err(FacetError::EmptyArray.into());
+                            }
+                            let mut inner = Vec::new();
+                            for expr in or_exprs {
+                                match expr {
+                                    Value::String(s) => {
+                                        let key = FacetKey::from_str( &s, schema, attributes_for_faceting)?;
+                                        inner.push(key);
+                                    }
+                                    bad_value => return Err(FacetError::unexpected_token(&["String"], bad_value).into()),
+                                }
+                            }
+                            filter.push(Either::Left(inner));
+                        }
+                        bad_value => return Err(FacetError::unexpected_token(&["Array", "String"], bad_value).into()),
+                    }
+                }
+                Ok(Self(filter))
+            }
+            bad_value => Err(FacetError::unexpected_token(&["Array"], bad_value).into()),
+        }
+    }
+}
+
+#[derive(Debug, Eq, PartialEq, Hash)]
+#[repr(C)]
+pub struct FacetKey(FieldId, String);
+
+impl FacetKey {
+    pub fn new(field_id: FieldId, value: String) -> Self {
+        let value = match value.cow_to_lowercase() {
+                Cow::Borrowed(_) => value,
+                Cow::Owned(s) => s,
+        };
+        Self(field_id, value)
+    }
+
+    pub fn key(&self) -> FieldId {
+        self.0
+    }
+
+    pub fn value(&self) -> &str {
+        &self.1
+    }
+
+    // TODO improve parser
+    fn from_str(
+        s: &str,
+        schema: &Schema,
+        attributes_for_faceting: &[FieldId],
+    ) -> Result<Self, FacetError> {
+        let mut split = s.splitn(2, ':');
+        let key = split
+            .next()
+            .ok_or_else(|| FacetError::InvalidFormat(s.to_string()))?
+            .trim();
+        let field_id = schema
+            .id(key)
+            .ok_or_else(|| FacetError::AttributeNotFound(key.to_string()))?;
+
+        if !attributes_for_faceting.contains(&field_id) {
+            return Err(FacetError::attribute_not_set(
+                    attributes_for_faceting
+                    .iter()
+                    .filter_map(|&id| schema.name(id))
+                    .map(str::to_string)
+                    .collect::<Vec<_>>(),
+                    key))
+        }
+        let value = split
+            .next()
+            .ok_or_else(|| FacetError::InvalidFormat(s.to_string()))?
+            .trim();
+        // unquoting the string if need be:
+        let mut indices = value.char_indices();
+        let value =  match (indices.next(), indices.last()) {
+            (Some((s, '\'')), Some((e, '\''))) |
+            (Some((s, '\"')), Some((e, '\"'))) => value[s + 1..e].to_string(),
+            _ => value.to_string(),
+        };
+        Ok(Self::new(field_id, value))
+    }
+}
+
+impl<'a> heed::BytesEncode<'a> for FacetKey {
+    type EItem = FacetKey;
+
+    fn bytes_encode(item: &'a Self::EItem) -> Option<Cow<'a, [u8]>> {
+        let mut buffer = Vec::with_capacity(2 + item.1.len());
+        let id = BEU16::new(item.key().into());
+        let id_bytes = OwnedType::bytes_encode(&id)?;
+        let value_bytes = Str::bytes_encode(item.value())?;
+        buffer.extend_from_slice(id_bytes.as_ref());
+        buffer.extend_from_slice(value_bytes.as_ref());
+        Some(Cow::Owned(buffer))
+    }
+}
+
+impl<'a> heed::BytesDecode<'a> for FacetKey {
+    type DItem = FacetKey;
+
+    fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
+        let (id_bytes, value_bytes) = bytes.split_at(2);
+        let id = OwnedType::<BEU16>::bytes_decode(id_bytes)?;
+        let id = id.get().into();
+        let string = Str::bytes_decode(&value_bytes)?;
+        Some(FacetKey(id, string.to_string()))
+    }
+}
+
+pub fn add_to_facet_map(
+    facet_map: &mut HashMap<FacetKey, (String, Vec<DocumentId>)>,
+    field_id: FieldId,
+    value: Value,
+    document_id: DocumentId,
+) -> Result<(), FacetError> {
+    let value = match value {
+        Value::String(s) => s,
+        // ignore null
+        Value::Null => return Ok(()),
+        value => return Err(FacetError::InvalidDocumentAttribute(value.to_string())),
+    };
+    let key = FacetKey::new(field_id, value.clone());
+    facet_map.entry(key).or_insert_with(|| (value, Vec::new())).1.push(document_id);
+    Ok(())
+}
+
+pub fn facet_map_from_docids(
+    rtxn: &heed::RoTxn<MainT>,
+    index: &crate::Index,
+    document_ids: &[DocumentId],
+    attributes_for_facetting: &[FieldId],
+) -> MResult<HashMap<FacetKey, (String, Vec<DocumentId>)>> {
+    // A hashmap that ascociate a facet key to a pair containing the original facet attribute
+    // string with it's case preserved, and a list of document ids for that facet attribute.
+    let mut facet_map: HashMap<FacetKey, (String, Vec<DocumentId>)> = HashMap::new();
+    for document_id in document_ids {
+        for result in index
+            .documents_fields
+            .document_fields(rtxn, *document_id)?
+        {
+            let (field_id, bytes) = result?;
+            if attributes_for_facetting.contains(&field_id) {
+                match serde_json::from_slice(bytes)? {
+                    Value::Array(values) => {
+                        for v in values {
+                            add_to_facet_map(&mut facet_map, field_id, v, *document_id)?;
+                        }
+                    }
+                    v => add_to_facet_map(&mut facet_map, field_id, v, *document_id)?,
+                };
+            }
+        }
+    }
+    Ok(facet_map)
+}
+
+pub fn facet_map_from_docs(
+    schema: &Schema,
+    documents: &HashMap<DocumentId, IndexMap<String, Value>>,
+    attributes_for_facetting: &[FieldId],
+) -> MResult<HashMap<FacetKey, (String, Vec<DocumentId>)>> {
+    let mut facet_map = HashMap::new();
+    let attributes_for_facetting = attributes_for_facetting
+        .iter()
+        .filter_map(|&id| schema.name(id).map(|name| (id, name)))
+        .collect::<Vec<_>>();
+
+    for (id, document) in documents {
+        for (field_id, name) in &attributes_for_facetting {
+            if let Some(value) = document.get(*name) {
+                match value {
+                    Value::Array(values) => {
+                        for v in values {
+                            add_to_facet_map(&mut facet_map, *field_id, v.clone(), *id)?;
+                        }
+                    }
+                    v => add_to_facet_map(&mut facet_map, *field_id, v.clone(), *id)?,
+                }
+            }
+        }
+    }
+    Ok(facet_map)
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use meilisearch_schema::Schema;
+
+    #[test]
+    fn test_facet_key() {
+        let mut schema = Schema::default();
+        let id = schema.insert_with_position("hello").unwrap().0;
+        let facet_list = [schema.id("hello").unwrap()];
+        assert_eq!(
+            FacetKey::from_str("hello:12", &schema, &facet_list).unwrap(),
+            FacetKey::new(id, "12".to_string())
+        );
+        assert_eq!(
+            FacetKey::from_str("hello:\"foo bar\"", &schema, &facet_list).unwrap(),
+            FacetKey::new(id, "foo bar".to_string())
+        );
+        assert_eq!(
+            FacetKey::from_str("hello:'foo bar'", &schema, &facet_list).unwrap(),
+            FacetKey::new(id, "foo bar".to_string())
+        );
+        // weird case
+        assert_eq!(
+            FacetKey::from_str("hello:blabla:machin", &schema, &facet_list).unwrap(),
+            FacetKey::new(id, "blabla:machin".to_string())
+        );
+
+        assert_eq!(
+            FacetKey::from_str("hello:\"\"", &schema, &facet_list).unwrap(),
+            FacetKey::new(id, "".to_string())
+        );
+
+        assert_eq!(
+            FacetKey::from_str("hello:'", &schema, &facet_list).unwrap(),
+            FacetKey::new(id, "'".to_string())
+        );
+        assert_eq!(
+            FacetKey::from_str("hello:''", &schema, &facet_list).unwrap(),
+            FacetKey::new(id, "".to_string())
+        );
+        assert!(FacetKey::from_str("hello", &schema, &facet_list).is_err());
+        assert!(FacetKey::from_str("toto:12", &schema, &facet_list).is_err());
+    }
+
+    #[test]
+    fn test_parse_facet_array() {
+        use either::Either::{Left, Right};
+        let mut schema = Schema::default();
+        let _id = schema.insert_with_position("hello").unwrap();
+        let facet_list = [schema.id("hello").unwrap()];
+        assert_eq!(
+            FacetFilter::from_str("[[\"hello:12\"]]", &schema, &facet_list).unwrap(),
+            FacetFilter(vec![Left(vec![FacetKey(FieldId(0), "12".to_string())])])
+        );
+        assert_eq!(
+            FacetFilter::from_str("[\"hello:12\"]", &schema, &facet_list).unwrap(),
+            FacetFilter(vec![Right(FacetKey(FieldId(0), "12".to_string()))])
+        );
+        assert_eq!(
+            FacetFilter::from_str("[\"hello:12\", \"hello:13\"]", &schema, &facet_list).unwrap(),
+            FacetFilter(vec![
+                Right(FacetKey(FieldId(0), "12".to_string())),
+                Right(FacetKey(FieldId(0), "13".to_string()))
+            ])
+        );
+        assert_eq!(
+            FacetFilter::from_str("[[\"hello:12\", \"hello:13\"]]", &schema, &facet_list).unwrap(),
+            FacetFilter(vec![Left(vec![
+                FacetKey(FieldId(0), "12".to_string()),
+                FacetKey(FieldId(0), "13".to_string())
+            ])])
+        );
+        assert_eq!(
+            FacetFilter::from_str(
+                "[[\"hello:12\", \"hello:13\"], \"hello:14\"]",
+                &schema,
+                &facet_list
+            )
+            .unwrap(),
+            FacetFilter(vec![
+                Left(vec![
+                    FacetKey(FieldId(0), "12".to_string()),
+                    FacetKey(FieldId(0), "13".to_string())
+                ]),
+                Right(FacetKey(FieldId(0), "14".to_string()))
+            ])
+        );
+
+        // invalid array depths
+        assert!(FacetFilter::from_str(
+            "[[[\"hello:12\", \"hello:13\"], \"hello:14\"]]",
+            &schema,
+            &facet_list
+        )
+        .is_err());
+        assert!(FacetFilter::from_str(
+            "[[[\"hello:12\", \"hello:13\"]], \"hello:14\"]]",
+            &schema,
+            &facet_list
+        )
+        .is_err());
+        assert!(FacetFilter::from_str("\"hello:14\"", &schema, &facet_list).is_err());
+
+        // unexisting key
+        assert!(FacetFilter::from_str("[\"foo:12\"]", &schema, &facet_list).is_err());
+
+        // invalid facet key
+        assert!(FacetFilter::from_str("[\"foo=12\"]", &schema, &facet_list).is_err());
+        assert!(FacetFilter::from_str("[\"foo12\"]", &schema, &facet_list).is_err());
+        assert!(FacetFilter::from_str("[\"\"]", &schema, &facet_list).is_err());
+
+        // empty array error
+        assert!(FacetFilter::from_str("[]", &schema, &facet_list).is_err());
+        assert!(FacetFilter::from_str("[\"hello:12\", []]", &schema, &facet_list).is_err());
+    }
+}

meilisearch-core/src/filters/condition.rs

@@ -0,0 +1,276 @@
+use std::str::FromStr;
+use std::cmp::Ordering;
+
+use crate::error::Error;
+use crate::{store::Index, DocumentId, MainT};
+use heed::RoTxn;
+use meilisearch_schema::{FieldId, Schema};
+use pest::error::{Error as PestError, ErrorVariant};
+use pest::iterators::Pair;
+use serde_json::{Value, Number};
+use super::parser::Rule;
+
+#[derive(Debug, PartialEq)]
+enum ConditionType {
+    Greater,
+    Less,
+    Equal,
+    LessEqual,
+    GreaterEqual,
+    NotEqual,
+}
+
+/// We need to infer type when the filter is constructed
+/// and match every possible types it can be parsed into.
+#[derive(Debug)]
+struct ConditionValue<'a> {
+    string: &'a str,
+    boolean: Option<bool>,
+    number: Option<Number>
+}
+
+impl<'a> ConditionValue<'a> {
+    pub fn new(value: &Pair<'a, Rule>) -> Self {
+        match value.as_rule() {
+            Rule::string | Rule::word => {
+                let string =  value.as_str();
+                let boolean = match value.as_str() {
+                    "true" => Some(true),
+                    "false" => Some(false),
+                    _ => None,
+                };
+                let number = Number::from_str(value.as_str()).ok();
+                ConditionValue { string, boolean, number }
+            },
+            _ => unreachable!(),
+        }
+    }
+
+    pub fn as_str(&self) -> &str {
+        self.string
+    }
+
+    pub fn as_number(&self) -> Option<&Number> {
+        self.number.as_ref()
+    }
+
+    pub fn as_bool(&self) -> Option<bool> {
+        self.boolean
+    }
+}
+
+#[derive(Debug)]
+pub struct Condition<'a> {
+    field: FieldId,
+    condition: ConditionType,
+    value: ConditionValue<'a>
+}
+
+fn get_field_value<'a>(schema: &Schema, pair: Pair<'a, Rule>) -> Result<(FieldId, ConditionValue<'a>), Error> {
+    let mut items = pair.into_inner();
+    // lexing ensures that we at least have a key
+    let key = items.next().unwrap();
+    let field = schema
+        .id(key.as_str())
+        .ok_or_else(|| PestError::new_from_span(
+                ErrorVariant::CustomError {
+                    message: format!(
+                                 "attribute `{}` not found, available attributes are: {}",
+                                 key.as_str(),
+                                 schema.names().collect::<Vec<_>>().join(", ")
+                             ),
+                },
+                key.as_span()))?;
+    let value = ConditionValue::new(&items.next().unwrap());
+    Ok((field, value))
+}
+
+// undefined behavior with big numbers
+fn compare_numbers(lhs: &Number, rhs: &Number) -> Option<Ordering> {
+    match (lhs.as_i64(), lhs.as_u64(), lhs.as_f64(),
+        rhs.as_i64(), rhs.as_u64(), rhs.as_f64()) {
+    //    i64   u64  f64  i64  u64  f64
+        (Some(lhs), _, _, Some(rhs), _, _) => lhs.partial_cmp(&rhs),
+        (_, Some(lhs), _, _, Some(rhs), _) => lhs.partial_cmp(&rhs),
+        (_, _, Some(lhs), _, _, Some(rhs)) => lhs.partial_cmp(&rhs),
+        (_, _, _, _, _, _) => None,
+    }
+}
+
+impl<'a> Condition<'a> {
+    pub fn less(
+        item: Pair<'a, Rule>,
+        schema: &'a Schema,
+    ) -> Result<Self, Error> {
+        let (field, value) = get_field_value(schema, item)?;
+        let condition = ConditionType::Less;
+        Ok(Self { field, condition, value })
+    }
+
+    pub fn greater(
+        item: Pair<'a, Rule>,
+        schema: &'a Schema,
+    ) -> Result<Self, Error> {
+        let (field, value) = get_field_value(schema, item)?;
+        let condition = ConditionType::Greater;
+        Ok(Self { field, condition, value })
+    }
+
+    pub fn neq(
+        item: Pair<'a, Rule>,
+        schema: &'a Schema,
+    ) -> Result<Self, Error> {
+        let (field, value) = get_field_value(schema, item)?;
+        let condition = ConditionType::NotEqual;
+        Ok(Self { field, condition, value })
+    }
+
+    pub fn geq(
+        item: Pair<'a, Rule>,
+        schema: &'a Schema,
+    ) -> Result<Self, Error> {
+        let (field, value) = get_field_value(schema, item)?;
+        let condition = ConditionType::GreaterEqual;
+        Ok(Self { field, condition, value })
+    }
+
+    pub fn leq(
+        item: Pair<'a, Rule>,
+        schema: &'a Schema,
+    ) -> Result<Self, Error> {
+        let (field, value) = get_field_value(schema, item)?;
+        let condition = ConditionType::LessEqual;
+        Ok(Self { field, condition, value })
+    }
+
+    pub fn eq(
+        item: Pair<'a, Rule>,
+        schema: &'a Schema,
+    ) -> Result<Self, Error> {
+        let (field, value) = get_field_value(schema, item)?;
+        let condition = ConditionType::Equal;
+        Ok(Self { field, condition, value })
+    }
+
+    pub fn test(
+        &self,
+        reader: &RoTxn<MainT>,
+        index: &Index,
+        document_id: DocumentId,
+    ) -> Result<bool, Error> {
+        match index.document_attribute::<Value>(reader, document_id, self.field)? {
+            Some(Value::Array(values)) => Ok(values.iter().any(|v| self.match_value(Some(v)))),
+            other => Ok(self.match_value(other.as_ref())),
+        }
+    }
+
+    fn match_value(&self, value: Option<&Value>) -> bool {
+        match value {
+            Some(Value::String(s)) => {
+                let value = self.value.as_str();
+                match self.condition {
+                    ConditionType::Equal => unicase::eq(value, &s),
+                    ConditionType::NotEqual => !unicase::eq(value, &s),
+                    _ => false
+                }
+            },
+            Some(Value::Number(n)) => { 
+                if let Some(value) = self.value.as_number() {
+                    if let Some(ord) = compare_numbers(&n, value) {
+                        let res =  match self.condition {
+                            ConditionType::Equal => ord == Ordering::Equal,
+                            ConditionType::NotEqual => ord != Ordering::Equal,
+                            ConditionType::GreaterEqual => ord != Ordering::Less,
+                            ConditionType::LessEqual => ord != Ordering::Greater,
+                            ConditionType::Greater => ord == Ordering::Greater,
+                            ConditionType::Less => ord == Ordering::Less,
+                        };
+                        return res
+                    } 
+                } 
+                false
+            },
+            Some(Value::Bool(b)) => {
+                if let Some(value) = self.value.as_bool() {
+                    let res = match self.condition {
+                        ConditionType::Equal => *b == value,
+                        ConditionType::NotEqual => *b != value,
+                        _ => false
+                    };
+                    return res
+                }
+                false
+            },
+            // if field is not supported (or not found), all values are different from it,
+            // so != should always return true in this case.
+            _ => self.condition == ConditionType::NotEqual,
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use serde_json::Number;
+    use std::cmp::Ordering;
+
+    #[test]
+    fn test_number_comp() {
+        // test both u64
+        let n1 = Number::from(1u64);
+        let n2 = Number::from(2u64);
+        assert_eq!(Some(Ordering::Less), compare_numbers(&n1, &n2));
+        assert_eq!(Some(Ordering::Greater), compare_numbers(&n2, &n1));
+        let n1 = Number::from(1u64);
+        let n2 = Number::from(1u64);
+        assert_eq!(Some(Ordering::Equal), compare_numbers(&n1, &n2));
+
+        // test both i64
+        let n1 = Number::from(1i64);
+        let n2 = Number::from(2i64);
+        assert_eq!(Some(Ordering::Less), compare_numbers(&n1, &n2));
+        assert_eq!(Some(Ordering::Greater), compare_numbers(&n2, &n1));
+        let n1 = Number::from(1i64);
+        let n2 = Number::from(1i64);
+        assert_eq!(Some(Ordering::Equal), compare_numbers(&n1, &n2));
+
+        // test both f64
+        let n1 = Number::from_f64(1f64).unwrap();
+        let n2 = Number::from_f64(2f64).unwrap();
+        assert_eq!(Some(Ordering::Less), compare_numbers(&n1, &n2));
+        assert_eq!(Some(Ordering::Greater), compare_numbers(&n2, &n1));
+        let n1 = Number::from_f64(1f64).unwrap();
+        let n2 = Number::from_f64(1f64).unwrap();
+        assert_eq!(Some(Ordering::Equal), compare_numbers(&n1, &n2));
+
+        // test one u64 and one f64
+        let n1 = Number::from_f64(1f64).unwrap();
+        let n2 = Number::from(2u64);
+        assert_eq!(Some(Ordering::Less), compare_numbers(&n1, &n2));
+        assert_eq!(Some(Ordering::Greater), compare_numbers(&n2, &n1));
+
+        // equality
+        let n1 = Number::from_f64(1f64).unwrap();
+        let n2 = Number::from(1u64);
+        assert_eq!(Some(Ordering::Equal), compare_numbers(&n1, &n2));
+        assert_eq!(Some(Ordering::Equal), compare_numbers(&n2, &n1));
+
+        // float is neg
+        let n1 = Number::from_f64(-1f64).unwrap();
+        let n2 = Number::from(1u64);
+        assert_eq!(Some(Ordering::Less), compare_numbers(&n1, &n2));
+        assert_eq!(Some(Ordering::Greater), compare_numbers(&n2, &n1));
+
+        // float is too big
+        let n1 = Number::from_f64(std::f64::MAX).unwrap();
+        let n2 = Number::from(1u64);
+        assert_eq!(Some(Ordering::Greater), compare_numbers(&n1, &n2));
+        assert_eq!(Some(Ordering::Less), compare_numbers(&n2, &n1));
+
+        // misc
+        let n1 = Number::from_f64(std::f64::MAX).unwrap();
+        let n2 = Number::from(std::u64::MAX);
+        assert_eq!(Some(Ordering::Greater), compare_numbers(&n1, &n2));
+        assert_eq!(Some( Ordering::Less ), compare_numbers(&n2, &n1));
+    }
+}

meilisearch-core/src/filters/mod.rs

@@ -0,0 +1,127 @@
+mod parser;
+mod condition;
+
+pub(crate) use parser::Rule;
+
+use std::ops::Not;
+
+use condition::Condition;
+use crate::error::Error;
+use crate::{DocumentId, MainT, store::Index};
+use heed::RoTxn;
+use meilisearch_schema::Schema;
+use parser::{PREC_CLIMBER, FilterParser};
+use pest::iterators::{Pair, Pairs};
+use pest::Parser;
+
+type FilterResult<'a> = Result<Filter<'a>, Error>;
+
+#[derive(Debug)]
+pub enum Filter<'a> {
+    Condition(Condition<'a>),
+    Or(Box<Self>, Box<Self>),
+    And(Box<Self>, Box<Self>),
+    Not(Box<Self>),
+}
+
+impl<'a> Filter<'a> {
+    pub fn parse(expr: &'a str, schema: &'a Schema) -> FilterResult<'a> {
+        let mut lexed = FilterParser::parse(Rule::prgm, expr)?;
+        Self::build(lexed.next().unwrap().into_inner(), schema)
+    }
+
+    pub fn test(
+        &self,
+        reader: &RoTxn<MainT>,
+        index: &Index,
+        document_id: DocumentId,
+    ) -> Result<bool, Error> {
+        use Filter::*;
+        match self {
+            Condition(c) => c.test(reader, index, document_id),
+            Or(lhs, rhs) => Ok(
+                lhs.test(reader, index, document_id)? || rhs.test(reader, index, document_id)?
+            ),
+            And(lhs, rhs) => Ok(
+                lhs.test(reader, index, document_id)? && rhs.test(reader, index, document_id)?
+            ),
+            Not(op) => op.test(reader, index, document_id).map(bool::not),
+        }
+    }
+
+    fn build(expression: Pairs<'a, Rule>, schema: &'a Schema) -> FilterResult<'a> {
+        PREC_CLIMBER.climb(
+            expression,
+            |pair: Pair<Rule>| match pair.as_rule() {
+                Rule::eq => Ok(Filter::Condition(Condition::eq(pair, schema)?)),
+                Rule::greater => Ok(Filter::Condition(Condition::greater(pair, schema)?)),
+                Rule::less => Ok(Filter::Condition(Condition::less(pair, schema)?)),
+                Rule::neq => Ok(Filter::Condition(Condition::neq(pair, schema)?)),
+                Rule::geq => Ok(Filter::Condition(Condition::geq(pair, schema)?)),
+                Rule::leq => Ok(Filter::Condition(Condition::leq(pair, schema)?)),
+                Rule::prgm => Self::build(pair.into_inner(), schema),
+                Rule::term => Self::build(pair.into_inner(), schema),
+                Rule::not => Ok(Filter::Not(Box::new(Self::build(
+                    pair.into_inner(),
+                    schema,
+                )?))),
+                _ => unreachable!(),
+            },
+            |lhs: FilterResult, op: Pair<Rule>, rhs: FilterResult| match op.as_rule() {
+                Rule::or => Ok(Filter::Or(Box::new(lhs?), Box::new(rhs?))),
+                Rule::and => Ok(Filter::And(Box::new(lhs?), Box::new(rhs?))),
+                _ => unreachable!(),
+            },
+        )
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn invalid_syntax() {
+        assert!(FilterParser::parse(Rule::prgm, "field : id").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "field=hello hello").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "field=hello OR OR").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "OR field:hello").is_err());
+        assert!(FilterParser::parse(Rule::prgm, r#"field="hello world"#).is_err());
+        assert!(FilterParser::parse(Rule::prgm, r#"field='hello world"#).is_err());
+        assert!(FilterParser::parse(Rule::prgm, "NOT field=").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "N").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "(field=1").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "(field=1))").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "field=1ORfield=2").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "field=1 ( OR field=2)").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "hello world=1").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "").is_err());
+        assert!(FilterParser::parse(Rule::prgm, r#"((((((hello=world)))))"#).is_err());
+    }
+
+    #[test]
+    fn valid_syntax() {
+        assert!(FilterParser::parse(Rule::prgm, "field = id").is_ok());
+        assert!(FilterParser::parse(Rule::prgm, "field=id").is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field >= 10"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field <= 10"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field="hello world""#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field='hello world'"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field > 10"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field < 10"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field < 10 AND NOT field=5"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field < 10 AND NOT field > 7.5"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field=true OR NOT field=5"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"NOT field=true OR NOT field=5"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field='hello world' OR ( NOT field=true OR NOT field=5 )"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field='hello \'worl\'d' OR ( NOT field=true OR NOT field=5 )"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field="hello \"worl\"d" OR ( NOT field=true OR NOT field=5 )"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"((((((hello=world))))))"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#""foo bar" > 10"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#""foo bar" = 10"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"'foo bar' = 10"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"'foo bar' <= 10"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"'foo bar' != 10"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"bar != 10"#).is_ok());
+    }
+}

meilisearch-core/src/filters/parser/grammar.pest

@@ -0,0 +1,28 @@
+key = _{quoted | word}
+value = _{quoted | word}
+quoted = _{ (PUSH("'") | PUSH("\"")) ~ string ~ POP  }
+string = {char*}
+word = ${(LETTER | NUMBER | "_" | "-" | ".")+}
+
+char =  _{ !(PEEK | "\\") ~ ANY
+    | "\\" ~ (PEEK | "\\" | "/" | "b" | "f" | "n" | "r" | "t")
+    | "\\" ~ ("u" ~ ASCII_HEX_DIGIT{4})}
+
+condition = _{eq | greater | less | geq | leq | neq}
+geq = {key ~ ">=" ~ value}
+leq = {key ~ "<=" ~ value}
+neq = {key ~ "!=" ~ value}
+eq = {key ~ "=" ~ value}
+greater = {key ~ ">" ~ value}
+less = {key ~ "<" ~ value}
+
+prgm = {SOI ~ expr ~ EOI}
+expr = _{ ( term ~ (operation ~ term)* ) }
+term = { ("(" ~ expr ~ ")") | condition | not }
+operation = _{ and | or }
+	and = {"AND"}
+	or = {"OR"}
+
+not = {"NOT" ~ term}
+
+WHITESPACE = _{ " " }

meilisearch-core/src/filters/parser/mod.rs

@@ -0,0 +1,12 @@
+use once_cell::sync::Lazy;
+use pest::prec_climber::{Operator, Assoc, PrecClimber};
+
+pub static PREC_CLIMBER: Lazy<PrecClimber<Rule>> = Lazy::new(|| {
+    use Assoc::*;
+    use Rule::*;
+    pest::prec_climber::PrecClimber::new(vec![Operator::new(or, Left), Operator::new(and, Left)])
+});
+
+#[derive(Parser)]
+#[grammar = "filters/parser/grammar.pest"]
+pub struct FilterParser;

meilisearch-core/src/lib.rs

@@ -1,12 +1,17 @@
+#![allow(clippy::type_complexity)]
+
 #[cfg(test)]
 #[macro_use]
 extern crate assert_matches;
+#[macro_use]
+extern crate pest_derive;
 
 mod automaton;
 mod bucket_sort;
 mod database;
 mod distinct_map;
 mod error;
+mod filters;
 mod levenshtein;
 mod number;
 mod query_builder;
@@ -15,31 +20,41 @@ mod query_words_mapper;
 mod ranked_map;
 mod raw_document;
 mod reordered_attrs;
-mod update;
 pub mod criterion;
+pub mod facets;
 pub mod raw_indexer;
 pub mod serde;
+pub mod settings;
 pub mod store;
+pub mod update;
 
-pub use self::database::{BoxUpdateFn, Database, MainT, UpdateT};
-pub use self::error::{Error, MResult};
+pub use self::database::{BoxUpdateFn, Database, DatabaseOptions, MainT, UpdateT, MainWriter, MainReader, UpdateWriter, UpdateReader};
+pub use self::error::{Error, HeedError, FstError, MResult, pest_error, FacetError};
+pub use self::filters::Filter;
 pub use self::number::{Number, ParseNumberError};
 pub use self::ranked_map::RankedMap;
 pub use self::raw_document::RawDocument;
 pub use self::store::Index;
 pub use self::update::{EnqueuedUpdateResult, ProcessedUpdateResult, UpdateStatus, UpdateType};
 pub use meilisearch_types::{DocIndex, DocumentId, Highlight};
+pub use meilisearch_schema::Schema;
 pub use query_words_mapper::QueryWordsMapper;
+pub use query_tree::MAX_QUERY_LEN;
 
-use std::convert::TryFrom;
-use std::collections::HashMap;
 use compact_arena::SmallArena;
+use log::{error, trace};
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::convert::TryFrom;
 
 use crate::bucket_sort::PostingsListView;
 use crate::levenshtein::prefix_damerau_levenshtein;
 use crate::query_tree::{QueryId, QueryKind};
 use crate::reordered_attrs::ReorderedAttrs;
 
+type FstSetCow<'a> = fst::Set<Cow<'a, [u8]>>;
+type FstMapCow<'a> = fst::Map<Cow<'a, [u8]>>;
+
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
 pub struct Document {
     pub id: DocumentId,
@@ -54,6 +69,7 @@ fn highlights_from_raw_document<'a, 'tag, 'txn>(
     queries_kinds: &HashMap<QueryId, &QueryKind>,
     arena: &SmallArena<'tag, PostingsListView<'txn>>,
     searchable_attrs: Option<&ReorderedAttrs>,
+    schema: &Schema,
 ) -> Vec<Highlight>
 {
     let mut highlights = Vec::new();
@@ -65,7 +81,7 @@ fn highlights_from_raw_document<'a, 'tag, 'txn>(
 
         for di in postings_list.iter() {
             let covered_area = match kind {
-                Some(QueryKind::Exact(query)) | Some(QueryKind::Tolerant(query)) => {
+                Some(QueryKind::NonTolerant(query)) | Some(QueryKind::Tolerant(query)) => {
                     let len = if query.len() > input.len() {
                         input.len()
                     } else {
@@ -80,8 +96,17 @@ fn highlights_from_raw_document<'a, 'tag, 'txn>(
                 .and_then(|sa| sa.reverse(di.attribute))
                 .unwrap_or(di.attribute);
 
+            let attribute = match schema.indexed_pos_to_field_id(attribute) {
+                Some(field_id) => field_id.0,
+                None => {
+                    error!("Cannot convert indexed_pos {} to field_id", attribute);
+                    trace!("Schema is compromized; {:?}", schema);
+                    continue
+                }
+            };
+
             let highlight = Highlight {
-                attribute: attribute,
+                attribute,
                 char_index: di.char_index,
                 char_length: covered_area,
             };
@@ -110,6 +135,7 @@ impl Document {
         queries_kinds: &HashMap<QueryId, &QueryKind>,
         arena: &SmallArena<'tag, PostingsListView<'txn>>,
         searchable_attrs: Option<&ReorderedAttrs>,
+        schema: &Schema,
     ) -> Document
     {
         let highlights = highlights_from_raw_document(
@@ -117,6 +143,7 @@ impl Document {
             queries_kinds,
             arena,
             searchable_attrs,
+            schema,
         );
 
         Document { id: raw_document.id, highlights }
@@ -128,6 +155,7 @@ impl Document {
         queries_kinds: &HashMap<QueryId, &QueryKind>,
         arena: &SmallArena<'tag, PostingsListView<'txn>>,
         searchable_attrs: Option<&ReorderedAttrs>,
+        schema: &Schema,
     ) -> Document
     {
         use crate::bucket_sort::SimpleMatch;
@@ -137,6 +165,7 @@ impl Document {
             queries_kinds,
             arena,
             searchable_attrs,
+            schema,
         );
 
         let mut matches = Vec::new();
@@ -145,6 +174,15 @@ impl Document {
                 .and_then(|sa| sa.reverse(sm.attribute))
                 .unwrap_or(sm.attribute);
 
+            let attribute = match schema.indexed_pos_to_field_id(attribute) {
+                Some(field_id) => field_id.0,
+                None => {
+                    error!("Cannot convert indexed_pos {} to field_id", attribute);
+                    trace!("Schema is compromized; {:?}", schema);
+                    continue
+                }
+            };
+
             matches.push(SimpleMatch { attribute, ..sm });
         }
         matches.sort_unstable();
@@ -160,6 +198,6 @@ mod tests {
 
     #[test]
     fn docindex_mem_size() {
-        assert_eq!(mem::size_of::<DocIndex>(), 16);
+        assert_eq!(mem::size_of::<DocIndex>(), 12);
     }
 }

meilisearch-core/src/number.rs

@@ -6,11 +6,18 @@ use std::str::FromStr;
 use ordered_float::OrderedFloat;
 use serde::{Deserialize, Serialize};
 
-#[derive(Serialize, Deserialize, Debug, Copy, Clone, Hash)]
+#[derive(Serialize, Deserialize, Debug, Copy, Clone)]
 pub enum Number {
     Unsigned(u64),
     Signed(i64),
     Float(OrderedFloat<f64>),
+    Null,
+}
+
+impl Default for Number {
+    fn default() -> Self {
+        Self::Null
+    }
 }
 
 impl FromStr for Number {
@@ -56,7 +63,7 @@ impl PartialOrd for Number {
 
 impl Ord for Number {
     fn cmp(&self, other: &Self) -> Ordering {
-        use Number::{Float, Signed, Unsigned};
+        use Number::{Float, Signed, Unsigned, Null};
 
         match (*self, *other) {
             (Unsigned(a), Unsigned(b)) => a.cmp(&b),
@@ -80,6 +87,9 @@ impl Ord for Number {
             (Float(a), Unsigned(b)) => a.cmp(&OrderedFloat(b as f64)),
             (Float(a), Signed(b)) => a.cmp(&OrderedFloat(b as f64)),
             (Float(a), Float(b)) => a.cmp(&b),
+            (Null, Null) => Ordering::Equal,
+            (_, Null) => Ordering::Less,
+            (Null, _) => Ordering::Greater,
         }
     }
 }

meilisearch-core/src/query_tree.rs

@@ -7,14 +7,17 @@ use std::{cmp, fmt, iter::once};
 
 use fst::{IntoStreamer, Streamer};
 use itertools::{EitherOrBoth, merge_join_by};
-use meilisearch_tokenizer::split_query_string;
+use log::debug;
+use meilisearch_tokenizer::analyzer::{Analyzer, AnalyzerConfig};
 use sdset::{Set, SetBuf, SetOperation};
 
 use crate::database::MainT;
-use crate::{store, DocumentId, DocIndex, MResult};
+use crate::{store, DocumentId, DocIndex, MResult, FstSetCow};
 use crate::automaton::{build_dfa, build_prefix_dfa, build_exact_dfa};
 use crate::QueryWordsMapper;
 
+pub const MAX_QUERY_LEN: usize = 10;
+
 #[derive(Clone, PartialEq, Eq, Hash)]
 pub enum Operation {
     And(Vec<Operation>),
@@ -44,16 +47,16 @@ impl fmt::Debug for Operation {
 
 impl Operation {
     fn tolerant(id: QueryId, prefix: bool, s: &str) -> Operation {
-        Operation::Query(Query { id, prefix, kind: QueryKind::Tolerant(s.to_string()) })
+        Operation::Query(Query { id, prefix, exact: true, kind: QueryKind::Tolerant(s.to_string()) })
     }
 
-    fn exact(id: QueryId, prefix: bool, s: &str) -> Operation {
-        Operation::Query(Query { id, prefix, kind: QueryKind::Exact(s.to_string()) })
+    fn non_tolerant(id: QueryId, prefix: bool, s: &str) -> Operation {
+        Operation::Query(Query { id, prefix, exact: true, kind: QueryKind::NonTolerant(s.to_string()) })
     }
 
     fn phrase2(id: QueryId, prefix: bool, (left, right): (&str, &str)) -> Operation {
         let kind = QueryKind::Phrase(vec![left.to_owned(), right.to_owned()]);
-        Operation::Query(Query { id, prefix, kind })
+        Operation::Query(Query { id, prefix, exact: true, kind })
     }
 }
 
@@ -63,6 +66,7 @@ pub type QueryId = usize;
 pub struct Query {
     pub id: QueryId,
     pub prefix: bool,
+    pub exact: bool,
     pub kind: QueryKind,
 }
 
@@ -82,17 +86,17 @@ impl Hash for Query {
 #[derive(Clone, PartialEq, Eq, Hash)]
 pub enum QueryKind {
     Tolerant(String),
-    Exact(String),
+    NonTolerant(String),
     Phrase(Vec<String>),
 }
 
 impl fmt::Debug for Query {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let Query { id, prefix, kind } = self;
+        let Query { id, prefix, kind, .. } = self;
         let prefix = if *prefix { String::from("Prefix") } else { String::default() };
         match kind {
-            QueryKind::Exact(word) => {
-                f.debug_struct(&(prefix + "Exact")).field("id", &id).field("word", &word).finish()
+            QueryKind::NonTolerant(word) => {
+                f.debug_struct(&(prefix + "NonTolerant")).field("id", &id).field("word", &word).finish()
             },
             QueryKind::Tolerant(word) => {
                 f.debug_struct(&(prefix + "Tolerant")).field("id", &id).field("word", &word).finish()
@@ -110,8 +114,9 @@ pub struct PostingsList {
     matches: SetBuf<DocIndex>,
 }
 
-pub struct Context {
-    pub words_set: fst::Set,
+pub struct Context<'a> {
+    pub words_set: FstSetCow<'a>,
+    pub stop_words: FstSetCow<'a>,
     pub synonyms: store::Synonyms,
     pub postings_lists: store::PostingsLists,
     pub prefix_postings_lists: store::PrefixPostingsListsCache,
@@ -143,8 +148,8 @@ fn split_best_frequency<'a>(reader: &heed::RoTxn<MainT>, ctx: &Context, word: &'
 }
 
 fn fetch_synonyms(reader: &heed::RoTxn<MainT>, ctx: &Context, words: &[&str]) -> MResult<Vec<Vec<String>>> {
-    let words = words.join(" ");
-    let set = ctx.synonyms.synonyms(reader, words.as_bytes())?.unwrap_or_default();
+    let words = &words.join(" ");
+    let set = ctx.synonyms.synonyms_fst(reader, words.as_bytes())?;
 
     let mut strings = Vec::new();
     let mut stream = set.stream();
@@ -158,13 +163,6 @@ fn fetch_synonyms(reader: &heed::RoTxn<MainT>, ctx: &Context, words: &[&str]) ->
     Ok(strings)
 }
 
-fn is_last<I: IntoIterator>(iter: I) -> impl Iterator<Item=(bool, I::Item)> {
-    let mut iter = iter.into_iter().peekable();
-    core::iter::from_fn(move || {
-        iter.next().map(|item| (iter.peek().is_none(), item))
-    })
-}
-
 fn create_operation<I, F>(iter: I, f: F) -> Operation
 where I: IntoIterator<Item=Operation>,
       F: Fn(Vec<Operation>) -> Operation,
@@ -178,36 +176,50 @@ where I: IntoIterator<Item=Operation>,
 
 const MAX_NGRAM: usize = 3;
 
+fn split_query_string<'a, A: AsRef<[u8]>>(s: &str, stop_words: &'a fst::Set<A>) -> Vec<(usize, String)> {
+    // TODO: Use global instance instead
+    Analyzer::new(AnalyzerConfig::default_with_stopwords(stop_words))
+        .analyze(s)
+        .tokens()
+        .filter(|t| t.is_word())
+        .map(|t| t.word.to_string())
+        .take(MAX_QUERY_LEN)
+        .enumerate()
+        .collect()
+}
+
 pub fn create_query_tree(
     reader: &heed::RoTxn<MainT>,
     ctx: &Context,
     query: &str,
 ) -> MResult<(Operation, HashMap<QueryId, Range<usize>>)>
 {
-    let words = split_query_string(query).map(str::to_lowercase);
-    let words: Vec<_> = words.filter(|s| !s.contains(char::is_whitespace)).enumerate().collect();
+    // TODO: use a shared analyzer instance
+    let words = split_query_string(query, &ctx.stop_words);
 
     let mut mapper = QueryWordsMapper::new(words.iter().map(|(_, w)| w));
-    let mut ngrams = Vec::new();
-    for ngram in 1..=MAX_NGRAM {
 
-        let ngiter = words.windows(ngram).enumerate().map(|(i, group)| {
-            let before = words[0..i].windows(1).enumerate().map(|(i, g)| (i..i+1, g));
-            let after = words[i + ngram..].windows(1)
-                .enumerate()
-                .map(move |(j, g)| (i + j + ngram..i + j + ngram + 1, g));
-            before.chain(Some((i..i + ngram, group))).chain(after)
-        });
+    fn create_inner(
+        reader: &heed::RoTxn<MainT>,
+        ctx: &Context,
+        mapper: &mut QueryWordsMapper,
+        words: &[(usize, String)],
+    ) -> MResult<Vec<Operation>>
+    {
+        let mut alts = Vec::new();
 
-        for group in ngiter {
+        for ngram in 1..=MAX_NGRAM {
+            if let Some(group) = words.get(..ngram) {
+                let mut group_ops = Vec::new();
 
-            let mut ops = Vec::new();
-            for (is_last, (range, words)) in is_last(group) {
+                let tail = &words[ngram..];
+                let is_last = tail.is_empty();
 
-                let mut alts = Vec::new();
-                match words {
+                let mut group_alts = Vec::new();
+                match group {
                     [(id, word)] => {
                         let mut idgen = ((id + 1) * 100)..;
+                        let range = (*id)..id+1;
 
                         let phrase = split_best_frequency(reader, ctx, word)?
                             .map(|ws| {
@@ -220,57 +232,69 @@ pub fn create_query_tree(
                         let synonyms = fetch_synonyms(reader, ctx, &[word])?
                             .into_iter()
                             .map(|alts| {
+                                let exact = alts.len() == 1;
                                 let id = idgen.next().unwrap();
                                 mapper.declare(range.clone(), id, &alts);
 
                                 let mut idgen = once(id).chain(&mut idgen);
                                 let iter = alts.into_iter().map(|w| {
                                     let id = idgen.next().unwrap();
-                                    Operation::exact(id, false, &w)
+                                    let kind = QueryKind::NonTolerant(w);
+                                    Operation::Query(Query { id, prefix: false, exact, kind })
                                 });
 
                                 create_operation(iter, Operation::And)
                             });
 
-                        let query = Operation::tolerant(*id, is_last, word);
+                        let original = Operation::tolerant(*id, is_last, word);
 
-                        alts.push(query);
-                        alts.extend(synonyms.chain(phrase));
+                        group_alts.push(original);
+                        group_alts.extend(synonyms.chain(phrase));
                     },
                     words => {
                         let id = words[0].0;
                         let mut idgen = ((id + 1) * 100_usize.pow(ngram as u32))..;
+                        let range = id..id+ngram;
 
                         let words: Vec<_> = words.iter().map(|(_, s)| s.as_str()).collect();
 
                         for synonym in fetch_synonyms(reader, ctx, &words)? {
+                            let exact = synonym.len() == 1;
                             let id = idgen.next().unwrap();
                             mapper.declare(range.clone(), id, &synonym);
 
                             let mut idgen = once(id).chain(&mut idgen);
                             let synonym = synonym.into_iter().map(|s| {
                                 let id = idgen.next().unwrap();
-                                Operation::exact(id, false, &s)
+                                let kind = QueryKind::NonTolerant(s);
+                                Operation::Query(Query { id, prefix: false, exact, kind })
                             });
-                            alts.push(create_operation(synonym, Operation::And));
+                            group_alts.push(create_operation(synonym, Operation::And));
                         }
 
                         let id = idgen.next().unwrap();
                         let concat = words.concat();
-                        alts.push(Operation::exact(id, is_last, &concat));
-                        mapper.declare(range.clone(), id, &[concat]);
+                        mapper.declare(range.clone(), id, &[&concat]);
+                        group_alts.push(Operation::non_tolerant(id, is_last, &concat));
                     }
                 }
 
-                ops.push(create_operation(alts, Operation::Or));
-            }
+                group_ops.push(create_operation(group_alts, Operation::Or));
 
-            ngrams.push(create_operation(ops, Operation::And));
-            if ngram == 1 { break }
+                if !tail.is_empty() {
+                    let tail_ops = create_inner(reader, ctx, mapper, tail)?;
+                    group_ops.push(create_operation(tail_ops, Operation::Or));
+                }
+
+                alts.push(create_operation(group_ops, Operation::And));
+            }
         }
+
+        Ok(alts)
     }
 
-    let operation = create_operation(ngrams, Operation::Or);
+    let alternatives = create_inner(reader, ctx, &mut mapper, &words)?;
+    let operation = Operation::Or(alternatives);
     let mapping = mapper.mapping();
 
     Ok((operation, mapping))
@@ -307,7 +331,7 @@ pub fn traverse_query_tree<'o, 'txn>(
         operations: &'o [Operation],
     ) -> MResult<Cow<'txn, Set<DocumentId>>>
     {
-        println!("{:1$}AND", "", depth * 2);
+        debug!("{:1$}AND", "", depth * 2);
 
         let before = Instant::now();
         let mut results = Vec::new();
@@ -332,7 +356,7 @@ pub fn traverse_query_tree<'o, 'txn>(
         let op = sdset::multi::Intersection::new(results);
         let docids = op.into_set_buf();
 
-        println!("{:3$}--- AND fetched {} documents in {:.02?}", "", docids.len(), before.elapsed(), depth * 2);
+        debug!("{:3$}--- AND fetched {} documents in {:.02?}", "", docids.len(), before.elapsed(), depth * 2);
 
         Ok(Cow::Owned(docids))
     }
@@ -346,7 +370,7 @@ pub fn traverse_query_tree<'o, 'txn>(
         operations: &'o [Operation],
     ) -> MResult<Cow<'txn, Set<DocumentId>>>
     {
-        println!("{:1$}OR", "", depth * 2);
+        debug!("{:1$}OR", "", depth * 2);
 
         let before = Instant::now();
         let mut results = Vec::new();
@@ -371,7 +395,7 @@ pub fn traverse_query_tree<'o, 'txn>(
         let op = sdset::multi::Union::new(results);
         let docids = op.into_set_buf();
 
-        println!("{:3$}--- OR fetched {} documents in {:.02?}", "", docids.len(), before.elapsed(), depth * 2);
+        debug!("{:3$}--- OR fetched {} documents in {:.02?}", "", docids.len(), before.elapsed(), depth * 2);
 
         Ok(Cow::Owned(docids))
     }
@@ -386,7 +410,7 @@ pub fn traverse_query_tree<'o, 'txn>(
     {
         let before = Instant::now();
 
-        let Query { prefix, kind, .. } = query;
+        let Query { prefix, kind, exact, .. } = query;
         let docids: Cow<Set<_>> = match kind {
             QueryKind::Tolerant(word) => {
                 if *prefix && word.len() <= 2 {
@@ -413,7 +437,7 @@ pub fn traverse_query_tree<'o, 'txn>(
 
                     let before = Instant::now();
                     let docids = sdset::duo::Union::new(prefix_docids, exact_docids).into_set_buf();
-                    println!("{:4$}prefix docids ({} and {}) construction took {:.02?}",
+                    debug!("{:4$}prefix docids ({} and {}) construction took {:.02?}",
                         "", prefix_docids.len(), exact_docids.len(), before.elapsed(), depth * 2);
 
                     Cow::Owned(docids)
@@ -433,13 +457,13 @@ pub fn traverse_query_tree<'o, 'txn>(
                     while let Some(input) = stream.next() {
                         if let Some(result) = ctx.postings_lists.postings_list(reader, input)? {
                             let distance = dfa.eval(input).to_u8();
-                            let is_exact = distance == 0 && input.len() == word.len();
+                            let is_exact = *exact && distance == 0 && input.len() == word.len();
                             results.push(result.docids);
                             let key = PostingsKey { query, input: input.to_owned(), distance, is_exact };
                             postings.insert(key, result.matches);
                         }
                     }
-                    println!("{:3$}docids retrieval ({:?}) took {:.02?}", "", results.len(), before.elapsed(), depth * 2);
+                    debug!("{:3$}docids retrieval ({:?}) took {:.02?}", "", results.len(), before.elapsed(), depth * 2);
 
                     let before = Instant::now();
                     let docids = if results.len() > 10 {
@@ -453,12 +477,12 @@ pub fn traverse_query_tree<'o, 'txn>(
                         let sets = results.iter().map(AsRef::as_ref).collect();
                         sdset::multi::Union::new(sets).into_set_buf()
                     };
-                    println!("{:2$}docids construction took {:.02?}", "", before.elapsed(), depth * 2);
+                    debug!("{:2$}docids construction took {:.02?}", "", before.elapsed(), depth * 2);
 
                     Cow::Owned(docids)
                 }
             },
-            QueryKind::Exact(word) => {
+            QueryKind::NonTolerant(word) => {
                 // TODO support prefix and non-prefix exact DFA
                 let dfa = build_exact_dfa(word);
 
@@ -475,11 +499,11 @@ pub fn traverse_query_tree<'o, 'txn>(
                     if let Some(result) = ctx.postings_lists.postings_list(reader, input)? {
                         let distance = dfa.eval(input).to_u8();
                         results.push(result.docids);
-                        let key = PostingsKey { query, input: input.to_owned(), distance, is_exact: true };
+                        let key = PostingsKey { query, input: input.to_owned(), distance, is_exact: *exact };
                         postings.insert(key, result.matches);
                     }
                 }
-                println!("{:3$}docids retrieval ({:?}) took {:.02?}", "", results.len(), before.elapsed(), depth * 2);
+                debug!("{:3$}docids retrieval ({:?}) took {:.02?}", "", results.len(), before.elapsed(), depth * 2);
 
                 let before = Instant::now();
                 let docids = if results.len() > 10 {
@@ -493,7 +517,7 @@ pub fn traverse_query_tree<'o, 'txn>(
                     let sets = results.iter().map(AsRef::as_ref).collect();
                     sdset::multi::Union::new(sets).into_set_buf()
                 };
-                println!("{:2$}docids construction took {:.02?}", "", before.elapsed(), depth * 2);
+                debug!("{:2$}docids construction took {:.02?}", "", before.elapsed(), depth * 2);
 
                 Cow::Owned(docids)
             },
@@ -518,21 +542,21 @@ pub fn traverse_query_tree<'o, 'txn>(
                     let mut docids: Vec<_> = matches.iter().map(|m| m.document_id).collect();
                     docids.dedup();
                     let docids = SetBuf::new(docids).unwrap();
-                    println!("{:2$}docids construction took {:.02?}", "", before.elapsed(), depth * 2);
+                    debug!("{:2$}docids construction took {:.02?}", "", before.elapsed(), depth * 2);
 
-                    let matches = Cow::Owned(SetBuf::new(matches).unwrap());
+                    let matches = Cow::Owned(SetBuf::from_dirty(matches));
                     let key = PostingsKey { query, input: vec![], distance: 0, is_exact: true };
                     postings.insert(key, matches);
 
                     Cow::Owned(docids)
                 } else {
-                    println!("{:2$}{:?} skipped", "", words, depth * 2);
+                    debug!("{:2$}{:?} skipped", "", words, depth * 2);
                     Cow::default()
                 }
             },
         };
 
-        println!("{:4$}{:?} fetched {:?} documents in {:.02?}", "", query, docids.len(), before.elapsed(), depth * 2);
+        debug!("{:4$}{:?} fetched {:?} documents in {:.02?}", "", query, docids.len(), before.elapsed(), depth * 2);
         Ok(docids)
     }
 

meilisearch-core/src/query_words_mapper.rs

@@ -19,6 +19,7 @@ impl QueryWordsMapper {
         QueryWordsMapper { originals, mappings: HashMap::new() }
     }
 
+    #[allow(clippy::len_zero)]
     pub fn declare<I, A>(&mut self, range: Range<usize>, id: QueryId, replacement: I)
     where I: IntoIterator<Item = A>,
           A: ToString,
@@ -53,7 +54,7 @@ impl QueryWordsMapper {
         }
 
         {
-            let replacement = replacement[common_left..replacement.len() - common_right].iter().cloned().collect();
+            let replacement = replacement[common_left..replacement.len() - common_right].to_vec();
             self.mappings.insert(id + common_left, (range.clone(), replacement));
         }
 

meilisearch-core/src/ranked_map.rs

@@ -1,14 +1,14 @@
 use std::io::{Read, Write};
 
 use hashbrown::HashMap;
-use meilisearch_schema::SchemaAttr;
+use meilisearch_schema::FieldId;
 use serde::{Deserialize, Serialize};
 
 use crate::{DocumentId, Number};
 
 #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(transparent)]
-pub struct RankedMap(HashMap<(DocumentId, SchemaAttr), Number>);
+pub struct RankedMap(HashMap<(DocumentId, FieldId), Number>);
 
 impl RankedMap {
     pub fn len(&self) -> usize {
@@ -19,16 +19,16 @@ impl RankedMap {
         self.0.is_empty()
     }
 
-    pub fn insert(&mut self, document: DocumentId, attribute: SchemaAttr, number: Number) {
-        self.0.insert((document, attribute), number);
+    pub fn insert(&mut self, document: DocumentId, field: FieldId, number: Number) {
+        self.0.insert((document, field), number);
     }
 
-    pub fn remove(&mut self, document: DocumentId, attribute: SchemaAttr) {
-        self.0.remove(&(document, attribute));
+    pub fn remove(&mut self, document: DocumentId, field: FieldId) {
+        self.0.remove(&(document, field));
     }
 
-    pub fn get(&self, document: DocumentId, attribute: SchemaAttr) -> Option<Number> {
-        self.0.get(&(document, attribute)).cloned()
+    pub fn get(&self, document: DocumentId, field: FieldId) -> Option<Number> {
+        self.0.get(&(document, field)).cloned()
     }
 
     pub fn read_from_bin<R: Read>(reader: R) -> bincode::Result<RankedMap> {

meilisearch-core/src/raw_indexer.rs

@@ -1,52 +1,60 @@
+use std::borrow::Cow;
 use std::collections::{BTreeMap, HashMap};
 use std::convert::TryFrom;
 
-use crate::{DocIndex, DocumentId};
-use deunicode::deunicode_with_tofu;
-use meilisearch_schema::SchemaAttr;
-use meilisearch_tokenizer::{is_cjk, SeqTokenizer, Token, Tokenizer};
+use meilisearch_schema::IndexedPos;
+use meilisearch_tokenizer::analyzer::{Analyzer, AnalyzerConfig};
+use meilisearch_tokenizer::{Token, token::SeparatorKind, TokenKind};
 use sdset::SetBuf;
 
+use crate::{DocIndex, DocumentId};
+use crate::FstSetCow;
+
 const WORD_LENGTH_LIMIT: usize = 80;
 
 type Word = Vec<u8>; // TODO make it be a SmallVec
 
-pub struct RawIndexer {
+pub struct RawIndexer<'a, A> {
     word_limit: usize, // the maximum number of indexed words
-    stop_words: fst::Set,
     words_doc_indexes: BTreeMap<Word, Vec<DocIndex>>,
     docs_words: HashMap<DocumentId, Vec<Word>>,
+    analyzer: Analyzer<'a, A>,
 }
 
-pub struct Indexed {
+pub struct Indexed<'a> {
     pub words_doc_indexes: BTreeMap<Word, SetBuf<DocIndex>>,
-    pub docs_words: HashMap<DocumentId, fst::Set>,
+    pub docs_words: HashMap<DocumentId, FstSetCow<'a>>,
 }
 
-impl RawIndexer {
-    pub fn new(stop_words: fst::Set) -> RawIndexer {
+impl<'a, A> RawIndexer<'a, A>
+where
+    A: AsRef<[u8]>
+{
+    pub fn new(stop_words: &'a fst::Set<A>) -> RawIndexer<'a, A> {
         RawIndexer::with_word_limit(stop_words, 1000)
     }
 
-    pub fn with_word_limit(stop_words: fst::Set, limit: usize) -> RawIndexer {
+    pub fn with_word_limit(stop_words: &'a fst::Set<A>, limit: usize) -> RawIndexer<A> {
         RawIndexer {
             word_limit: limit,
-            stop_words,
             words_doc_indexes: BTreeMap::new(),
             docs_words: HashMap::new(),
+            analyzer: Analyzer::new(AnalyzerConfig::default_with_stopwords(stop_words)),
         }
     }
 
-    pub fn index_text(&mut self, id: DocumentId, attr: SchemaAttr, text: &str) -> usize {
+    pub fn index_text(&mut self, id: DocumentId, indexed_pos: IndexedPos, text: &str) -> usize {
         let mut number_of_words = 0;
 
-        for token in Tokenizer::new(text) {
+        let analyzed_text = self.analyzer.analyze(text);
+        for (token_pos, (word_pos, token)) in process_tokens(analyzed_text.tokens()).enumerate() {
             let must_continue = index_token(
                 token,
+                word_pos,
+                token_pos,
                 id,
-                attr,
+                indexed_pos,
                 self.word_limit,
-                &self.stop_words,
                 &mut self.words_doc_indexes,
                 &mut self.docs_words,
             );
@@ -61,29 +69,42 @@ impl RawIndexer {
         number_of_words
     }
 
-    pub fn index_text_seq<'a, I>(&mut self, id: DocumentId, attr: SchemaAttr, iter: I)
+    pub fn index_text_seq<'s, I>(&mut self, id: DocumentId, indexed_pos: IndexedPos, text_iter: I)
     where
-        I: IntoIterator<Item = &'a str>,
+        I: IntoIterator<Item = &'s str>,
     {
-        let iter = iter.into_iter();
-        for token in SeqTokenizer::new(iter) {
-            let must_continue = index_token(
-                token,
-                id,
-                attr,
-                self.word_limit,
-                &self.stop_words,
-                &mut self.words_doc_indexes,
-                &mut self.docs_words,
-            );
+        let mut word_offset = 0;
 
-            if !must_continue {
-                break;
+        for text in text_iter.into_iter() {
+            let current_word_offset = word_offset;
+
+            let analyzed_text = self.analyzer.analyze(text);
+            let tokens = process_tokens(analyzed_text.tokens())
+                .map(|(i, t)| (i + current_word_offset, t))
+                .enumerate();
+
+            for (token_pos, (word_pos, token)) in tokens  {
+                word_offset = word_pos + 1;
+
+                let must_continue = index_token(
+                    token,
+                    word_pos,
+                    token_pos,
+                    id,
+                    indexed_pos,
+                    self.word_limit,
+                    &mut self.words_doc_indexes,
+                    &mut self.docs_words,
+                );
+
+                if !must_continue {
+                    break;
+                }
             }
         }
     }
 
-    pub fn build(self) -> Indexed {
+    pub fn build(self) -> Indexed<'static> {
         let words_doc_indexes = self
             .words_doc_indexes
             .into_iter()
@@ -96,7 +117,8 @@ impl RawIndexer {
             .map(|(id, mut words)| {
                 words.sort_unstable();
                 words.dedup();
-                (id, fst::Set::from_iter(words).unwrap())
+                let fst = fst::Set::from_iter(words).unwrap().map_data(Cow::Owned).unwrap();
+                (id, fst)
             })
             .collect();
 
@@ -107,29 +129,53 @@ impl RawIndexer {
     }
 }
 
+fn process_tokens<'a>(tokens: impl Iterator<Item = Token<'a>>) -> impl Iterator<Item = (usize, Token<'a>)> {
+    tokens
+        .skip_while(|token| !token.is_word())
+        .scan((0, None), |(offset, prev_kind), token| {
+                match token.kind {
+                    TokenKind::Word | TokenKind::StopWord | TokenKind::Unknown => {
+                        *offset += match *prev_kind {
+                            Some(TokenKind::Separator(SeparatorKind::Hard)) => 8,
+                            Some(_) => 1,
+                            None => 0,
+                        };
+                        *prev_kind = Some(token.kind)
+                    }
+                    TokenKind::Separator(SeparatorKind::Hard) => {
+                        *prev_kind = Some(token.kind);
+                    }
+                    TokenKind::Separator(SeparatorKind::Soft)
+                        if *prev_kind != Some(TokenKind::Separator(SeparatorKind::Hard)) => {
+                        *prev_kind = Some(token.kind);
+                    }
+                    _ => (),
+                }
+            Some((*offset, token))
+        })
+    .filter(|(_, t)| t.is_word())
+}
+
+#[allow(clippy::too_many_arguments)]
 fn index_token(
     token: Token,
+    word_pos: usize,
+    token_pos: usize,
     id: DocumentId,
-    attr: SchemaAttr,
+    indexed_pos: IndexedPos,
     word_limit: usize,
-    stop_words: &fst::Set,
     words_doc_indexes: &mut BTreeMap<Word, Vec<DocIndex>>,
     docs_words: &mut HashMap<DocumentId, Vec<Word>>,
-) -> bool {
-    if token.word_index >= word_limit {
+) -> bool
+{
+    if token_pos >= word_limit {
         return false;
     }
 
-    let lower = token.word.to_lowercase();
-    let token = Token {
-        word: &lower,
-        ..token
-    };
-
-    if !stop_words.contains(&token.word) {
-        match token_to_docindex(id, attr, token) {
+    if !token.is_stopword() {
+        match token_to_docindex(id, indexed_pos, &token, word_pos) {
             Some(docindex) => {
-                let word = Vec::from(token.word);
+                let word = Vec::from(token.word.as_ref());
 
                 if word.len() <= WORD_LENGTH_LIMIT {
                     words_doc_indexes
@@ -137,20 +183,6 @@ fn index_token(
                         .or_insert_with(Vec::new)
                         .push(docindex);
                     docs_words.entry(id).or_insert_with(Vec::new).push(word);
-
-                    if !lower.contains(is_cjk) {
-                        let unidecoded = deunicode_with_tofu(&lower, "");
-                        if unidecoded != lower && !unidecoded.is_empty() {
-                            let word = Vec::from(unidecoded);
-                            if word.len() <= WORD_LENGTH_LIMIT {
-                                words_doc_indexes
-                                    .entry(word.clone())
-                                    .or_insert_with(Vec::new)
-                                    .push(docindex);
-                                docs_words.entry(id).or_insert_with(Vec::new).push(word);
-                            }
-                        }
-                    }
                 }
             }
             None => return false,
@@ -160,14 +192,14 @@ fn index_token(
     true
 }
 
-fn token_to_docindex(id: DocumentId, attr: SchemaAttr, token: Token) -> Option<DocIndex> {
-    let word_index = u16::try_from(token.word_index).ok()?;
-    let char_index = u16::try_from(token.char_index).ok()?;
-    let char_length = u16::try_from(token.word.chars().count()).ok()?;
+fn token_to_docindex(id: DocumentId, indexed_pos: IndexedPos, token: &Token, word_index: usize) -> Option<DocIndex> {
+    let word_index = u16::try_from(word_index).ok()?;
+    let char_index = u16::try_from(token.byte_start).ok()?;
+    let char_length = u16::try_from(token.word.len()).ok()?;
 
     let docindex = DocIndex {
         document_id: id,
-        attribute: attr.0,
+        attribute: indexed_pos.0,
         word_index,
         char_index,
         char_length,
@@ -179,15 +211,29 @@ fn token_to_docindex(id: DocumentId, attr: SchemaAttr, token: Token) -> Option<D
 #[cfg(test)]
 mod tests {
     use super::*;
+    use meilisearch_schema::IndexedPos;
+    use meilisearch_tokenizer::{Analyzer, AnalyzerConfig};
+    use fst::Set;
+
+    #[test]
+    fn test_process_token() {
+        let text = " 為一包含一千多萬目詞的帶標記平衡語料庫";
+        let stopwords = Set::default();
+        let analyzer = Analyzer::new(AnalyzerConfig::default_with_stopwords(&stopwords));
+        let analyzer = analyzer.analyze(text);
+        let tokens: Vec<_> = process_tokens(analyzer.tokens()).map(|(_, t)| t.text().to_string()).collect();
+        assert_eq!(tokens, ["为", "一", "包含", "一千多万", "目词", "的", "带", "标记", "平衡", "语料库"]);
+    }
 
     #[test]
     fn strange_apostrophe() {
-        let mut indexer = RawIndexer::new(fst::Set::default());
+        let stop_words = fst::Set::default();
+        let mut indexer = RawIndexer::new(&stop_words);
 
         let docid = DocumentId(0);
-        let attr = SchemaAttr(0);
+        let indexed_pos = IndexedPos(0);
         let text = "Zut, l’aspirateur, j’ai oublié de l’éteindre !";
-        indexer.index_text(docid, attr, text);
+        indexer.index_text(docid, indexed_pos, text);
 
         let Indexed {
             words_doc_indexes, ..
@@ -197,19 +243,17 @@ mod tests {
         assert!(words_doc_indexes.get(&b"aspirateur"[..]).is_some());
         assert!(words_doc_indexes.get(&b"ai"[..]).is_some());
         assert!(words_doc_indexes.get(&b"eteindre"[..]).is_some());
-        assert!(words_doc_indexes
-            .get(&"éteindre".to_owned().into_bytes())
-            .is_some());
     }
 
     #[test]
     fn strange_apostrophe_in_sequence() {
-        let mut indexer = RawIndexer::new(fst::Set::default());
+        let stop_words = fst::Set::default();
+        let mut indexer = RawIndexer::new(&stop_words);
 
         let docid = DocumentId(0);
-        let attr = SchemaAttr(0);
+        let indexed_pos = IndexedPos(0);
         let text = vec!["Zut, l’aspirateur, j’ai oublié de l’éteindre !"];
-        indexer.index_text_seq(docid, attr, text);
+        indexer.index_text_seq(docid, indexed_pos, text);
 
         let Indexed {
             words_doc_indexes, ..
@@ -219,9 +263,6 @@ mod tests {
         assert!(words_doc_indexes.get(&b"aspirateur"[..]).is_some());
         assert!(words_doc_indexes.get(&b"ai"[..]).is_some());
         assert!(words_doc_indexes.get(&b"eteindre"[..]).is_some());
-        assert!(words_doc_indexes
-            .get(&"éteindre".to_owned().into_bytes())
-            .is_some());
     }
 
     #[test]
@@ -229,12 +270,12 @@ mod tests {
         let stop_words = sdset::SetBuf::from_dirty(vec!["l", "j", "ai", "de"]);
         let stop_words = fst::Set::from_iter(stop_words).unwrap();
 
-        let mut indexer = RawIndexer::new(stop_words);
+        let mut indexer = RawIndexer::new(&stop_words);
 
         let docid = DocumentId(0);
-        let attr = SchemaAttr(0);
+        let indexed_pos = IndexedPos(0);
         let text = "Zut, l’aspirateur, j’ai oublié de l’éteindre !";
-        indexer.index_text(docid, attr, text);
+        indexer.index_text(docid, indexed_pos, text);
 
         let Indexed {
             words_doc_indexes, ..
@@ -246,19 +287,17 @@ mod tests {
         assert!(words_doc_indexes.get(&b"ai"[..]).is_none());
         assert!(words_doc_indexes.get(&b"de"[..]).is_none());
         assert!(words_doc_indexes.get(&b"eteindre"[..]).is_some());
-        assert!(words_doc_indexes
-            .get(&"éteindre".to_owned().into_bytes())
-            .is_some());
     }
 
     #[test]
     fn no_empty_unidecode() {
-        let mut indexer = RawIndexer::new(fst::Set::default());
+        let stop_words = fst::Set::default();
+        let mut indexer = RawIndexer::new(&stop_words);
 
         let docid = DocumentId(0);
-        let attr = SchemaAttr(0);
+        let indexed_pos = IndexedPos(0);
         let text = "🇯🇵";
-        indexer.index_text(docid, attr, text);
+        indexer.index_text(docid, indexed_pos, text);
 
         let Indexed {
             words_doc_indexes, ..
@@ -268,4 +307,38 @@ mod tests {
             .get(&"🇯🇵".to_owned().into_bytes())
             .is_some());
     }
+
+    #[test]
+    // test sample from 807
+    fn very_long_text() {
+        let stop_words = fst::Set::default();
+        let mut indexer = RawIndexer::new(&stop_words);
+        let indexed_pos = IndexedPos(0);
+        let docid = DocumentId(0);
+        let text = " The locations block is the most powerful, and potentially most involved, section of the .platform.app.yaml file. It allows you to control how the application container responds to incoming requests at a very fine-grained level. Common patterns also vary between language containers due to the way PHP-FPM handles incoming requests.\nEach entry of the locations block is an absolute URI path (with leading /) and its value includes the configuration directives for how the web server should handle matching requests. That is, if your domain is example.com then '/' means &ldquo;requests for example.com/&rdquo;, while '/admin' means &ldquo;requests for example.com/admin&rdquo;. If multiple blocks could match an incoming request then the most-specific will apply.\nweb:locations:&#39;/&#39;:# Rules for all requests that don&#39;t otherwise match....&#39;/sites/default/files&#39;:# Rules for any requests that begin with /sites/default/files....The simplest possible locations configuration is one that simply passes all requests on to your application unconditionally:\nweb:locations:&#39;/&#39;:passthru:trueThat is, all requests to /* should be forwarded to the process started by web.commands.start above. Note that for PHP containers the passthru key must specify what PHP file the request should be forwarded to, and must also specify a docroot under which the file lives. For example:\nweb:locations:&#39;/&#39;:root:&#39;web&#39;passthru:&#39;/app.php&#39;This block will serve requests to / from the web directory in the application, and if a file doesn&rsquo;t exist on disk then the request will be forwarded to the /app.php script.\nA full list of the possible subkeys for locations is below.\n  root: The folder from which to serve static assets for this location relative to the application root. The application root is the directory in which the .platform.app.yaml file is located. Typical values for this property include public or web. Setting it to '' is not recommended, and its behavior may vary depending on the type of application. Absolute paths are not supported.\n  passthru: Whether to forward disallowed and missing resources from this location to the application and can be true, false or an absolute URI path (with leading /). The default value is false. For non-PHP applications it will generally be just true or false. In a PHP application this will typically be the front controller such as /index.php or /app.php. This entry works similar to mod_rewrite under Apache. Note: If the value of passthru does not begin with the same value as the location key it is under, the passthru may evaluate to another entry. That may be useful when you want different cache settings for different paths, for instance, but want missing files in all of them to map back to the same front controller. See the example block below.\n  index: The files to consider when serving a request for a directory: an array of file names or null. (typically ['index.html']). Note that in order for this to work, access to the static files named must be allowed by the allow or rules keys for this location.\n  expires: How long to allow static assets from this location to be cached (this enables the Cache-Control and Expires headers) and can be a time or -1 for no caching (default). Times can be suffixed with &ldquo;ms&rdquo; (milliseconds), &ldquo;s&rdquo; (seconds), &ldquo;m&rdquo; (minutes), &ldquo;h&rdquo; (hours), &ldquo;d&rdquo; (days), &ldquo;w&rdquo; (weeks), &ldquo;M&rdquo; (months, 30d) or &ldquo;y&rdquo; (years, 365d).\n  scripts: Whether to allow loading scripts in that location (true or false). This directive is only meaningful on PHP.\n  allow: Whether to allow serving files which don&rsquo;t match a rule (true or false, default: true).\n  headers: Any additional headers to apply to static assets. This section is a mapping of header names to header values. Responses from the application aren&rsquo;t affected, to avoid overlap with the application&rsquo;s own ability to include custom headers in the response.\n  rules: Specific overrides for a specific location. The key is a PCRE (regular expression) that is matched against the full request path.\n  request_buffering: Most application servers do not support chunked requests (e.g. fpm, uwsgi), so Platform.sh enables request_buffering by default to handle them. That default configuration would look like this if it was present in .platform.app.yaml:\nweb:locations:&#39;/&#39;:passthru:truerequest_buffering:enabled:truemax_request_size:250mIf the application server can already efficiently handle chunked requests, the request_buffering subkey can be modified to disable it entirely (enabled: false). Additionally, applications that frequently deal with uploads greater than 250MB in size can update the max_request_size key to the application&rsquo;s needs. Note that modifications to request_buffering will need to be specified at each location where it is desired.\n ";
+        indexer.index_text(docid, indexed_pos, text);
+        let Indexed {
+            words_doc_indexes, ..
+        } = indexer.build();
+        assert!(words_doc_indexes.get(&"request".to_owned().into_bytes()).is_some());
+    }
+
+    #[test]
+    fn words_over_index_1000_not_indexed() {
+        let stop_words = fst::Set::default();
+        let mut indexer = RawIndexer::new(&stop_words);
+        let indexed_pos = IndexedPos(0);
+        let docid = DocumentId(0);
+        let mut text = String::with_capacity(5000);
+        for _ in 0..1000 {
+            text.push_str("less ");
+        }
+        text.push_str("more");
+        indexer.index_text(docid, indexed_pos, &text);
+        let Indexed {
+            words_doc_indexes, ..
+        } = indexer.build();
+        assert!(words_doc_indexes.get(&"less".to_owned().into_bytes()).is_some());
+        assert!(words_doc_indexes.get(&"more".to_owned().into_bytes()).is_none());
+    }
 }

meilisearch-core/src/serde/convert_to_number.rs

@@ -1,198 +0,0 @@
-use std::str::FromStr;
-
-use ordered_float::OrderedFloat;
-use serde::ser;
-use serde::Serialize;
-
-use super::SerializerError;
-use crate::Number;
-
-pub struct ConvertToNumber;
-
-impl ser::Serializer for ConvertToNumber {
-    type Ok = Number;
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    fn serialize_bool(self, value: bool) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_char(self, _value: char) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "char" })
-    }
-
-    fn serialize_i8(self, value: i8) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(i64::from(value)))
-    }
-
-    fn serialize_i16(self, value: i16) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(i64::from(value)))
-    }
-
-    fn serialize_i32(self, value: i32) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(i64::from(value)))
-    }
-
-    fn serialize_i64(self, value: i64) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(value))
-    }
-
-    fn serialize_u8(self, value: u8) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_u16(self, value: u16) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_u32(self, value: u32) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_u64(self, value: u64) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(value))
-    }
-
-    fn serialize_f32(self, value: f32) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Float(OrderedFloat(f64::from(value))))
-    }
-
-    fn serialize_f64(self, value: f64) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Float(OrderedFloat(value)))
-    }
-
-    fn serialize_str(self, value: &str) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::from_str(value)?)
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnrankableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnrankableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "map" })
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "struct",
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "struct variant",
-        })
-    }
-}

meilisearch-core/src/serde/convert_to_string.rs

@@ -1,258 +0,0 @@
-use serde::ser;
-use serde::Serialize;
-
-use super::SerializerError;
-
-pub struct ConvertToString;
-
-impl ser::Serializer for ConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = MapConvertToString;
-    type SerializeStruct = StructConvertToString;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    fn serialize_bool(self, _value: bool) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "boolean",
-        })
-    }
-
-    fn serialize_char(self, value: char) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i8(self, value: i8) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i16(self, value: i16) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i32(self, value: i32) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i64(self, value: i64) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u8(self, value: u8) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u16(self, value: u16) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u32(self, value: u32) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u64(self, value: u64) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_f32(self, value: f32) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_f64(self, value: f64) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_str(self, value: &str) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        Ok(MapConvertToString {
-            text: String::new(),
-        })
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Ok(StructConvertToString {
-            text: String::new(),
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct MapConvertToString {
-    text: String,
-}
-
-impl ser::SerializeMap for MapConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = key.serialize(ConvertToString)?;
-        self.text.push_str(&text);
-        self.text.push_str(" ");
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.text.push_str(&text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.text)
-    }
-}
-
-pub struct StructConvertToString {
-    text: String,
-}
-
-impl ser::SerializeStruct for StructConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let value = value.serialize(ConvertToString)?;
-        self.text.push_str(key);
-        self.text.push_str(" ");
-        self.text.push_str(&value);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.text)
-    }
-}

meilisearch-core/src/serde/deserializer.rs

@@ -2,7 +2,7 @@ use std::collections::HashSet;
 use std::io::Cursor;
 use std::{error::Error, fmt};
 
-use meilisearch_schema::{Schema, SchemaAttr};
+use meilisearch_schema::{Schema, FieldId};
 use serde::{de, forward_to_deserialize_any};
 use serde_json::de::IoRead as SerdeJsonIoRead;
 use serde_json::Deserializer as SerdeJsonDeserializer;
@@ -51,10 +51,10 @@ impl From<heed::Error> for DeserializerError {
 
 pub struct Deserializer<'a> {
     pub document_id: DocumentId,
-    pub reader: &'a heed::RoTxn<MainT>,
+    pub reader: &'a heed::RoTxn<'a, MainT>,
     pub documents_fields: DocumentsFields,
     pub schema: &'a Schema,
-    pub attributes: Option<&'a HashSet<SchemaAttr>>,
+    pub fields: Option<&'a HashSet<FieldId>>,
 }
 
 impl<'de, 'a, 'b> de::Deserializer<'de> for &'b mut Deserializer<'a> {
@@ -92,15 +92,17 @@ impl<'de, 'a, 'b> de::Deserializer<'de> for &'b mut Deserializer<'a> {
                     }
                 };
 
-                let is_displayed = self.schema.props(attr).is_displayed();
-                if is_displayed && self.attributes.map_or(true, |f| f.contains(&attr)) {
-                    let attribute_name = self.schema.attribute_name(attr);
+                let is_displayed = self.schema.is_displayed(attr);
+                if is_displayed && self.fields.map_or(true, |f| f.contains(&attr)) {
+                    if let Some(attribute_name) = self.schema.name(attr) {
+                        let cursor = Cursor::new(value.to_owned());
+                        let ioread = SerdeJsonIoRead::new(cursor);
+                        let value = Value(SerdeJsonDeserializer::new(ioread));
 
-                    let cursor = Cursor::new(value.to_owned());
-                    let ioread = SerdeJsonIoRead::new(cursor);
-                    let value = Value(SerdeJsonDeserializer::new(ioread));
-
-                    Some((attribute_name, value))
+                        Some((attribute_name, value))
+                    } else {
+                        None
+                    }
                 } else {
                     None
                 }

meilisearch-core/src/serde/extract_document_id.rs

@@ -1,295 +0,0 @@
-use std::hash::{Hash, Hasher};
-
-use crate::DocumentId;
-use serde::{ser, Serialize};
-use serde_json::Value;
-use siphasher::sip::SipHasher;
-
-use super::{ConvertToString, SerializerError};
-
-pub fn extract_document_id<D>(
-    identifier: &str,
-    document: &D,
-) -> Result<Option<DocumentId>, SerializerError>
-where
-    D: serde::Serialize,
-{
-    let serializer = ExtractDocumentId { identifier };
-    document.serialize(serializer)
-}
-
-pub fn value_to_string(value: &Value) -> Option<String> {
-    match value {
-        Value::Null => None,
-        Value::Bool(_) => None,
-        Value::Number(value) => Some(value.to_string()),
-        Value::String(value) => Some(value.to_string()),
-        Value::Array(_) => None,
-        Value::Object(_) => None,
-    }
-}
-
-pub fn compute_document_id<H: Hash>(t: H) -> DocumentId {
-    let mut s = SipHasher::new();
-    t.hash(&mut s);
-    let hash = s.finish();
-    DocumentId(hash)
-}
-
-struct ExtractDocumentId<'a> {
-    identifier: &'a str,
-}
-
-impl<'a> ser::Serializer for ExtractDocumentId<'a> {
-    type Ok = Option<DocumentId>;
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = ExtractDocumentIdMapSerializer<'a>;
-    type SerializeStruct = ExtractDocumentIdStructSerializer<'a>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    forward_to_unserializable_type! {
-        bool => serialize_bool,
-        char => serialize_char,
-
-        i8  => serialize_i8,
-        i16 => serialize_i16,
-        i32 => serialize_i32,
-        i64 => serialize_i64,
-
-        u8  => serialize_u8,
-        u16 => serialize_u16,
-        u32 => serialize_u32,
-        u64 => serialize_u64,
-
-        f32 => serialize_f32,
-        f64 => serialize_f64,
-    }
-
-    fn serialize_str(self, _value: &str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "str" })
-    }
-
-    fn serialize_bytes(self, _value: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        let serializer = ExtractDocumentIdMapSerializer {
-            identifier: self.identifier,
-            document_id: None,
-            current_key_name: None,
-        };
-
-        Ok(serializer)
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        let serializer = ExtractDocumentIdStructSerializer {
-            identifier: self.identifier,
-            document_id: None,
-        };
-
-        Ok(serializer)
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct ExtractDocumentIdMapSerializer<'a> {
-    identifier: &'a str,
-    document_id: Option<DocumentId>,
-    current_key_name: Option<String>,
-}
-
-impl<'a> ser::SerializeMap for ExtractDocumentIdMapSerializer<'a> {
-    type Ok = Option<DocumentId>;
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-        self.current_key_name = Some(key);
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        let key = self.current_key_name.take().unwrap();
-        self.serialize_entry(&key, value)
-    }
-
-    fn serialize_entry<K: ?Sized, V: ?Sized>(
-        &mut self,
-        key: &K,
-        value: &V,
-    ) -> Result<(), Self::Error>
-    where
-        K: Serialize,
-        V: Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-
-        if self.identifier == key {
-            let value = serde_json::to_string(value).and_then(|s| serde_json::from_str(&s))?;
-            match value_to_string(&value).map(|s| compute_document_id(&s)) {
-                Some(document_id) => self.document_id = Some(document_id),
-                None => return Err(SerializerError::InvalidDocumentIdType),
-            }
-        }
-
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.document_id)
-    }
-}
-
-pub struct ExtractDocumentIdStructSerializer<'a> {
-    identifier: &'a str,
-    document_id: Option<DocumentId>,
-}
-
-impl<'a> ser::SerializeStruct for ExtractDocumentIdStructSerializer<'a> {
-    type Ok = Option<DocumentId>;
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        if self.identifier == key {
-            let value = serde_json::to_string(value).and_then(|s| serde_json::from_str(&s))?;
-            match value_to_string(&value).map(compute_document_id) {
-                Some(document_id) => self.document_id = Some(document_id),
-                None => return Err(SerializerError::InvalidDocumentIdType),
-            }
-        }
-
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.document_id)
-    }
-}

meilisearch-core/src/serde/indexer.rs

@@ -1,362 +0,0 @@
-use meilisearch_schema::SchemaAttr;
-use serde::ser;
-use serde::Serialize;
-
-use super::{ConvertToString, SerializerError};
-use crate::raw_indexer::RawIndexer;
-use crate::DocumentId;
-
-pub struct Indexer<'a> {
-    pub attribute: SchemaAttr,
-    pub indexer: &'a mut RawIndexer,
-    pub document_id: DocumentId,
-}
-
-impl<'a> ser::Serializer for Indexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-    type SerializeSeq = SeqIndexer<'a>;
-    type SerializeTuple = TupleIndexer<'a>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = MapIndexer<'a>;
-    type SerializeStruct = StructIndexer<'a>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    fn serialize_bool(self, _value: bool) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_char(self, value: char) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i8(self, value: i8) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i16(self, value: i16) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i32(self, value: i32) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i64(self, value: i64) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u8(self, value: u8) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u16(self, value: u16) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u32(self, value: u32) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u64(self, value: u64) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_f32(self, value: f32) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_f64(self, value: f64) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_str(self, text: &str) -> Result<Self::Ok, Self::Error> {
-        let number_of_words = self
-            .indexer
-            .index_text(self.document_id, self.attribute, text);
-        Ok(Some(number_of_words))
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_some<T: ?Sized>(self, value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        let number_of_words = self
-            .indexer
-            .index_text(self.document_id, self.attribute, &text);
-        Ok(Some(number_of_words))
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        Err(SerializerError::UnindexableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        let indexer = SeqIndexer {
-            attribute: self.attribute,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        let indexer = TupleIndexer {
-            attribute: self.attribute,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        let indexer = MapIndexer {
-            attribute: self.attribute,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        let indexer = StructIndexer {
-            attribute: self.attribute,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct SeqIndexer<'a> {
-    attribute: SchemaAttr,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeSeq for SeqIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_element<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.attribute, texts);
-        Ok(None)
-    }
-}
-
-pub struct MapIndexer<'a> {
-    attribute: SchemaAttr,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeMap for MapIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = key.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.attribute, texts);
-        Ok(None)
-    }
-}
-
-pub struct StructIndexer<'a> {
-    attribute: SchemaAttr,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeStruct for StructIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let key_text = key.to_owned();
-        let value_text = value.serialize(ConvertToString)?;
-        self.texts.push(key_text);
-        self.texts.push(value_text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.attribute, texts);
-        Ok(None)
-    }
-}
-
-pub struct TupleIndexer<'a> {
-    attribute: SchemaAttr,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeTuple for TupleIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_element<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.attribute, texts);
-        Ok(None)
-    }
-}

meilisearch-core/src/serde/mod.rs

@@ -1,41 +1,23 @@
-macro_rules! forward_to_unserializable_type {
-    ($($ty:ident => $se_method:ident,)*) => {
-        $(
-            fn $se_method(self, _v: $ty) -> Result<Self::Ok, Self::Error> {
-                Err(SerializerError::UnserializableType { type_name: "$ty" })
-            }
-        )*
-    }
-}
-
-mod convert_to_number;
-mod convert_to_string;
 mod deserializer;
-mod extract_document_id;
-mod indexer;
-mod serializer;
 
-pub use self::convert_to_number::ConvertToNumber;
-pub use self::convert_to_string::ConvertToString;
 pub use self::deserializer::{Deserializer, DeserializerError};
-pub use self::extract_document_id::{compute_document_id, extract_document_id, value_to_string};
-pub use self::indexer::Indexer;
-pub use self::serializer::{serialize_value, Serializer};
 
 use std::{error::Error, fmt};
 
 use serde::ser;
 use serde_json::Error as SerdeJsonError;
+use meilisearch_schema::Error as SchemaError;
 
 use crate::ParseNumberError;
 
 #[derive(Debug)]
 pub enum SerializerError {
     DocumentIdNotFound,
-    InvalidDocumentIdType,
+    InvalidDocumentIdFormat,
     Zlmdb(heed::Error),
     SerdeJson(SerdeJsonError),
     ParseNumber(ParseNumberError),
+    Schema(SchemaError),
     UnserializableType { type_name: &'static str },
     UnindexableType { type_name: &'static str },
     UnrankableType { type_name: &'static str },
@@ -52,16 +34,17 @@ impl fmt::Display for SerializerError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
             SerializerError::DocumentIdNotFound => {
-                f.write_str("serialized document does not have an id according to the schema")
+                f.write_str("Primary key is missing.")
             }
-            SerializerError::InvalidDocumentIdType => {
-                f.write_str("document identifier can only be of type string or number")
+            SerializerError::InvalidDocumentIdFormat => {
+                f.write_str("a document primary key can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_).")
             }
             SerializerError::Zlmdb(e) => write!(f, "heed related error: {}", e),
             SerializerError::SerdeJson(e) => write!(f, "serde json error: {}", e),
             SerializerError::ParseNumber(e) => {
                 write!(f, "error while trying to parse a number: {}", e)
             }
+            SerializerError::Schema(e) => write!(f, "impossible to update schema: {}", e),
             SerializerError::UnserializableType { type_name } => {
                 write!(f, "{} is not a serializable type", type_name)
             }
@@ -101,3 +84,9 @@ impl From<ParseNumberError> for SerializerError {
         SerializerError::ParseNumber(error)
     }
 }
+
+impl From<SchemaError> for SerializerError {
+    fn from(error: SchemaError) -> SerializerError {
+        SerializerError::Schema(error)
+   }
+}

meilisearch-core/src/serde/serializer.rs

@@ -1,339 +0,0 @@
-use meilisearch_schema::{Schema, SchemaAttr, SchemaProps};
-use serde::ser;
-
-use crate::database::MainT;
-use crate::raw_indexer::RawIndexer;
-use crate::store::{DocumentsFields, DocumentsFieldsCounts};
-use crate::{DocumentId, RankedMap};
-
-use super::{ConvertToNumber, ConvertToString, Indexer, SerializerError};
-
-pub struct Serializer<'a, 'b> {
-    pub txn: &'a mut heed::RwTxn<'b, MainT>,
-    pub schema: &'a Schema,
-    pub document_store: DocumentsFields,
-    pub document_fields_counts: DocumentsFieldsCounts,
-    pub indexer: &'a mut RawIndexer,
-    pub ranked_map: &'a mut RankedMap,
-    pub document_id: DocumentId,
-}
-
-impl<'a, 'b> ser::Serializer for Serializer<'a, 'b> {
-    type Ok = ();
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = MapSerializer<'a, 'b>;
-    type SerializeStruct = StructSerializer<'a, 'b>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    forward_to_unserializable_type! {
-        bool => serialize_bool,
-        char => serialize_char,
-
-        i8  => serialize_i8,
-        i16 => serialize_i16,
-        i32 => serialize_i32,
-        i64 => serialize_i64,
-
-        u8  => serialize_u8,
-        u16 => serialize_u16,
-        u32 => serialize_u32,
-        u64 => serialize_u64,
-
-        f32 => serialize_f32,
-        f64 => serialize_f64,
-    }
-
-    fn serialize_str(self, _v: &str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "str" })
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        Ok(MapSerializer {
-            txn: self.txn,
-            schema: self.schema,
-            document_id: self.document_id,
-            document_store: self.document_store,
-            document_fields_counts: self.document_fields_counts,
-            indexer: self.indexer,
-            ranked_map: self.ranked_map,
-            current_key_name: None,
-        })
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Ok(StructSerializer {
-            txn: self.txn,
-            schema: self.schema,
-            document_id: self.document_id,
-            document_store: self.document_store,
-            document_fields_counts: self.document_fields_counts,
-            indexer: self.indexer,
-            ranked_map: self.ranked_map,
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct MapSerializer<'a, 'b> {
-    txn: &'a mut heed::RwTxn<'b, MainT>,
-    schema: &'a Schema,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    document_fields_counts: DocumentsFieldsCounts,
-    indexer: &'a mut RawIndexer,
-    ranked_map: &'a mut RankedMap,
-    current_key_name: Option<String>,
-}
-
-impl<'a, 'b> ser::SerializeMap for MapSerializer<'a, 'b> {
-    type Ok = ();
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-        self.current_key_name = Some(key);
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let key = self.current_key_name.take().unwrap();
-        self.serialize_entry(&key, value)
-    }
-
-    fn serialize_entry<K: ?Sized, V: ?Sized>(
-        &mut self,
-        key: &K,
-        value: &V,
-    ) -> Result<(), Self::Error>
-    where
-        K: ser::Serialize,
-        V: ser::Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-        match self.schema.attribute(&key) {
-            Some(attribute) => serialize_value(
-                self.txn,
-                attribute,
-                self.schema.props(attribute),
-                self.document_id,
-                self.document_store,
-                self.document_fields_counts,
-                self.indexer,
-                self.ranked_map,
-                value,
-            ),
-            None => Ok(()),
-        }
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(())
-    }
-}
-
-pub struct StructSerializer<'a, 'b> {
-    txn: &'a mut heed::RwTxn<'b, MainT>,
-    schema: &'a Schema,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    document_fields_counts: DocumentsFieldsCounts,
-    indexer: &'a mut RawIndexer,
-    ranked_map: &'a mut RankedMap,
-}
-
-impl<'a, 'b> ser::SerializeStruct for StructSerializer<'a, 'b> {
-    type Ok = ();
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        match self.schema.attribute(key) {
-            Some(attribute) => serialize_value(
-                self.txn,
-                attribute,
-                self.schema.props(attribute),
-                self.document_id,
-                self.document_store,
-                self.document_fields_counts,
-                self.indexer,
-                self.ranked_map,
-                value,
-            ),
-            None => Ok(()),
-        }
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(())
-    }
-}
-
-pub fn serialize_value<T: ?Sized>(
-    txn: &mut heed::RwTxn<MainT>,
-    attribute: SchemaAttr,
-    props: SchemaProps,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    documents_fields_counts: DocumentsFieldsCounts,
-    indexer: &mut RawIndexer,
-    ranked_map: &mut RankedMap,
-    value: &T,
-) -> Result<(), SerializerError>
-where
-    T: ser::Serialize,
-{
-    let serialized = serde_json::to_vec(value)?;
-    document_store.put_document_field(txn, document_id, attribute, &serialized)?;
-
-    if props.is_indexed() {
-        let indexer = Indexer {
-            attribute,
-            indexer,
-            document_id,
-        };
-        if let Some(number_of_words) = value.serialize(indexer)? {
-            documents_fields_counts.put_document_field_count(
-                txn,
-                document_id,
-                attribute,
-                number_of_words as u16,
-            )?;
-        }
-    }
-
-    if props.is_ranked() {
-        let number = value.serialize(ConvertToNumber)?;
-        ranked_map.insert(document_id, attribute, number);
-    }
-
-    Ok(())
-}

meilisearch-core/src/settings.rs

@@ -0,0 +1,183 @@
+use std::collections::{BTreeMap, BTreeSet};
+use std::str::FromStr;
+use std::iter::IntoIterator;
+
+use serde::{Deserialize, Deserializer, Serialize};
+use once_cell::sync::Lazy;
+
+use self::RankingRule::*;
+
+pub const DEFAULT_RANKING_RULES: [RankingRule; 6] = [Typo, Words, Proximity, Attribute, WordsPosition, Exactness];
+
+static RANKING_RULE_REGEX: Lazy<regex::Regex> = Lazy::new(|| {
+    regex::Regex::new(r"(asc|desc)\(([a-zA-Z0-9-_]*)\)").unwrap()
+});
+
+#[derive(Default, Clone, Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+pub struct Settings {
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub ranking_rules: Option<Option<Vec<String>>>,
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub distinct_attribute: Option<Option<String>>,
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub searchable_attributes: Option<Option<Vec<String>>>,
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub displayed_attributes: Option<Option<BTreeSet<String>>>,
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub stop_words: Option<Option<BTreeSet<String>>>,
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub synonyms: Option<Option<BTreeMap<String, Vec<String>>>>,
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub attributes_for_faceting: Option<Option<Vec<String>>>,
+}
+
+// Any value that is present is considered Some value, including null.
+fn deserialize_some<'de, T, D>(deserializer: D) -> Result<Option<T>, D::Error>
+    where T: Deserialize<'de>,
+          D: Deserializer<'de>
+{
+    Deserialize::deserialize(deserializer).map(Some)
+}
+
+impl Settings {
+    pub fn to_update(&self) -> Result<SettingsUpdate, RankingRuleConversionError> {
+        let settings = self.clone();
+
+        let ranking_rules = match settings.ranking_rules {
+            Some(Some(rules)) => UpdateState::Update(RankingRule::try_from_iter(rules.iter())?),
+            Some(None) => UpdateState::Clear,
+            None => UpdateState::Nothing,
+        };
+
+        Ok(SettingsUpdate {
+            ranking_rules,
+            distinct_attribute: settings.distinct_attribute.into(),
+            primary_key: UpdateState::Nothing,
+            searchable_attributes: settings.searchable_attributes.into(),
+            displayed_attributes: settings.displayed_attributes.into(),
+            stop_words: settings.stop_words.into(),
+            synonyms: settings.synonyms.into(),
+            attributes_for_faceting: settings.attributes_for_faceting.into(),
+        })
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum UpdateState<T> {
+    Update(T),
+    Clear,
+    Nothing,
+}
+
+impl <T> From<Option<Option<T>>> for UpdateState<T> {
+    fn from(opt: Option<Option<T>>) -> UpdateState<T> {
+        match opt {
+            Some(Some(t)) => UpdateState::Update(t),
+            Some(None) => UpdateState::Clear,
+            None => UpdateState::Nothing,
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct RankingRuleConversionError;
+
+impl std::fmt::Display for RankingRuleConversionError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "impossible to convert into RankingRule")
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum RankingRule {
+    Typo,
+    Words,
+    Proximity,
+    Attribute,
+    WordsPosition,
+    Exactness,
+    Asc(String),
+    Desc(String),
+}
+
+impl std::fmt::Display for RankingRule {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match self {
+            RankingRule::Typo => f.write_str("typo"),
+            RankingRule::Words => f.write_str("words"),
+            RankingRule::Proximity => f.write_str("proximity"),
+            RankingRule::Attribute => f.write_str("attribute"),
+            RankingRule::WordsPosition => f.write_str("wordsPosition"),
+            RankingRule::Exactness => f.write_str("exactness"),
+            RankingRule::Asc(field) => write!(f, "asc({})", field),
+            RankingRule::Desc(field) => write!(f, "desc({})", field),
+        }
+    }
+}
+
+impl FromStr for RankingRule {
+    type Err = RankingRuleConversionError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let rule = match s {
+            "typo" => RankingRule::Typo,
+            "words" => RankingRule::Words,
+            "proximity" => RankingRule::Proximity,
+            "attribute" => RankingRule::Attribute,
+            "wordsPosition" => RankingRule::WordsPosition,
+            "exactness" => RankingRule::Exactness,
+            _ => {
+                let captures = RANKING_RULE_REGEX.captures(s).ok_or(RankingRuleConversionError)?;
+                match (captures.get(1).map(|m| m.as_str()), captures.get(2)) {
+                    (Some("asc"), Some(field)) => RankingRule::Asc(field.as_str().to_string()),
+                    (Some("desc"), Some(field)) => RankingRule::Desc(field.as_str().to_string()),
+                    _ => return Err(RankingRuleConversionError)
+                }
+            }
+        };
+        Ok(rule)
+    }
+}
+
+impl RankingRule {
+    pub fn field(&self) -> Option<&str> {
+        match self {
+            RankingRule::Asc(field) | RankingRule::Desc(field) => Some(field),
+            _ => None,
+        }
+    }
+
+    pub fn try_from_iter(rules: impl IntoIterator<Item = impl AsRef<str>>) -> Result<Vec<RankingRule>, RankingRuleConversionError> {
+        rules.into_iter()
+            .map(|s| RankingRule::from_str(s.as_ref()))
+            .collect()
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SettingsUpdate {
+    pub ranking_rules: UpdateState<Vec<RankingRule>>,
+    pub distinct_attribute: UpdateState<String>,
+    pub primary_key: UpdateState<String>,
+    pub searchable_attributes: UpdateState<Vec<String>>,
+    pub displayed_attributes: UpdateState<BTreeSet<String>>,
+    pub stop_words: UpdateState<BTreeSet<String>>,
+    pub synonyms: UpdateState<BTreeMap<String, Vec<String>>>,
+    pub attributes_for_faceting: UpdateState<Vec<String>>,
+}
+
+impl Default for SettingsUpdate {
+    fn default() -> Self {
+        Self {
+            ranking_rules: UpdateState::Nothing,
+            distinct_attribute: UpdateState::Nothing,
+            primary_key: UpdateState::Nothing,
+            searchable_attributes: UpdateState::Nothing,
+            displayed_attributes: UpdateState::Nothing,
+            stop_words: UpdateState::Nothing,
+            synonyms: UpdateState::Nothing,
+            attributes_for_faceting: UpdateState::Nothing,
+        }
+    }
+}

meilisearch-core/src/store/cow_set.rs

@@ -0,0 +1,32 @@
+use std::borrow::Cow;
+
+use heed::{types::CowSlice, BytesEncode, BytesDecode};
+use sdset::{Set, SetBuf};
+use zerocopy::{AsBytes, FromBytes};
+
+pub struct CowSet<T>(std::marker::PhantomData<T>);
+
+impl<'a, T: 'a> BytesEncode<'a> for CowSet<T>
+where
+    T: AsBytes,
+{
+    type EItem = Set<T>;
+
+    fn bytes_encode(item: &'a Self::EItem) -> Option<Cow<[u8]>> {
+        CowSlice::bytes_encode(item.as_slice())
+    }
+}
+
+impl<'a, T: 'a> BytesDecode<'a> for CowSet<T>
+where
+    T: FromBytes + Copy,
+{
+    type DItem = Cow<'a, Set<T>>;
+
+    fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
+        match CowSlice::<T>::bytes_decode(bytes)? {
+            Cow::Owned(vec) => Some(Cow::Owned(SetBuf::new_unchecked(vec))),
+            Cow::Borrowed(slice) => Some(Cow::Borrowed(Set::new_unchecked(slice))),
+        }
+    }
+}

meilisearch-core/src/store/docs_words.rs

@@ -1,13 +1,15 @@
-use super::BEU64;
-use crate::database::MainT;
-use crate::DocumentId;
-use heed::types::{ByteSlice, OwnedType};
+use std::borrow::Cow;
+
 use heed::Result as ZResult;
-use std::sync::Arc;
+use heed::types::{ByteSlice, OwnedType};
+
+use crate::database::MainT;
+use crate::{DocumentId, FstSetCow};
+use super::BEU32;
 
 #[derive(Copy, Clone)]
 pub struct DocsWords {
-    pub(crate) docs_words: heed::Database<OwnedType<BEU64>, ByteSlice>,
+    pub(crate) docs_words: heed::Database<OwnedType<BEU32>, ByteSlice>,
 }
 
 impl DocsWords {
@@ -15,15 +17,15 @@ impl DocsWords {
         self,
         writer: &mut heed::RwTxn<MainT>,
         document_id: DocumentId,
-        words: &fst::Set,
+        words: &FstSetCow,
     ) -> ZResult<()> {
-        let document_id = BEU64::new(document_id.0);
+        let document_id = BEU32::new(document_id.0);
         let bytes = words.as_fst().as_bytes();
         self.docs_words.put(writer, &document_id, bytes)
     }
 
     pub fn del_doc_words(self, writer: &mut heed::RwTxn<MainT>, document_id: DocumentId) -> ZResult<bool> {
-        let document_id = BEU64::new(document_id.0);
+        let document_id = BEU32::new(document_id.0);
         self.docs_words.delete(writer, &document_id)
     }
 
@@ -31,20 +33,11 @@ impl DocsWords {
         self.docs_words.clear(writer)
     }
 
-    pub fn doc_words(
-        self,
-        reader: &heed::RoTxn<MainT>,
-        document_id: DocumentId,
-    ) -> ZResult<Option<fst::Set>> {
-        let document_id = BEU64::new(document_id.0);
+    pub fn doc_words<'a>(self, reader: &'a heed::RoTxn<'a, MainT>, document_id: DocumentId) -> ZResult<FstSetCow> {
+        let document_id = BEU32::new(document_id.0);
         match self.docs_words.get(reader, &document_id)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
+            Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
+            None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()),
         }
     }
 }

meilisearch-core/src/store/documents_fields.rs

@@ -1,14 +1,14 @@
 use heed::types::{ByteSlice, OwnedType};
 use crate::database::MainT;
 use heed::Result as ZResult;
-use meilisearch_schema::SchemaAttr;
+use meilisearch_schema::FieldId;
 
-use super::DocumentAttrKey;
+use super::DocumentFieldStoredKey;
 use crate::DocumentId;
 
 #[derive(Copy, Clone)]
 pub struct DocumentsFields {
-    pub(crate) documents_fields: heed::Database<OwnedType<DocumentAttrKey>, ByteSlice>,
+    pub(crate) documents_fields: heed::Database<OwnedType<DocumentFieldStoredKey>, ByteSlice>,
 }
 
 impl DocumentsFields {
@@ -16,10 +16,10 @@ impl DocumentsFields {
         self,
         writer: &mut heed::RwTxn<MainT>,
         document_id: DocumentId,
-        attribute: SchemaAttr,
+        field: FieldId,
         value: &[u8],
     ) -> ZResult<()> {
-        let key = DocumentAttrKey::new(document_id, attribute);
+        let key = DocumentFieldStoredKey::new(document_id, field);
         self.documents_fields.put(writer, &key, value)
     }
 
@@ -28,8 +28,8 @@ impl DocumentsFields {
         writer: &mut heed::RwTxn<MainT>,
         document_id: DocumentId,
     ) -> ZResult<usize> {
-        let start = DocumentAttrKey::new(document_id, SchemaAttr::min());
-        let end = DocumentAttrKey::new(document_id, SchemaAttr::max());
+        let start = DocumentFieldStoredKey::new(document_id, FieldId::min());
+        let end = DocumentFieldStoredKey::new(document_id, FieldId::max());
         self.documents_fields.delete_range(writer, &(start..=end))
     }
 
@@ -41,9 +41,9 @@ impl DocumentsFields {
         self,
         reader: &'txn heed::RoTxn<MainT>,
         document_id: DocumentId,
-        attribute: SchemaAttr,
+        field: FieldId,
     ) -> ZResult<Option<&'txn [u8]>> {
-        let key = DocumentAttrKey::new(document_id, attribute);
+        let key = DocumentFieldStoredKey::new(document_id, field);
         self.documents_fields.get(reader, &key)
     }
 
@@ -52,25 +52,25 @@ impl DocumentsFields {
         reader: &'txn heed::RoTxn<MainT>,
         document_id: DocumentId,
     ) -> ZResult<DocumentFieldsIter<'txn>> {
-        let start = DocumentAttrKey::new(document_id, SchemaAttr::min());
-        let end = DocumentAttrKey::new(document_id, SchemaAttr::max());
+        let start = DocumentFieldStoredKey::new(document_id, FieldId::min());
+        let end = DocumentFieldStoredKey::new(document_id, FieldId::max());
         let iter = self.documents_fields.range(reader, &(start..=end))?;
         Ok(DocumentFieldsIter { iter })
     }
 }
 
 pub struct DocumentFieldsIter<'txn> {
-    iter: heed::RoRange<'txn, OwnedType<DocumentAttrKey>, ByteSlice>,
+    iter: heed::RoRange<'txn, OwnedType<DocumentFieldStoredKey>, ByteSlice>,
 }
 
 impl<'txn> Iterator for DocumentFieldsIter<'txn> {
-    type Item = ZResult<(SchemaAttr, &'txn [u8])>;
+    type Item = ZResult<(FieldId, &'txn [u8])>;
 
     fn next(&mut self) -> Option<Self::Item> {
         match self.iter.next() {
             Some(Ok((key, bytes))) => {
-                let attr = SchemaAttr(key.attr.get());
-                Some(Ok((attr, bytes)))
+                let field_id = FieldId(key.field_id.get());
+                Some(Ok((field_id, bytes)))
             }
             Some(Err(e)) => Some(Err(e)),
             None => None,

meilisearch-core/src/store/documents_fields_counts.rs

@@ -1,13 +1,14 @@
-use super::DocumentAttrKey;
+use super::DocumentFieldIndexedKey;
 use crate::database::MainT;
 use crate::DocumentId;
 use heed::types::OwnedType;
 use heed::Result as ZResult;
-use meilisearch_schema::SchemaAttr;
+use meilisearch_schema::IndexedPos;
+use crate::MResult;
 
 #[derive(Copy, Clone)]
 pub struct DocumentsFieldsCounts {
-    pub(crate) documents_fields_counts: heed::Database<OwnedType<DocumentAttrKey>, OwnedType<u16>>,
+    pub(crate) documents_fields_counts: heed::Database<OwnedType<DocumentFieldIndexedKey>, OwnedType<u16>>,
 }
 
 impl DocumentsFieldsCounts {
@@ -15,10 +16,10 @@ impl DocumentsFieldsCounts {
         self,
         writer: &mut heed::RwTxn<MainT>,
         document_id: DocumentId,
-        attribute: SchemaAttr,
+        attribute: IndexedPos,
         value: u16,
     ) -> ZResult<()> {
-        let key = DocumentAttrKey::new(document_id, attribute);
+        let key = DocumentFieldIndexedKey::new(document_id, attribute);
         self.documents_fields_counts.put(writer, &key, &value)
     }
 
@@ -27,10 +28,9 @@ impl DocumentsFieldsCounts {
         writer: &mut heed::RwTxn<MainT>,
         document_id: DocumentId,
     ) -> ZResult<usize> {
-        let start = DocumentAttrKey::new(document_id, SchemaAttr::min());
-        let end = DocumentAttrKey::new(document_id, SchemaAttr::max());
-        self.documents_fields_counts
-            .delete_range(writer, &(start..=end))
+        let start = DocumentFieldIndexedKey::new(document_id, IndexedPos::min());
+        let end = DocumentFieldIndexedKey::new(document_id, IndexedPos::max());
+        self.documents_fields_counts.delete_range(writer, &(start..=end))
     }
 
     pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
@@ -41,9 +41,9 @@ impl DocumentsFieldsCounts {
         self,
         reader: &heed::RoTxn<MainT>,
         document_id: DocumentId,
-        attribute: SchemaAttr,
+        attribute: IndexedPos,
     ) -> ZResult<Option<u16>> {
-        let key = DocumentAttrKey::new(document_id, attribute);
+        let key = DocumentFieldIndexedKey::new(document_id, attribute);
         match self.documents_fields_counts.get(reader, &key)? {
             Some(count) => Ok(Some(count)),
             None => Ok(None),
@@ -55,13 +55,13 @@ impl DocumentsFieldsCounts {
         reader: &'txn heed::RoTxn<MainT>,
         document_id: DocumentId,
     ) -> ZResult<DocumentFieldsCountsIter<'txn>> {
-        let start = DocumentAttrKey::new(document_id, SchemaAttr::min());
-        let end = DocumentAttrKey::new(document_id, SchemaAttr::max());
+        let start = DocumentFieldIndexedKey::new(document_id, IndexedPos::min());
+        let end = DocumentFieldIndexedKey::new(document_id, IndexedPos::max());
         let iter = self.documents_fields_counts.range(reader, &(start..=end))?;
         Ok(DocumentFieldsCountsIter { iter })
     }
 
-    pub fn documents_ids<'txn>(self, reader: &'txn heed::RoTxn<MainT>) -> ZResult<DocumentsIdsIter<'txn>> {
+    pub fn documents_ids<'txn>(self, reader: &'txn heed::RoTxn<MainT>) -> MResult<DocumentsIdsIter<'txn>> {
         let iter = self.documents_fields_counts.iter(reader)?;
         Ok(DocumentsIdsIter {
             last_seen_id: None,
@@ -79,17 +79,17 @@ impl DocumentsFieldsCounts {
 }
 
 pub struct DocumentFieldsCountsIter<'txn> {
-    iter: heed::RoRange<'txn, OwnedType<DocumentAttrKey>, OwnedType<u16>>,
+    iter: heed::RoRange<'txn, OwnedType<DocumentFieldIndexedKey>, OwnedType<u16>>,
 }
 
 impl Iterator for DocumentFieldsCountsIter<'_> {
-    type Item = ZResult<(SchemaAttr, u16)>;
+    type Item = ZResult<(IndexedPos, u16)>;
 
     fn next(&mut self) -> Option<Self::Item> {
         match self.iter.next() {
             Some(Ok((key, count))) => {
-                let attr = SchemaAttr(key.attr.get());
-                Some(Ok((attr, count)))
+                let indexed_pos = IndexedPos(key.indexed_pos.get());
+                Some(Ok((indexed_pos, count)))
             }
             Some(Err(e)) => Some(Err(e)),
             None => None,
@@ -99,11 +99,11 @@ impl Iterator for DocumentFieldsCountsIter<'_> {
 
 pub struct DocumentsIdsIter<'txn> {
     last_seen_id: Option<DocumentId>,
-    iter: heed::RoIter<'txn, OwnedType<DocumentAttrKey>, OwnedType<u16>>,
+    iter: heed::RoIter<'txn, OwnedType<DocumentFieldIndexedKey>, OwnedType<u16>>,
 }
 
 impl Iterator for DocumentsIdsIter<'_> {
-    type Item = ZResult<DocumentId>;
+    type Item = MResult<DocumentId>;
 
     fn next(&mut self) -> Option<Self::Item> {
         for result in &mut self.iter {
@@ -115,7 +115,7 @@ impl Iterator for DocumentsIdsIter<'_> {
                         return Some(Ok(document_id));
                     }
                 }
-                Err(e) => return Some(Err(e)),
+                Err(e) => return Some(Err(e.into())),
             }
         }
         None
@@ -123,18 +123,18 @@ impl Iterator for DocumentsIdsIter<'_> {
 }
 
 pub struct AllDocumentsFieldsCountsIter<'txn> {
-    iter: heed::RoIter<'txn, OwnedType<DocumentAttrKey>, OwnedType<u16>>,
+    iter: heed::RoIter<'txn, OwnedType<DocumentFieldIndexedKey>, OwnedType<u16>>,
 }
 
 impl Iterator for AllDocumentsFieldsCountsIter<'_> {
-    type Item = ZResult<(DocumentId, SchemaAttr, u16)>;
+    type Item = ZResult<(DocumentId, IndexedPos, u16)>;
 
     fn next(&mut self) -> Option<Self::Item> {
         match self.iter.next() {
             Some(Ok((key, count))) => {
                 let docid = DocumentId(key.docid.get());
-                let attr = SchemaAttr(key.attr.get());
-                Some(Ok((docid, attr, count)))
+                let indexed_pos = IndexedPos(key.indexed_pos.get());
+                Some(Ok((docid, indexed_pos, count)))
             }
             Some(Err(e)) => Some(Err(e)),
             None => None,

meilisearch-core/src/store/documents_ids.rs

@@ -0,0 +1,75 @@
+use std::borrow::Cow;
+
+use heed::{BytesDecode, BytesEncode};
+use sdset::Set;
+
+use crate::DocumentId;
+use super::cow_set::CowSet;
+
+pub struct DocumentsIds;
+
+impl BytesEncode<'_> for DocumentsIds {
+    type EItem = Set<DocumentId>;
+
+    fn bytes_encode(item: &Self::EItem) -> Option<Cow<[u8]>> {
+        CowSet::bytes_encode(item)
+    }
+}
+
+impl<'a> BytesDecode<'a> for DocumentsIds {
+    type DItem = Cow<'a, Set<DocumentId>>;
+
+    fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
+        CowSet::bytes_decode(bytes)
+    }
+}
+
+pub struct DiscoverIds<'a> {
+    ids_iter: std::slice::Iter<'a, DocumentId>,
+    left_id: Option<u32>,
+    right_id: Option<u32>,
+    available_range: std::ops::Range<u32>,
+}
+
+impl DiscoverIds<'_> {
+    pub fn new(ids: &Set<DocumentId>) -> DiscoverIds {
+        let mut ids_iter = ids.iter();
+        let right_id = ids_iter.next().map(|id| id.0);
+        let available_range = 0..right_id.unwrap_or(u32::max_value());
+        DiscoverIds { ids_iter, left_id: None, right_id, available_range }
+    }
+}
+
+impl Iterator for DiscoverIds<'_> {
+    type Item = DocumentId;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            match self.available_range.next() {
+                // The available range gives us a new id, we return it.
+                Some(id) => return Some(DocumentId(id)),
+                // The available range is exhausted, we need to find the next one.
+                None if self.available_range.end == u32::max_value() => return None,
+                None => loop {
+                    self.left_id = self.right_id.take();
+                    self.right_id = self.ids_iter.next().map(|id| id.0);
+                    match (self.left_id, self.right_id) {
+                        // We found a gap in the used ids, we can yield all ids
+                        // until the end of the gap
+                        (Some(l), Some(r)) => if l.saturating_add(1) != r {
+                            self.available_range = (l + 1)..r;
+                            break;
+                        },
+                        // The last used id has been reached, we can use all ids
+                        // until u32 MAX
+                        (Some(l), None) => {
+                            self.available_range = l.saturating_add(1)..u32::max_value();
+                            break;
+                        },
+                        _ => (),
+                    }
+                },
+            }
+        }
+    }
+}

meilisearch-core/src/store/facets.rs

@@ -0,0 +1,97 @@
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::mem;
+
+use heed::{RwTxn, RoTxn, RoPrefix, types::Str, BytesEncode, BytesDecode};
+use sdset::{SetBuf, Set, SetOperation};
+
+use meilisearch_types::DocumentId;
+use meilisearch_schema::FieldId;
+
+use crate::MResult;
+use crate::database::MainT;
+use crate::facets::FacetKey;
+use super::cow_set::CowSet;
+
+/// contains facet info
+#[derive(Clone, Copy)]
+pub struct Facets {
+    pub(crate) facets: heed::Database<FacetKey, FacetData>,
+}
+
+pub struct FacetData;
+
+impl<'a> BytesEncode<'a> for FacetData {
+    type EItem = (&'a str, &'a Set<DocumentId>);
+
+    fn bytes_encode(item: &'a Self::EItem) -> Option<Cow<'a, [u8]>> {
+        // get size of the first item
+        let first_size =  item.0.as_bytes().len();
+        let size = mem::size_of::<u64>()
+            + first_size
+            + item.1.len() * mem::size_of::<DocumentId>();
+        let mut buffer = Vec::with_capacity(size);
+        // encode the length of the first item
+        buffer.extend_from_slice(&first_size.to_be_bytes());
+        buffer.extend_from_slice(Str::bytes_encode(&item.0)?.as_ref());
+        let second_slice = CowSet::bytes_encode(&item.1)?;
+        buffer.extend_from_slice(second_slice.as_ref());
+        Some(Cow::Owned(buffer))
+    }
+}
+
+impl<'a> BytesDecode<'a> for FacetData {
+    type DItem = (&'a str, Cow<'a, Set<DocumentId>>);
+
+    fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
+        const LEN: usize = mem::size_of::<u64>();
+        let mut size_buf = [0; LEN];
+        size_buf.copy_from_slice(bytes.get(0..LEN)?);
+        // decode size of the first item from the bytes
+        let first_size = usize::from_be_bytes(size_buf);
+        // decode first and second items
+        let first_item = Str::bytes_decode(bytes.get(LEN..(LEN + first_size))?)?;
+        let second_item = CowSet::bytes_decode(bytes.get((LEN + first_size)..)?)?;
+        Some((first_item, second_item))
+    }
+}
+
+impl Facets {
+    // we use sdset::SetBuf to ensure the docids are sorted.
+    pub fn put_facet_document_ids(&self, writer: &mut RwTxn<MainT>, facet_key: FacetKey, doc_ids: &Set<DocumentId>, facet_value: &str) -> MResult<()> {
+        Ok(self.facets.put(writer, &facet_key, &(facet_value, doc_ids))?)
+    }
+
+    pub fn field_document_ids<'txn>(&self, reader: &'txn RoTxn<MainT>, field_id: FieldId) -> MResult<RoPrefix<'txn, FacetKey, FacetData>> {
+        Ok(self.facets.prefix_iter(reader, &FacetKey::new(field_id, String::new()))?)
+    }
+
+    pub fn facet_document_ids<'txn>(&self, reader: &'txn RoTxn<MainT>, facet_key: &FacetKey) -> MResult<Option<(&'txn str,Cow<'txn, Set<DocumentId>>)>> {
+        Ok(self.facets.get(reader, &facet_key)?)
+    }
+
+    /// updates the facets  store, revmoving the documents from the facets provided in the
+    /// `facet_map` argument
+    pub fn remove(&self, writer: &mut RwTxn<MainT>, facet_map: HashMap<FacetKey, (String, Vec<DocumentId>)>) -> MResult<()> {
+        for (key, (name, document_ids)) in facet_map {
+            if let Some((_, old)) = self.facets.get(writer, &key)? {
+                let to_remove = SetBuf::from_dirty(document_ids);
+                let new = sdset::duo::OpBuilder::new(old.as_ref(), to_remove.as_set()).difference().into_set_buf();
+                self.facets.put(writer, &key, &(&name, new.as_set()))?;
+            }
+        }
+        Ok(())
+    }
+
+    pub fn add(&self, writer: &mut RwTxn<MainT>, facet_map: HashMap<FacetKey, (String, Vec<DocumentId>)>) -> MResult<()> {
+        for (key, (facet_name, document_ids)) in facet_map {
+            let set = SetBuf::from_dirty(document_ids);
+            self.put_facet_document_ids(writer, key, set.as_set(), &facet_name)?;
+        }
+        Ok(())
+    }
+
+    pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> MResult<()> {
+        Ok(self.facets.clear(writer)?)
+    }
+}

meilisearch-core/src/store/main.rs

@@ -1,25 +1,37 @@
-use crate::database::MainT;
-use crate::RankedMap;
-use chrono::{DateTime, Utc};
-use heed::types::{ByteSlice, OwnedType, SerdeBincode, Str};
-use heed::Result as ZResult;
-use meilisearch_schema::Schema;
-use std::collections::HashMap;
-use std::sync::Arc;
+use std::borrow::Cow;
+use std::collections::BTreeMap;
 
+use chrono::{DateTime, Utc};
+use heed::types::{ByteSlice, OwnedType, SerdeBincode, Str, CowSlice};
+use meilisearch_schema::{FieldId, Schema};
+use meilisearch_types::DocumentId;
+use sdset::Set;
+
+use crate::database::MainT;
+use crate::{RankedMap, MResult};
+use crate::settings::RankingRule;
+use crate::{FstSetCow, FstMapCow};
+use super::{CowSet, DocumentsIds};
+
+const ATTRIBUTES_FOR_FACETING_KEY: &str = "attributes-for-faceting";
 const CREATED_AT_KEY: &str = "created-at";
-const CUSTOMS_KEY: &str = "customs-key";
-const FIELDS_FREQUENCY_KEY: &str = "fields-frequency";
+const CUSTOMS_KEY: &str = "customs";
+const DISTINCT_ATTRIBUTE_KEY: &str = "distinct-attribute";
+const EXTERNAL_DOCIDS_KEY: &str = "external-docids";
+const FIELDS_DISTRIBUTION_KEY: &str = "fields-distribution";
+const INTERNAL_DOCIDS_KEY: &str = "internal-docids";
 const NAME_KEY: &str = "name";
 const NUMBER_OF_DOCUMENTS_KEY: &str = "number-of-documents";
 const RANKED_MAP_KEY: &str = "ranked-map";
+const RANKING_RULES_KEY: &str = "ranking-rules";
 const SCHEMA_KEY: &str = "schema";
+const SORTED_DOCUMENT_IDS_CACHE_KEY: &str = "sorted-document-ids-cache";
 const STOP_WORDS_KEY: &str = "stop-words";
 const SYNONYMS_KEY: &str = "synonyms";
 const UPDATED_AT_KEY: &str = "updated-at";
 const WORDS_KEY: &str = "words";
 
-pub type FreqsMap = HashMap<String, usize>;
+pub type FreqsMap = BTreeMap<String, usize>;
 type SerdeFreqsMap = SerdeBincode<FreqsMap>;
 type SerdeDatetime = SerdeBincode<DateTime<Utc>>;
 
@@ -29,123 +41,200 @@ pub struct Main {
 }
 
 impl Main {
-    pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
-        self.main.clear(writer)
+    pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> MResult<()> {
+        Ok(self.main.clear(writer)?)
     }
 
-    pub fn put_name(self, writer: &mut heed::RwTxn<MainT>, name: &str) -> ZResult<()> {
-        self.main.put::<_, Str, Str>(writer, NAME_KEY, name)
+    pub fn put_name(self, writer: &mut heed::RwTxn<MainT>, name: &str) -> MResult<()> {
+        Ok(self.main.put::<_, Str, Str>(writer, NAME_KEY, name)?)
     }
 
-    pub fn name(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<String>> {
+    pub fn name(self, reader: &heed::RoTxn<MainT>) -> MResult<Option<String>> {
         Ok(self
             .main
             .get::<_, Str, Str>(reader, NAME_KEY)?
             .map(|name| name.to_owned()))
     }
 
-    pub fn put_created_at(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
-        self.main
-            .put::<_, Str, SerdeDatetime>(writer, CREATED_AT_KEY, &Utc::now())
+    pub fn put_created_at(self, writer: &mut heed::RwTxn<MainT>) -> MResult<()> {
+        Ok(self.main.put::<_, Str, SerdeDatetime>(writer, CREATED_AT_KEY, &Utc::now())?)
     }
 
-    pub fn created_at(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<DateTime<Utc>>> {
-        self.main.get::<_, Str, SerdeDatetime>(reader, CREATED_AT_KEY)
+    pub fn created_at(self, reader: &heed::RoTxn<MainT>) -> MResult<Option<DateTime<Utc>>> {
+        Ok(self.main.get::<_, Str, SerdeDatetime>(reader, CREATED_AT_KEY)?)
     }
 
-    pub fn put_updated_at(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
-        self.main
-            .put::<_, Str, SerdeDatetime>(writer, UPDATED_AT_KEY, &Utc::now())
+    pub fn put_updated_at(self, writer: &mut heed::RwTxn<MainT>) -> MResult<()> {
+        Ok(self.main.put::<_, Str, SerdeDatetime>(writer, UPDATED_AT_KEY, &Utc::now())?)
     }
 
-    pub fn updated_at(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<DateTime<Utc>>> {
-        self.main.get::<_, Str, SerdeDatetime>(reader, UPDATED_AT_KEY)
+    pub fn updated_at(self, reader: &heed::RoTxn<MainT>) -> MResult<Option<DateTime<Utc>>> {
+        Ok(self.main.get::<_, Str, SerdeDatetime>(reader, UPDATED_AT_KEY)?)
     }
 
-    pub fn put_words_fst(self, writer: &mut heed::RwTxn<MainT>, fst: &fst::Set) -> ZResult<()> {
-        let bytes = fst.as_fst().as_bytes();
-        self.main.put::<_, Str, ByteSlice>(writer, WORDS_KEY, bytes)
+    pub fn put_internal_docids(self, writer: &mut heed::RwTxn<MainT>, ids: &sdset::Set<DocumentId>) -> MResult<()> {
+        Ok(self.main.put::<_, Str, DocumentsIds>(writer, INTERNAL_DOCIDS_KEY, ids)?)
     }
 
-    pub unsafe fn static_words_fst(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<fst::Set>> {
-        match self.main.get::<_, Str, ByteSlice>(reader, WORDS_KEY)? {
-            Some(bytes) => {
-                let bytes: &'static [u8] = std::mem::transmute(bytes);
-                let set = fst::Set::from_static_slice(bytes).unwrap();
-                Ok(Some(set))
-            }
-            None => Ok(None),
+    pub fn internal_docids<'txn>(self, reader: &'txn heed::RoTxn<MainT>) -> MResult<Cow<'txn, sdset::Set<DocumentId>>> {
+        match self.main.get::<_, Str, DocumentsIds>(reader, INTERNAL_DOCIDS_KEY)? {
+            Some(ids) => Ok(ids),
+            None => Ok(Cow::default()),
         }
     }
 
-    pub fn words_fst(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<fst::Set>> {
-        match self.main.get::<_, Str, ByteSlice>(reader, WORDS_KEY)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
+    pub fn merge_internal_docids(self, writer: &mut heed::RwTxn<MainT>, new_ids: &sdset::Set<DocumentId>) -> MResult<()> {
+        use sdset::SetOperation;
+
+        // We do an union of the old and new internal ids.
+        let internal_docids = self.internal_docids(writer)?;
+        let internal_docids = sdset::duo::Union::new(&internal_docids, new_ids).into_set_buf();
+        Ok(self.put_internal_docids(writer, &internal_docids)?)
+    }
+
+    pub fn remove_internal_docids(self, writer: &mut heed::RwTxn<MainT>, ids: &sdset::Set<DocumentId>) -> MResult<()> {
+        use sdset::SetOperation;
+
+        // We do a difference of the old and new internal ids.
+        let internal_docids = self.internal_docids(writer)?;
+        let internal_docids = sdset::duo::Difference::new(&internal_docids, ids).into_set_buf();
+        Ok(self.put_internal_docids(writer, &internal_docids)?)
+    }
+
+    pub fn put_external_docids<A>(self, writer: &mut heed::RwTxn<MainT>, ids: &fst::Map<A>) -> MResult<()>
+    where A: AsRef<[u8]>,
+    {
+        Ok(self.main.put::<_, Str, ByteSlice>(writer, EXTERNAL_DOCIDS_KEY, ids.as_fst().as_bytes())?)
+    }
+
+    pub fn merge_external_docids<A>(self, writer: &mut heed::RwTxn<MainT>, new_docids: &fst::Map<A>) -> MResult<()>
+    where A: AsRef<[u8]>,
+    {
+        use fst::{Streamer, IntoStreamer};
+
+        // Do an union of the old and the new set of external docids.
+        let external_docids = self.external_docids(writer)?;
+        let mut op = external_docids.op().add(new_docids.into_stream()).r#union();
+        let mut build = fst::MapBuilder::memory();
+        while let Some((docid, values)) = op.next() {
+            build.insert(docid, values[0].value).unwrap();
+        }
+        drop(op);
+
+        let external_docids = build.into_map();
+        Ok(self.put_external_docids(writer, &external_docids)?)
+    }
+
+    pub fn remove_external_docids<A>(self, writer: &mut heed::RwTxn<MainT>, ids: &fst::Map<A>) -> MResult<()>
+    where A: AsRef<[u8]>,
+    {
+        use fst::{Streamer, IntoStreamer};
+
+        // Do an union of the old and the new set of external docids.
+        let external_docids = self.external_docids(writer)?;
+        let mut op = external_docids.op().add(ids.into_stream()).difference();
+        let mut build = fst::MapBuilder::memory();
+        while let Some((docid, values)) = op.next() {
+            build.insert(docid, values[0].value).unwrap();
+        }
+        drop(op);
+
+        let external_docids = build.into_map();
+        self.put_external_docids(writer, &external_docids)
+    }
+
+    pub fn external_docids<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult<FstMapCow> {
+        match self.main.get::<_, Str, ByteSlice>(reader, EXTERNAL_DOCIDS_KEY)? {
+            Some(bytes) => Ok(fst::Map::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
+            None => Ok(fst::Map::default().map_data(Cow::Owned).unwrap()),
         }
     }
 
-    pub fn put_schema(self, writer: &mut heed::RwTxn<MainT>, schema: &Schema) -> ZResult<()> {
-        self.main
-            .put::<_, Str, SerdeBincode<Schema>>(writer, SCHEMA_KEY, schema)
+    pub fn external_to_internal_docid(self, reader: &heed::RoTxn<MainT>, external_docid: &str) -> MResult<Option<DocumentId>> {
+        let external_ids = self.external_docids(reader)?;
+        Ok(external_ids.get(external_docid).map(|id| DocumentId(id as u32)))
     }
 
-    pub fn schema(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<Schema>> {
-        self.main
-            .get::<_, Str, SerdeBincode<Schema>>(reader, SCHEMA_KEY)
+    pub fn words_fst<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult<FstSetCow> {
+        match self.main.get::<_, Str, ByteSlice>(reader, WORDS_KEY)? {
+            Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
+            None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()),
+        }
     }
 
-    pub fn put_ranked_map(self, writer: &mut heed::RwTxn<MainT>, ranked_map: &RankedMap) -> ZResult<()> {
-        self.main
-            .put::<_, Str, SerdeBincode<RankedMap>>(writer, RANKED_MAP_KEY, &ranked_map)
+    pub fn put_words_fst<A: AsRef<[u8]>>(self, writer: &mut heed::RwTxn<MainT>, fst: &fst::Set<A>) -> MResult<()> {
+        Ok(self.main.put::<_, Str, ByteSlice>(writer, WORDS_KEY, fst.as_fst().as_bytes())?)
     }
 
-    pub fn ranked_map(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<RankedMap>> {
-        self.main
-            .get::<_, Str, SerdeBincode<RankedMap>>(reader, RANKED_MAP_KEY)
+    pub fn put_sorted_document_ids_cache(self, writer: &mut heed::RwTxn<MainT>, documents_ids: &[DocumentId]) -> MResult<()> {
+        Ok(self.main.put::<_, Str, CowSlice<DocumentId>>(writer, SORTED_DOCUMENT_IDS_CACHE_KEY, documents_ids)?)
     }
 
-    pub fn put_synonyms_fst(self, writer: &mut heed::RwTxn<MainT>, fst: &fst::Set) -> ZResult<()> {
+    pub fn sorted_document_ids_cache<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult<Option<Cow<[DocumentId]>>> {
+        Ok(self.main.get::<_, Str, CowSlice<DocumentId>>(reader, SORTED_DOCUMENT_IDS_CACHE_KEY)?)
+    }
+
+    pub fn put_schema(self, writer: &mut heed::RwTxn<MainT>, schema: &Schema) -> MResult<()> {
+        Ok(self.main.put::<_, Str, SerdeBincode<Schema>>(writer, SCHEMA_KEY, schema)?)
+    }
+
+    pub fn schema(self, reader: &heed::RoTxn<MainT>) -> MResult<Option<Schema>> {
+        Ok(self.main.get::<_, Str, SerdeBincode<Schema>>(reader, SCHEMA_KEY)?)
+    }
+
+    pub fn delete_schema(self, writer: &mut heed::RwTxn<MainT>) -> MResult<bool> {
+        Ok(self.main.delete::<_, Str>(writer, SCHEMA_KEY)?)
+    }
+
+    pub fn put_ranked_map(self, writer: &mut heed::RwTxn<MainT>, ranked_map: &RankedMap) -> MResult<()> {
+        Ok(self.main.put::<_, Str, SerdeBincode<RankedMap>>(writer, RANKED_MAP_KEY, &ranked_map)?)
+    }
+
+    pub fn ranked_map(self, reader: &heed::RoTxn<MainT>) -> MResult<Option<RankedMap>> {
+        Ok(self.main.get::<_, Str, SerdeBincode<RankedMap>>(reader, RANKED_MAP_KEY)?)
+    }
+
+    pub fn put_synonyms_fst<A: AsRef<[u8]>>(self, writer: &mut heed::RwTxn<MainT>, fst: &fst::Set<A>) -> MResult<()> {
         let bytes = fst.as_fst().as_bytes();
-        self.main.put::<_, Str, ByteSlice>(writer, SYNONYMS_KEY, bytes)
+        Ok(self.main.put::<_, Str, ByteSlice>(writer, SYNONYMS_KEY, bytes)?)
     }
 
-    pub fn synonyms_fst(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<fst::Set>> {
+    pub(crate) fn synonyms_fst<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult<FstSetCow> {
         match self.main.get::<_, Str, ByteSlice>(reader, SYNONYMS_KEY)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
+            Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
+            None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()),
         }
     }
 
-    pub fn put_stop_words_fst(self, writer: &mut heed::RwTxn<MainT>, fst: &fst::Set) -> ZResult<()> {
+    pub fn synonyms(self, reader: &heed::RoTxn<MainT>) -> MResult<Vec<String>> {
+        let synonyms = self
+            .synonyms_fst(&reader)?
+            .stream()
+            .into_strs()?;
+        Ok(synonyms)
+    }
+
+    pub fn put_stop_words_fst<A: AsRef<[u8]>>(self, writer: &mut heed::RwTxn<MainT>, fst: &fst::Set<A>) -> MResult<()> {
         let bytes = fst.as_fst().as_bytes();
-        self.main
-            .put::<_, Str, ByteSlice>(writer, STOP_WORDS_KEY, bytes)
+        Ok(self.main.put::<_, Str, ByteSlice>(writer, STOP_WORDS_KEY, bytes)?)
     }
 
-    pub fn stop_words_fst(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<fst::Set>> {
+    pub(crate) fn stop_words_fst<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult<FstSetCow> {
         match self.main.get::<_, Str, ByteSlice>(reader, STOP_WORDS_KEY)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
+            Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
+            None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()),
         }
     }
 
-    pub fn put_number_of_documents<F>(self, writer: &mut heed::RwTxn<MainT>, f: F) -> ZResult<u64>
+    pub fn stop_words(self, reader: &heed::RoTxn<MainT>) -> MResult<Vec<String>> {
+        let stop_word_list = self
+            .stop_words_fst(reader)?
+            .stream()
+            .into_strs()?;
+        Ok(stop_word_list)
+    }
+
+    pub fn put_number_of_documents<F>(self, writer: &mut heed::RwTxn<MainT>, f: F) -> MResult<u64>
     where
         F: Fn(u64) -> u64,
     {
@@ -155,41 +244,77 @@ impl Main {
         Ok(new)
     }
 
-    pub fn number_of_documents(self, reader: &heed::RoTxn<MainT>) -> ZResult<u64> {
+    pub fn number_of_documents(self, reader: &heed::RoTxn<MainT>) -> MResult<u64> {
         match self
             .main
-            .get::<_, Str, OwnedType<u64>>(reader, NUMBER_OF_DOCUMENTS_KEY)?
-        {
+            .get::<_, Str, OwnedType<u64>>(reader, NUMBER_OF_DOCUMENTS_KEY)? {
             Some(value) => Ok(value),
             None => Ok(0),
         }
     }
 
-    pub fn put_fields_frequency(
+    pub fn put_fields_distribution(
         self,
         writer: &mut heed::RwTxn<MainT>,
         fields_frequency: &FreqsMap,
-    ) -> ZResult<()> {
-        self.main
-            .put::<_, Str, SerdeFreqsMap>(writer, FIELDS_FREQUENCY_KEY, fields_frequency)
+    ) -> MResult<()> {
+        Ok(self.main.put::<_, Str, SerdeFreqsMap>(writer, FIELDS_DISTRIBUTION_KEY, fields_frequency)?)
     }
 
-    pub fn fields_frequency(&self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<FreqsMap>> {
+    pub fn fields_distribution(&self, reader: &heed::RoTxn<MainT>) -> MResult<Option<FreqsMap>> {
         match self
             .main
-            .get::<_, Str, SerdeFreqsMap>(reader, FIELDS_FREQUENCY_KEY)?
+            .get::<_, Str, SerdeFreqsMap>(reader, FIELDS_DISTRIBUTION_KEY)?
         {
             Some(freqs) => Ok(Some(freqs)),
             None => Ok(None),
         }
     }
 
-    pub fn put_customs(self, writer: &mut heed::RwTxn<MainT>, customs: &[u8]) -> ZResult<()> {
-        self.main
-            .put::<_, Str, ByteSlice>(writer, CUSTOMS_KEY, customs)
+    pub fn attributes_for_faceting<'txn>(&self, reader: &'txn heed::RoTxn<MainT>) -> MResult<Option<Cow<'txn, Set<FieldId>>>> {
+        Ok(self.main.get::<_, Str, CowSet<FieldId>>(reader, ATTRIBUTES_FOR_FACETING_KEY)?)
     }
 
-    pub fn customs<'txn>(self, reader: &'txn heed::RoTxn<MainT>) -> ZResult<Option<&'txn [u8]>> {
-        self.main.get::<_, Str, ByteSlice>(reader, CUSTOMS_KEY)
+    pub fn put_attributes_for_faceting(self, writer: &mut heed::RwTxn<MainT>, attributes: &Set<FieldId>) -> MResult<()> {
+        Ok(self.main.put::<_, Str, CowSet<FieldId>>(writer, ATTRIBUTES_FOR_FACETING_KEY, attributes)?)
+    }
+
+    pub fn delete_attributes_for_faceting(self, writer: &mut heed::RwTxn<MainT>) -> MResult<bool> {
+        Ok(self.main.delete::<_, Str>(writer, ATTRIBUTES_FOR_FACETING_KEY)?)
+    }
+
+    pub fn ranking_rules(&self, reader: &heed::RoTxn<MainT>) -> MResult<Option<Vec<RankingRule>>> {
+        Ok(self.main.get::<_, Str, SerdeBincode<Vec<RankingRule>>>(reader, RANKING_RULES_KEY)?)
+    }
+
+    pub fn put_ranking_rules(self, writer: &mut heed::RwTxn<MainT>, value: &[RankingRule]) -> MResult<()> {
+        Ok(self.main.put::<_, Str, SerdeBincode<Vec<RankingRule>>>(writer, RANKING_RULES_KEY, &value.to_vec())?)
+    }
+
+    pub fn delete_ranking_rules(self, writer: &mut heed::RwTxn<MainT>) -> MResult<bool> {
+        Ok(self.main.delete::<_, Str>(writer, RANKING_RULES_KEY)?)
+    }
+
+    pub fn distinct_attribute(&self, reader: &heed::RoTxn<MainT>) -> MResult<Option<FieldId>> {
+        match self.main.get::<_, Str, OwnedType<u16>>(reader, DISTINCT_ATTRIBUTE_KEY)? {
+            Some(value) => Ok(Some(FieldId(value.to_owned()))),
+            None => Ok(None),
+        }
+    }
+
+    pub fn put_distinct_attribute(self, writer: &mut heed::RwTxn<MainT>, value: FieldId) -> MResult<()> {
+        Ok(self.main.put::<_, Str, OwnedType<u16>>(writer, DISTINCT_ATTRIBUTE_KEY, &value.0)?)
+    }
+
+    pub fn delete_distinct_attribute(self, writer: &mut heed::RwTxn<MainT>) -> MResult<bool> {
+        Ok(self.main.delete::<_, Str>(writer, DISTINCT_ATTRIBUTE_KEY)?)
+    }
+
+    pub fn put_customs(self, writer: &mut heed::RwTxn<MainT>, customs: &[u8]) -> MResult<()> {
+        Ok(self.main.put::<_, Str, ByteSlice>(writer, CUSTOMS_KEY, customs)?)
+    }
+
+    pub fn customs<'txn>(self, reader: &'txn heed::RoTxn<MainT>) -> MResult<Option<&'txn [u8]>> {
+        Ok(self.main.get::<_, Str, ByteSlice>(reader, CUSTOMS_KEY)?)
     }
 }

meilisearch-core/src/store/mod.rs

@@ -1,23 +1,27 @@
+mod cow_set;
 mod docs_words;
-mod prefix_documents_cache;
-mod prefix_postings_lists_cache;
+mod documents_ids;
 mod documents_fields;
 mod documents_fields_counts;
+mod facets;
 mod main;
 mod postings_lists;
+mod prefix_documents_cache;
+mod prefix_postings_lists_cache;
 mod synonyms;
 mod updates;
 mod updates_results;
 
+pub use self::cow_set::CowSet;
 pub use self::docs_words::DocsWords;
-pub use self::prefix_documents_cache::PrefixDocumentsCache;
-pub use self::prefix_postings_lists_cache::PrefixPostingsListsCache;
 pub use self::documents_fields::{DocumentFieldsIter, DocumentsFields};
-pub use self::documents_fields_counts::{
-    DocumentFieldsCountsIter, DocumentsFieldsCounts, DocumentsIdsIter,
-};
+pub use self::documents_fields_counts::{DocumentFieldsCountsIter, DocumentsFieldsCounts, DocumentsIdsIter};
+pub use self::documents_ids::{DocumentsIds, DiscoverIds};
+pub use self::facets::Facets;
 pub use self::main::Main;
 pub use self::postings_lists::PostingsLists;
+pub use self::prefix_documents_cache::PrefixDocumentsCache;
+pub use self::prefix_postings_lists_cache::PrefixPostingsListsCache;
 pub use self::synonyms::Synonyms;
 pub use self::updates::Updates;
 pub use self::updates_results::UpdatesResults;
@@ -27,34 +31,51 @@ use std::collections::HashSet;
 use std::convert::TryInto;
 use std::{mem, ptr};
 
-use heed::Result as ZResult;
 use heed::{BytesEncode, BytesDecode};
-use meilisearch_schema::{Schema, SchemaAttr};
+use meilisearch_schema::{IndexedPos, FieldId};
 use sdset::{Set, SetBuf};
 use serde::de::{self, Deserialize};
 use zerocopy::{AsBytes, FromBytes};
 
 use crate::criterion::Criteria;
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
 use crate::database::{MainT, UpdateT};
+use crate::database::{UpdateEvent, UpdateEventsEmitter};
 use crate::serde::Deserializer;
+use crate::settings::SettingsUpdate;
 use crate::{query_builder::QueryBuilder, update, DocIndex, DocumentId, Error, MResult};
 
+type BEU32 = zerocopy::U32<byteorder::BigEndian>;
 type BEU64 = zerocopy::U64<byteorder::BigEndian>;
-type BEU16 = zerocopy::U16<byteorder::BigEndian>;
+pub type BEU16 = zerocopy::U16<byteorder::BigEndian>;
 
 #[derive(Debug, Copy, Clone, AsBytes, FromBytes)]
 #[repr(C)]
-pub struct DocumentAttrKey {
-    docid: BEU64,
-    attr: BEU16,
+pub struct DocumentFieldIndexedKey {
+    docid: BEU32,
+    indexed_pos: BEU16,
 }
 
-impl DocumentAttrKey {
-    fn new(docid: DocumentId, attr: SchemaAttr) -> DocumentAttrKey {
-        DocumentAttrKey {
-            docid: BEU64::new(docid.0),
-            attr: BEU16::new(attr.0),
+impl DocumentFieldIndexedKey {
+    fn new(docid: DocumentId, indexed_pos: IndexedPos) -> DocumentFieldIndexedKey {
+        DocumentFieldIndexedKey {
+            docid: BEU32::new(docid.0),
+            indexed_pos: BEU16::new(indexed_pos.0),
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone, AsBytes, FromBytes)]
+#[repr(C)]
+pub struct DocumentFieldStoredKey {
+    docid: BEU32,
+    field_id: BEU16,
+}
+
+impl DocumentFieldStoredKey {
+    fn new(docid: DocumentId, field_id: FieldId) -> DocumentFieldStoredKey {
+        DocumentFieldStoredKey {
+            docid: BEU32::new(docid.0),
+            field_id: BEU16::new(field_id.0),
         }
     }
 }
@@ -77,7 +98,7 @@ impl<'a> BytesEncode<'a> for PostingsCodec {
 
         let mut buffer = Vec::with_capacity(u64_size + docids_size + matches_size);
 
-        let docids_len = item.docids.len();
+        let docids_len = item.docids.len() as u64;
         buffer.extend_from_slice(&docids_len.to_be_bytes());
         buffer.extend_from_slice(item.docids.as_bytes());
         buffer.extend_from_slice(item.matches.as_bytes());
@@ -180,12 +201,17 @@ fn updates_results_name(name: &str) -> String {
     format!("store-{}-updates-results", name)
 }
 
+fn facets_name(name: &str) -> String {
+    format!("store-{}-facets", name)
+}
+
 #[derive(Clone)]
 pub struct Index {
     pub main: Main,
     pub postings_lists: PostingsLists,
     pub documents_fields: DocumentsFields,
     pub documents_fields_counts: DocumentsFieldsCounts,
+    pub facets: Facets,
     pub synonyms: Synonyms,
     pub docs_words: DocsWords,
     pub prefix_documents_cache: PrefixDocumentsCache,
@@ -207,10 +233,7 @@ impl Index {
         let schema = schema.ok_or(Error::SchemaMissing)?;
 
         let attributes = match attributes {
-            Some(attributes) => attributes
-                .iter()
-                .map(|name| schema.attribute(name))
-                .collect(),
+            Some(attributes) => Some(attributes.iter().filter_map(|name| schema.id(*name)).collect()),
             None => None,
         };
 
@@ -219,7 +242,7 @@ impl Index {
             reader,
             documents_fields: self.documents_fields,
             schema: &schema,
-            attributes: attributes.as_ref(),
+            fields: attributes.as_ref(),
         };
 
         Ok(Option::<T>::deserialize(&mut deserializer)?)
@@ -229,7 +252,7 @@ impl Index {
         &self,
         reader: &heed::RoTxn<MainT>,
         document_id: DocumentId,
-        attribute: SchemaAttr,
+        attribute: FieldId,
     ) -> MResult<Option<T>> {
         let bytes = self
             .documents_fields
@@ -240,14 +263,29 @@ impl Index {
         }
     }
 
-    pub fn schema_update(&self, writer: &mut heed::RwTxn<UpdateT>, schema: Schema) -> MResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        update::push_schema_update(writer, self, schema)
+    pub fn document_attribute_bytes<'txn>(
+        &self,
+        reader: &'txn heed::RoTxn<MainT>,
+        document_id: DocumentId,
+        attribute: FieldId,
+    ) -> MResult<Option<&'txn [u8]>> {
+        let bytes = self
+            .documents_fields
+            .document_attribute(reader, document_id, attribute)?;
+        match bytes {
+            Some(bytes) => Ok(Some(bytes)),
+            None => Ok(None),
+        }
     }
 
-    pub fn customs_update(&self, writer: &mut heed::RwTxn<UpdateT>, customs: Vec<u8>) -> ZResult<u64> {
+    pub fn customs_update(&self, writer: &mut heed::RwTxn<UpdateT>, customs: Vec<u8>) -> MResult<u64> {
         let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        update::push_customs_update(writer, self.updates, self.updates_results, customs)
+        Ok(update::push_customs_update(writer, self.updates, self.updates_results, customs)?)
+    }
+
+    pub fn settings_update(&self, writer: &mut heed::RwTxn<UpdateT>, update: SettingsUpdate) -> MResult<u64> {
+        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
+        Ok(update::push_settings_update(writer, self.updates, self.updates_results, update)?)
     }
 
     pub fn documents_addition<D>(&self) -> update::DocumentsAddition<D> {
@@ -279,30 +317,6 @@ impl Index {
         update::push_clear_all(writer, self.updates, self.updates_results)
     }
 
-    pub fn synonyms_update(&self) -> update::SynonymsUpdate {
-        update::SynonymsUpdate::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn stop_words_addition(&self) -> update::StopWordsAddition {
-        update::StopWordsAddition::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn stop_words_deletion(&self) -> update::StopWordsDeletion {
-        update::StopWordsDeletion::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
     pub fn current_update_id(&self, reader: &heed::RoTxn<UpdateT>) -> MResult<Option<u64>> {
         match self.updates.last_update(reader)? {
             Some((id, _)) => Ok(Some(id)),
@@ -329,14 +343,14 @@ impl Index {
             for id in 0..=last_id {
                 if let Some(update) = self.update_status(reader, id)? {
                     updates.push(update);
-                    last_update_result_id = id;
+                    last_update_result_id = id + 1;
                 }
             }
         }
 
         // retrieve all enqueued updates
         if let Some((last_id, _)) = self.updates.last_update(reader)? {
-            for id in last_update_result_id + 1..=last_id {
+            for id in last_update_result_id..=last_id {
                 if let Some(update) = self.update_status(reader, id)? {
                     updates.push(update);
                 }
@@ -347,29 +361,14 @@ impl Index {
     }
 
     pub fn query_builder(&self) -> QueryBuilder {
-        QueryBuilder::new(
-            self.main,
-            self.postings_lists,
-            self.documents_fields_counts,
-            self.synonyms,
-            self.prefix_documents_cache,
-            self.prefix_postings_lists_cache,
-        )
+        QueryBuilder::new(self)
     }
 
-    pub fn query_builder_with_criteria<'c, 'f, 'd>(
-        &self,
+    pub fn query_builder_with_criteria<'c, 'f, 'd, 'i>(
+        &'i self,
         criteria: Criteria<'c>,
-    ) -> QueryBuilder<'c, 'f, 'd> {
-        QueryBuilder::with_criteria(
-            self.main,
-            self.postings_lists,
-            self.documents_fields_counts,
-            self.synonyms,
-            self.prefix_documents_cache,
-            self.prefix_postings_lists_cache,
-            criteria,
-        )
+    ) -> QueryBuilder<'c, 'f, 'd, 'i> {
+        QueryBuilder::with_criteria(self, criteria)
     }
 }
 
@@ -390,12 +389,14 @@ pub fn create(
     let prefix_postings_lists_cache_name = prefix_postings_lists_cache_name(name);
     let updates_name = updates_name(name);
     let updates_results_name = updates_results_name(name);
+    let facets_name = facets_name(name);
 
     // open all the stores
     let main = env.create_poly_database(Some(&main_name))?;
     let postings_lists = env.create_database(Some(&postings_lists_name))?;
     let documents_fields = env.create_database(Some(&documents_fields_name))?;
     let documents_fields_counts = env.create_database(Some(&documents_fields_counts_name))?;
+    let facets = env.create_database(Some(&facets_name))?;
     let synonyms = env.create_database(Some(&synonyms_name))?;
     let docs_words = env.create_database(Some(&docs_words_name))?;
     let prefix_documents_cache = env.create_database(Some(&prefix_documents_cache_name))?;
@@ -412,6 +413,8 @@ pub fn create(
         docs_words: DocsWords { docs_words },
         prefix_postings_lists_cache: PrefixPostingsListsCache { prefix_postings_lists_cache },
         prefix_documents_cache: PrefixDocumentsCache { prefix_documents_cache },
+        facets: Facets { facets },
+
         updates: Updates { updates },
         updates_results: UpdatesResults { updates_results },
         updates_notifier,
@@ -432,6 +435,7 @@ pub fn open(
     let synonyms_name = synonyms_name(name);
     let docs_words_name = docs_words_name(name);
     let prefix_documents_cache_name = prefix_documents_cache_name(name);
+    let facets_name = facets_name(name);
     let prefix_postings_lists_cache_name = prefix_postings_lists_cache_name(name);
     let updates_name = updates_name(name);
     let updates_results_name = updates_results_name(name);
@@ -465,6 +469,10 @@ pub fn open(
         Some(prefix_documents_cache) => prefix_documents_cache,
         None => return Ok(None),
     };
+    let facets = match env.open_database(Some(&facets_name))? {
+        Some(facets) => facets,
+        None => return Ok(None),
+    };
     let prefix_postings_lists_cache = match env.open_database(Some(&prefix_postings_lists_cache_name))? {
         Some(prefix_postings_lists_cache) => prefix_postings_lists_cache,
         None => return Ok(None),
@@ -486,6 +494,7 @@ pub fn open(
         synonyms: Synonyms { synonyms },
         docs_words: DocsWords { docs_words },
         prefix_documents_cache: PrefixDocumentsCache { prefix_documents_cache },
+        facets: Facets { facets },
         prefix_postings_lists_cache: PrefixPostingsListsCache { prefix_postings_lists_cache },
         updates: Updates { updates },
         updates_results: UpdatesResults { updates_results },

meilisearch-core/src/store/prefix_documents_cache.rs

@@ -4,7 +4,7 @@ use heed::types::{OwnedType, CowSlice};
 use heed::Result as ZResult;
 use zerocopy::{AsBytes, FromBytes};
 
-use super::BEU64;
+use super::{BEU64, BEU32};
 use crate::{DocumentId, Highlight};
 use crate::database::MainT;
 
@@ -13,15 +13,15 @@ use crate::database::MainT;
 pub struct PrefixKey {
     prefix: [u8; 4],
     index: BEU64,
-    docid: BEU64,
+    docid: BEU32,
 }
 
 impl PrefixKey {
-    pub fn new(prefix: [u8; 4], index: u64, docid: u64) -> PrefixKey {
+    pub fn new(prefix: [u8; 4], index: u64, docid: u32) -> PrefixKey {
         PrefixKey {
-            prefix: prefix,
+            prefix,
             index: BEU64::new(index),
-            docid: BEU64::new(docid),
+            docid: BEU32::new(docid),
         }
     }
 }
@@ -54,7 +54,7 @@ impl PrefixDocumentsCache {
         prefix: [u8; 4],
     ) -> ZResult<PrefixDocumentsIter<'txn>> {
         let start = PrefixKey::new(prefix, 0, 0);
-        let end = PrefixKey::new(prefix, u64::max_value(), u64::max_value());
+        let end = PrefixKey::new(prefix, u64::max_value(), u32::max_value());
         let iter = self.prefix_documents_cache.range(reader, &(start..=end))?;
         Ok(PrefixDocumentsIter { iter })
     }

meilisearch-core/src/store/synonyms.rs

@@ -1,7 +1,10 @@
-use heed::types::ByteSlice;
-use crate::database::MainT;
+use std::borrow::Cow;
+
 use heed::Result as ZResult;
-use std::sync::Arc;
+use heed::types::ByteSlice;
+
+use crate::database::MainT;
+use crate::{FstSetCow, MResult};
 
 #[derive(Copy, Clone)]
 pub struct Synonyms {
@@ -9,12 +12,9 @@ pub struct Synonyms {
 }
 
 impl Synonyms {
-    pub fn put_synonyms(
-        self,
-        writer: &mut heed::RwTxn<MainT>,
-        word: &[u8],
-        synonyms: &fst::Set,
-    ) -> ZResult<()> {
+    pub fn put_synonyms<A>(self, writer: &mut heed::RwTxn<MainT>, word: &[u8], synonyms: &fst::Set<A>) -> ZResult<()>
+    where A: AsRef<[u8]>,
+    {
         let bytes = synonyms.as_fst().as_bytes();
         self.synonyms.put(writer, word, bytes)
     }
@@ -27,15 +27,18 @@ impl Synonyms {
         self.synonyms.clear(writer)
     }
 
-    pub fn synonyms(self, reader: &heed::RoTxn<MainT>, word: &[u8]) -> ZResult<Option<fst::Set>> {
+    pub(crate) fn synonyms_fst<'txn>(self, reader: &'txn heed::RoTxn<MainT>, word: &[u8]) -> ZResult<FstSetCow<'txn>> {
         match self.synonyms.get(reader, word)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
+            Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
+            None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()),
         }
     }
+
+    pub fn synonyms(self, reader: &heed::RoTxn<MainT>, word: &[u8]) -> MResult<Vec<String>> {
+        let synonyms = self
+            .synonyms_fst(&reader, word)?
+            .stream()
+            .into_strs()?;
+        Ok(synonyms)
+    }
 }

meilisearch-core/src/update/clear_all.rs

@@ -7,14 +7,18 @@ pub fn apply_clear_all(
     index: &store::Index,
 ) -> MResult<()> {
     index.main.put_words_fst(writer, &fst::Set::default())?;
+    index.main.put_external_docids(writer, &fst::Map::default())?;
+    index.main.put_internal_docids(writer, &sdset::SetBuf::default())?;
     index.main.put_ranked_map(writer, &RankedMap::default())?;
     index.main.put_number_of_documents(writer, |_| 0)?;
+    index.main.put_sorted_document_ids_cache(writer, &[])?;
     index.documents_fields.clear(writer)?;
     index.documents_fields_counts.clear(writer)?;
     index.postings_lists.clear(writer)?;
     index.docs_words.clear(writer)?;
     index.prefix_documents_cache.clear(writer)?;
     index.prefix_postings_lists_cache.clear(writer)?;
+    index.facets.clear(writer)?;
 
     Ok(())
 }

meilisearch-core/src/update/customs_update.rs

@@ -1,14 +1,13 @@
-use heed::Result as ZResult;
 
 use crate::database::{MainT, UpdateT};
-use crate::store;
+use crate::{store, MResult};
 use crate::update::{next_update_id, Update};
 
 pub fn apply_customs_update(
     writer: &mut heed::RwTxn<MainT>,
     main_store: store::Main,
     customs: &[u8],
-) -> ZResult<()> {
+) -> MResult<()> {
     main_store.put_customs(writer, customs)
 }
 
@@ -17,7 +16,7 @@ pub fn push_customs_update(
     updates_store: store::Updates,
     updates_results_store: store::UpdatesResults,
     customs: Vec<u8>,
-) -> ZResult<u64> {
+) -> MResult<u64> {
     let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
 
     let update = Update::customs(customs);

meilisearch-core/src/update/documents_addition.rs

@@ -1,14 +1,21 @@
-use std::collections::HashMap;
+use std::borrow::Cow;
+use std::collections::{HashMap, BTreeMap};
 
 use fst::{set::OpBuilder, SetBuilder};
+use indexmap::IndexMap;
+use meilisearch_schema::{Schema, FieldId};
+use meilisearch_types::DocumentId;
 use sdset::{duo::Union, SetOperation};
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;
+use serde_json::Value;
 
 use crate::database::{MainT, UpdateT};
 use crate::database::{UpdateEvent, UpdateEventsEmitter};
+use crate::facets;
 use crate::raw_indexer::RawIndexer;
-use crate::serde::{extract_document_id, serialize_value, Deserializer, Serializer};
-use crate::store;
+use crate::serde::Deserializer;
+use crate::store::{self, DocumentsFields, DocumentsFieldsCounts, DiscoverIds};
+use crate::update::helpers::{index_value, value_to_number, extract_document_id};
 use crate::update::{apply_documents_deletion, compute_short_prefixes, next_update_id, Update};
 use crate::{Error, MResult, RankedMap};
 
@@ -16,6 +23,8 @@ pub struct DocumentsAddition<D> {
     updates_store: store::Updates,
     updates_results_store: store::UpdatesResults,
     updates_notifier: UpdateEventsEmitter,
+    // Whether the user explicitly set the primary key in the update
+    primary_key: Option<String>,
     documents: Vec<D>,
     is_partial: bool,
 }
@@ -32,6 +41,7 @@ impl<D> DocumentsAddition<D> {
             updates_notifier,
             documents: Vec::new(),
             is_partial: false,
+            primary_key: None,
         }
     }
 
@@ -46,9 +56,14 @@ impl<D> DocumentsAddition<D> {
             updates_notifier,
             documents: Vec::new(),
             is_partial: true,
+            primary_key: None,
         }
     }
 
+    pub fn set_primary_key(&mut self, primary_key: String) {
+        self.primary_key = Some(primary_key);
+    }
+
     pub fn update_document(&mut self, document: D) {
         self.documents.push(document);
     }
@@ -64,6 +79,7 @@ impl<D> DocumentsAddition<D> {
             self.updates_results_store,
             self.documents,
             self.is_partial,
+            self.primary_key,
         )?;
         Ok(update_id)
     }
@@ -81,6 +97,7 @@ pub fn push_documents_addition<D: serde::Serialize>(
     updates_results_store: store::UpdatesResults,
     addition: Vec<D>,
     is_partial: bool,
+    primary_key: Option<String>,
 ) -> MResult<u64> {
     let mut values = Vec::with_capacity(addition.len());
     for add in addition {
@@ -92,9 +109,9 @@ pub fn push_documents_addition<D: serde::Serialize>(
     let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
 
     let update = if is_partial {
-        Update::documents_partial(values)
+        Update::documents_partial(primary_key, values)
     } else {
-        Update::documents_addition(values)
+        Update::documents_addition(primary_key, values)
     };
 
     updates_store.put_update(writer, last_update_id, &update)?;
@@ -102,119 +119,116 @@ pub fn push_documents_addition<D: serde::Serialize>(
     Ok(last_update_id)
 }
 
-pub fn apply_documents_addition<'a, 'b>(
-    writer: &'a mut heed::RwTxn<'b, MainT>,
-    index: &store::Index,
-    addition: Vec<HashMap<String, serde_json::Value>>,
-) -> MResult<()> {
-    let mut documents_additions = HashMap::new();
+#[allow(clippy::too_many_arguments)]
+fn index_document<A: AsRef<[u8]>>(
+    writer: &mut heed::RwTxn<MainT>,
+    documents_fields: DocumentsFields,
+    documents_fields_counts: DocumentsFieldsCounts,
+    ranked_map: &mut RankedMap,
+    indexer: &mut RawIndexer<A>,
+    schema: &Schema,
+    field_id: FieldId,
+    document_id: DocumentId,
+    value: &Value,
+) -> MResult<()>
+{
+    let serialized = serde_json::to_vec(value)?;
+    documents_fields.put_document_field(writer, document_id, field_id, &serialized)?;
 
-    let schema = match index.main.schema(writer)? {
-        Some(schema) => schema,
-        None => return Err(Error::SchemaMissing),
-    };
-
-    let identifier = schema.identifier_name();
-
-    // 1. store documents ids for future deletion
-    for document in addition {
-        let document_id = match extract_document_id(identifier, &document)? {
-            Some(id) => id,
-            None => return Err(Error::MissingDocumentId),
-        };
-
-        documents_additions.insert(document_id, document);
+    if let Some(indexed_pos) = schema.is_searchable(field_id) {
+        let number_of_words = index_value(indexer, document_id, indexed_pos, value);
+        if let Some(number_of_words) = number_of_words {
+            documents_fields_counts.put_document_field_count(
+                writer,
+                document_id,
+                indexed_pos,
+                number_of_words as u16,
+            )?;
+        }
     }
 
-    // 2. remove the documents posting lists
-    let number_of_inserted_documents = documents_additions.len();
-    let documents_ids = documents_additions.iter().map(|(id, _)| *id).collect();
-    apply_documents_deletion(writer, index, documents_ids)?;
-
-    let mut ranked_map = match index.main.ranked_map(writer)? {
-        Some(ranked_map) => ranked_map,
-        None => RankedMap::default(),
-    };
-
-    let stop_words = match index.main.stop_words_fst(writer)? {
-        Some(stop_words) => stop_words,
-        None => fst::Set::default(),
-    };
-
-    // 3. index the documents fields in the stores
-    let mut indexer = RawIndexer::new(stop_words);
-
-    for (document_id, document) in documents_additions {
-        let serializer = Serializer {
-            txn: writer,
-            schema: &schema,
-            document_store: index.documents_fields,
-            document_fields_counts: index.documents_fields_counts,
-            indexer: &mut indexer,
-            ranked_map: &mut ranked_map,
-            document_id,
-        };
-
-        document.serialize(serializer)?;
+    if schema.is_ranked(field_id) {
+        let number = value_to_number(value).unwrap_or_default();
+        ranked_map.insert(document_id, field_id, number);
     }
 
-    write_documents_addition_index(
-        writer,
-        index,
-        &ranked_map,
-        number_of_inserted_documents,
-        indexer,
-    )?;
-
-    compute_short_prefixes(writer, index)?;
-
     Ok(())
 }
 
-pub fn apply_documents_partial_addition<'a, 'b>(
-    writer: &'a mut heed::RwTxn<'b, MainT>,
+pub fn apply_addition(
+    writer: &mut heed::RwTxn<MainT>,
     index: &store::Index,
-    addition: Vec<HashMap<String, serde_json::Value>>,
-) -> MResult<()> {
-    let mut documents_additions = HashMap::new();
-
-    let schema = match index.main.schema(writer)? {
+    new_documents: Vec<IndexMap<String, Value>>,
+    partial: bool,
+    primary_key: Option<String>,
+) -> MResult<()>
+{
+    let mut schema = match index.main.schema(writer)? {
         Some(schema) => schema,
         None => return Err(Error::SchemaMissing),
     };
 
-    let identifier = schema.identifier_name();
+    // Retrieve the documents ids related structures
+    let external_docids = index.main.external_docids(writer)?;
+    let internal_docids = index.main.internal_docids(writer)?;
+    let mut available_ids = DiscoverIds::new(&internal_docids);
+
+    let primary_key = match schema.primary_key() {
+        Some(primary_key) => primary_key.to_string(),
+        None => {
+            let name = primary_key.ok_or(Error::MissingPrimaryKey)?;
+            schema.set_primary_key(&name)?;
+            name
+        }
+    };
 
     // 1. store documents ids for future deletion
-    for mut document in addition {
-        let document_id = match extract_document_id(identifier, &document)? {
-            Some(id) => id,
-            None => return Err(Error::MissingDocumentId),
+    let mut documents_additions = HashMap::new();
+    let mut new_external_docids = BTreeMap::new();
+    let mut new_internal_docids = Vec::with_capacity(new_documents.len());
+
+    for mut document in new_documents {
+        let external_docids_get = |docid: &str| {
+            match (external_docids.get(docid), new_external_docids.get(docid)) {
+                (_, Some(&id))
+                | (Some(id), _) => Some(id as u32),
+                (None, None) => None,
+            }
         };
 
-        let mut deserializer = Deserializer {
-            document_id,
-            reader: writer,
-            documents_fields: index.documents_fields,
-            schema: &schema,
-            attributes: None,
-        };
+        let (internal_docid, external_docid) =
+            extract_document_id(
+                &primary_key,
+                &document,
+                &external_docids_get,
+                &mut available_ids,
+            )?;
 
-        // retrieve the old document and
-        // update the new one with missing keys found in the old one
-        let result = Option::<HashMap<String, serde_json::Value>>::deserialize(&mut deserializer)?;
-        if let Some(old_document) = result {
-            for (key, value) in old_document {
-                document.entry(key).or_insert(value);
+        new_external_docids.insert(external_docid, internal_docid.0 as u64);
+        new_internal_docids.push(internal_docid);
+
+        if partial {
+            let mut deserializer = Deserializer {
+                document_id: internal_docid,
+                reader: writer,
+                documents_fields: index.documents_fields,
+                schema: &schema,
+                fields: None,
+            };
+
+            let old_document = Option::<HashMap<String, Value>>::deserialize(&mut deserializer)?;
+            if let Some(old_document) = old_document {
+                for (key, value) in old_document {
+                    document.entry(key).or_insert(value);
+                }
             }
         }
-
-        documents_additions.insert(document_id, document);
+        documents_additions.insert(internal_docid, document);
     }
 
-    // 2. remove the documents posting lists
+    // 2. remove the documents postings lists
     let number_of_inserted_documents = documents_additions.len();
-    let documents_ids = documents_additions.iter().map(|(id, _)| *id).collect();
+    let documents_ids = new_external_docids.iter().map(|(id, _)| id.clone()).collect();
     apply_documents_deletion(writer, index, documents_ids)?;
 
     let mut ranked_map = match index.main.ranked_map(writer)? {
@@ -222,26 +236,28 @@ pub fn apply_documents_partial_addition<'a, 'b>(
         None => RankedMap::default(),
     };
 
-    let stop_words = match index.main.stop_words_fst(writer)? {
-        Some(stop_words) => stop_words,
-        None => fst::Set::default(),
-    };
+    let stop_words = index.main.stop_words_fst(writer)?.map_data(Cow::into_owned)?;
 
-    // 3. index the documents fields in the stores
-    let mut indexer = RawIndexer::new(stop_words);
 
-    for (document_id, document) in documents_additions {
-        let serializer = Serializer {
-            txn: writer,
-            schema: &schema,
-            document_store: index.documents_fields,
-            document_fields_counts: index.documents_fields_counts,
-            indexer: &mut indexer,
-            ranked_map: &mut ranked_map,
-            document_id,
-        };
+    let mut indexer = RawIndexer::new(&stop_words);
 
-        document.serialize(serializer)?;
+    // For each document in this update
+    for (document_id, document) in &documents_additions {
+        // For each key-value pair in the document.
+        for (attribute, value) in document {
+            let (field_id, _) = schema.insert_with_position(&attribute)?;
+            index_document(
+                writer,
+                index.documents_fields,
+                index.documents_fields_counts,
+                &mut ranked_map,
+                &mut indexer,
+                &schema,
+                field_id,
+                *document_id,
+                &value,
+            )?;
+        }
     }
 
     write_documents_addition_index(
@@ -252,11 +268,45 @@ pub fn apply_documents_partial_addition<'a, 'b>(
         indexer,
     )?;
 
-    compute_short_prefixes(writer, index)?;
+    index.main.put_schema(writer, &schema)?;
+
+    let new_external_docids = fst::Map::from_iter(new_external_docids.iter().map(|(ext, id)| (ext, *id as u64)))?;
+    let new_internal_docids = sdset::SetBuf::from_dirty(new_internal_docids);
+    index.main.merge_external_docids(writer, &new_external_docids)?;
+    index.main.merge_internal_docids(writer, &new_internal_docids)?;
+
+    // recompute all facet attributes after document update.
+    if let Some(attributes_for_facetting) = index.main.attributes_for_faceting(writer)? {
+        let docids = index.main.internal_docids(writer)?;
+        let facet_map = facets::facet_map_from_docids(writer, index, &docids, attributes_for_facetting.as_ref())?;
+        index.facets.add(writer, facet_map)?;
+    }
+
+    // update is finished; update sorted document id cache with new state
+    let mut document_ids = index.main.internal_docids(writer)?.to_vec();
+    super::cache_document_ids_sorted(writer, &ranked_map, index, &mut document_ids)?;
 
     Ok(())
 }
 
+pub fn apply_documents_partial_addition(
+    writer: &mut heed::RwTxn<MainT>,
+    index: &store::Index,
+    new_documents: Vec<IndexMap<String, Value>>,
+    primary_key: Option<String>,
+) -> MResult<()> {
+    apply_addition(writer, index, new_documents, true, primary_key)
+}
+
+pub fn apply_documents_addition(
+    writer: &mut heed::RwTxn<MainT>,
+    index: &store::Index,
+    new_documents: Vec<IndexMap<String, Value>>,
+    primary_key: Option<String>,
+) -> MResult<()> {
+    apply_addition(writer, index, new_documents, false, primary_key)
+}
+
 pub fn reindex_all_documents(writer: &mut heed::RwTxn<MainT>, index: &store::Index) -> MResult<()> {
     let schema = match index.main.schema(writer)? {
         Some(schema) => schema,
@@ -276,64 +326,80 @@ pub fn reindex_all_documents(writer: &mut heed::RwTxn<MainT>, index: &store::Ind
     index.main.put_words_fst(writer, &fst::Set::default())?;
     index.main.put_ranked_map(writer, &ranked_map)?;
     index.main.put_number_of_documents(writer, |_| 0)?;
+    index.facets.clear(writer)?;
     index.postings_lists.clear(writer)?;
     index.docs_words.clear(writer)?;
 
-    // 3. re-index chunks of documents (otherwise we make the borrow checker unhappy)
-    for documents_ids in documents_ids_to_reindex.chunks(100) {
-        let stop_words = match index.main.stop_words_fst(writer)? {
-            Some(stop_words) => stop_words,
-            None => fst::Set::default(),
-        };
+    let stop_words = index.main
+        .stop_words_fst(writer)?
+        .map_data(Cow::into_owned)
+        .unwrap();
 
-        let number_of_inserted_documents = documents_ids.len();
-        let mut indexer = RawIndexer::new(stop_words);
-        let mut ram_store = HashMap::new();
+    let number_of_inserted_documents = documents_ids_to_reindex.len();
+    let mut indexer = RawIndexer::new(&stop_words);
+    let mut ram_store = HashMap::new();
 
-        for document_id in documents_ids {
-            for result in index.documents_fields.document_fields(writer, *document_id)? {
-                let (attr, bytes) = result?;
-                let value: serde_json::Value = serde_json::from_slice(bytes)?;
-                ram_store.insert((document_id, attr), value);
-            }
-
-            for ((docid, attr), value) in ram_store.drain() {
-                serialize_value(
-                    writer,
-                    attr,
-                    schema.props(attr),
-                    *docid,
-                    index.documents_fields,
-                    index.documents_fields_counts,
-                    &mut indexer,
-                    &mut ranked_map,
-                    &value,
-                )?;
-            }
+    if let Some(ref attributes_for_facetting) = index.main.attributes_for_faceting(writer)? {
+        let facet_map = facets::facet_map_from_docids(writer, &index, &documents_ids_to_reindex, &attributes_for_facetting)?;
+        index.facets.add(writer, facet_map)?;
+    }
+    // ^-- https://github.com/meilisearch/MeiliSearch/pull/631#issuecomment-626624470 --v
+    for document_id in &documents_ids_to_reindex {
+        for result in index.documents_fields.document_fields(writer, *document_id)? {
+            let (field_id, bytes) = result?;
+            let value: Value = serde_json::from_slice(bytes)?;
+            ram_store.insert((document_id, field_id), value);
         }
 
-        // 4. write the new index in the main store
-        write_documents_addition_index(
-            writer,
-            index,
-            &ranked_map,
-            number_of_inserted_documents,
-            indexer,
-        )?;
+        // For each key-value pair in the document.
+        for ((document_id, field_id), value) in ram_store.drain() {
+            index_document(
+                writer,
+                index.documents_fields,
+                index.documents_fields_counts,
+                &mut ranked_map,
+                &mut indexer,
+                &schema,
+                field_id,
+                *document_id,
+                &value,
+            )?;
+        }
     }
 
-    compute_short_prefixes(writer, index)?;
+    // 4. write the new index in the main store
+    write_documents_addition_index(
+        writer,
+        index,
+        &ranked_map,
+        number_of_inserted_documents,
+        indexer,
+    )?;
+
+    index.main.put_schema(writer, &schema)?;
+
+    // recompute all facet attributes after document update.
+    if let Some(attributes_for_facetting) = index.main.attributes_for_faceting(writer)? {
+        let docids = index.main.internal_docids(writer)?;
+        let facet_map = facets::facet_map_from_docids(writer, index, &docids, attributes_for_facetting.as_ref())?;
+        index.facets.add(writer, facet_map)?;
+    }
+
+    // update is finished; update sorted document id cache with new state
+    let mut document_ids = index.main.internal_docids(writer)?.to_vec();
+    super::cache_document_ids_sorted(writer, &ranked_map, index, &mut document_ids)?;
 
     Ok(())
 }
 
-pub fn write_documents_addition_index(
+pub fn write_documents_addition_index<A: AsRef<[u8]>>(
     writer: &mut heed::RwTxn<MainT>,
     index: &store::Index,
     ranked_map: &RankedMap,
     number_of_inserted_documents: usize,
-    indexer: RawIndexer,
-) -> MResult<()> {
+    indexer: RawIndexer<A>,
+) -> MResult<()>
+{
     let indexed = indexer.build();
     let mut delta_words_builder = SetBuilder::memory();
 
@@ -352,33 +418,27 @@ pub fn write_documents_addition_index(
         index.docs_words.put_doc_words(writer, id, &words)?;
     }
 
-    let delta_words = delta_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
+    let delta_words = delta_words_builder.into_set();
 
-    let words = match index.main.words_fst(writer)? {
-        Some(words) => {
-            let op = OpBuilder::new()
-                .add(words.stream())
-                .add(delta_words.stream())
-                .r#union();
+    let words_fst = index.main.words_fst(writer)?;
+    let words = if !words_fst.is_empty() {
+        let op = OpBuilder::new()
+            .add(words_fst.stream())
+            .add(delta_words.stream())
+            .r#union();
 
-            let mut words_builder = SetBuilder::memory();
-            words_builder.extend_stream(op).unwrap();
-            words_builder
-                .into_inner()
-                .and_then(fst::Set::from_bytes)
-                .unwrap()
-        }
-        None => delta_words,
+        let mut words_builder = SetBuilder::memory();
+        words_builder.extend_stream(op).unwrap();
+        words_builder.into_set()
+    } else {
+        delta_words
     };
 
     index.main.put_words_fst(writer, &words)?;
     index.main.put_ranked_map(writer, ranked_map)?;
     index.main.put_number_of_documents(writer, |old| old + number_of_inserted_documents as u64)?;
 
-    compute_short_prefixes(writer, index)?;
+    compute_short_prefixes(writer, &words, index)?;
 
     Ok(())
 }

meilisearch-core/src/update/documents_deletion.rs

@@ -1,21 +1,20 @@
 use std::collections::{BTreeSet, HashMap, HashSet};
 
 use fst::{SetBuilder, Streamer};
-use meilisearch_schema::Schema;
 use sdset::{duo::DifferenceByKey, SetBuf, SetOperation};
 
 use crate::database::{MainT, UpdateT};
 use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::serde::extract_document_id;
+use crate::facets;
 use crate::store;
 use crate::update::{next_update_id, compute_short_prefixes, Update};
-use crate::{DocumentId, Error, MResult, RankedMap};
+use crate::{DocumentId, Error, MResult, RankedMap, MainWriter, Index};
 
 pub struct DocumentsDeletion {
     updates_store: store::Updates,
     updates_results_store: store::UpdatesResults,
     updates_notifier: UpdateEventsEmitter,
-    documents: Vec<DocumentId>,
+    external_docids: Vec<String>,
 }
 
 impl DocumentsDeletion {
@@ -28,27 +27,12 @@ impl DocumentsDeletion {
             updates_store,
             updates_results_store,
             updates_notifier,
-            documents: Vec::new(),
+            external_docids: Vec::new(),
         }
     }
 
-    pub fn delete_document_by_id(&mut self, document_id: DocumentId) {
-        self.documents.push(document_id);
-    }
-
-    pub fn delete_document<D>(&mut self, schema: &Schema, document: D) -> MResult<()>
-    where
-        D: serde::Serialize,
-    {
-        let identifier = schema.identifier_name();
-        let document_id = match extract_document_id(identifier, &document)? {
-            Some(id) => id,
-            None => return Err(Error::MissingDocumentId),
-        };
-
-        self.delete_document_by_id(document_id);
-
-        Ok(())
+    pub fn delete_document_by_external_docid(&mut self, document_id: String) {
+        self.external_docids.push(document_id);
     }
 
     pub fn finalize(self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64> {
@@ -57,15 +41,15 @@ impl DocumentsDeletion {
             writer,
             self.updates_store,
             self.updates_results_store,
-            self.documents,
+            self.external_docids,
         )?;
         Ok(update_id)
     }
 }
 
-impl Extend<DocumentId> for DocumentsDeletion {
-    fn extend<T: IntoIterator<Item = DocumentId>>(&mut self, iter: T) {
-        self.documents.extend(iter)
+impl Extend<String> for DocumentsDeletion {
+    fn extend<T: IntoIterator<Item=String>>(&mut self, iter: T) {
+        self.external_docids.extend(iter)
     }
 }
 
@@ -73,11 +57,11 @@ pub fn push_documents_deletion(
     writer: &mut heed::RwTxn<UpdateT>,
     updates_store: store::Updates,
     updates_results_store: store::UpdatesResults,
-    deletion: Vec<DocumentId>,
+    external_docids: Vec<String>,
 ) -> MResult<u64> {
     let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
 
-    let update = Update::documents_deletion(deletion);
+    let update = Update::documents_deletion(external_docids);
     updates_store.put_update(writer, last_update_id, &update)?;
 
     Ok(last_update_id)
@@ -86,9 +70,23 @@ pub fn push_documents_deletion(
 pub fn apply_documents_deletion(
     writer: &mut heed::RwTxn<MainT>,
     index: &store::Index,
-    deletion: Vec<DocumentId>,
-) -> MResult<()> {
-    let idset = SetBuf::from_dirty(deletion);
+    external_docids: Vec<String>,
+) -> MResult<()>
+{
+    let (external_docids, internal_docids) = {
+        let new_external_docids = SetBuf::from_dirty(external_docids);
+        let mut internal_docids = Vec::new();
+
+        let old_external_docids = index.main.external_docids(writer)?;
+        for external_docid in new_external_docids.as_slice() {
+            if let Some(id) = old_external_docids.get(external_docid) {
+                internal_docids.push(DocumentId(id as u32));
+            }
+        }
+
+        let new_external_docids = fst::Map::from_iter(new_external_docids.into_iter().map(|k| (k, 0))).unwrap();
+        (new_external_docids, SetBuf::from_dirty(internal_docids))
+    };
 
     let schema = match index.main.schema(writer)? {
         Some(schema) => schema,
@@ -100,28 +98,24 @@ pub fn apply_documents_deletion(
         None => RankedMap::default(),
     };
 
+    // facet filters deletion
+    if let Some(attributes_for_facetting) = index.main.attributes_for_faceting(writer)? {
+        let facet_map = facets::facet_map_from_docids(writer, &index, &internal_docids, &attributes_for_facetting)?;
+        index.facets.remove(writer, facet_map)?;
+    }
+
     // collect the ranked attributes according to the schema
-    let ranked_attrs: Vec<_> = schema
-        .iter()
-        .filter_map(
-            |(_, attr, prop)| {
-                if prop.is_ranked() {
-                    Some(attr)
-                } else {
-                    None
-                }
-            },
-        )
-        .collect();
+    let ranked_fields = schema.ranked();
 
     let mut words_document_ids = HashMap::new();
-    for id in idset {
+    for id in internal_docids.iter().cloned() {
         // remove all the ranked attributes from the ranked_map
-        for ranked_attr in &ranked_attrs {
+        for ranked_attr in ranked_fields {
             ranked_map.remove(id, *ranked_attr);
         }
 
-        if let Some(words) = index.docs_words.doc_words(writer, id)? {
+        let words = index.docs_words.doc_words(writer, id)?;
+        if !words.is_empty() {
             let mut stream = words.stream();
             while let Some(word) = stream.next() {
                 let word = word.to_vec();
@@ -159,33 +153,55 @@ pub fn apply_documents_deletion(
     }
 
     let deleted_documents_len = deleted_documents.len() as u64;
-    for id in deleted_documents {
-        index.docs_words.del_doc_words(writer, id)?;
+    for id in &deleted_documents {
+        index.docs_words.del_doc_words(writer, *id)?;
     }
 
     let removed_words = fst::Set::from_iter(removed_words).unwrap();
-    let words = match index.main.words_fst(writer)? {
-        Some(words_set) => {
-            let op = fst::set::OpBuilder::new()
-                .add(words_set.stream())
-                .add(removed_words.stream())
-                .difference();
+    let words = {
+        let words_set = index.main.words_fst(writer)?;
+        let op = fst::set::OpBuilder::new()
+            .add(words_set.stream())
+            .add(removed_words.stream())
+            .difference();
 
-            let mut words_builder = SetBuilder::memory();
-            words_builder.extend_stream(op).unwrap();
-            words_builder
-                .into_inner()
-                .and_then(fst::Set::from_bytes)
-                .unwrap()
-        }
-        None => fst::Set::default(),
+        let mut words_builder = SetBuilder::memory();
+        words_builder.extend_stream(op).unwrap();
+        words_builder.into_set()
     };
 
     index.main.put_words_fst(writer, &words)?;
     index.main.put_ranked_map(writer, &ranked_map)?;
     index.main.put_number_of_documents(writer, |old| old - deleted_documents_len)?;
 
-    compute_short_prefixes(writer, index)?;
+    // We apply the changes to the user and internal ids
+    index.main.remove_external_docids(writer, &external_docids)?;
+    index.main.remove_internal_docids(writer, &internal_docids)?;
+
+    compute_short_prefixes(writer, &words, index)?;
+
+    // update is finished; update sorted document id cache with new state
+    document_cache_remove_deleted(writer, index, &ranked_map, &deleted_documents)?;
 
     Ok(())
 }
+
+/// rebuilds the document id cache by either removing deleted documents from the existing cache,
+/// and generating a new one from docs in store
+fn document_cache_remove_deleted(writer: &mut MainWriter, index: &Index, ranked_map: &RankedMap, documents_to_delete: &HashSet<DocumentId>) -> MResult<()> {
+    let new_cache = match index.main.sorted_document_ids_cache(writer)? {
+        // only keep documents that are not in the list of deleted documents. Order is preserved,
+        // no need to resort
+        Some(old_cache) => {
+            old_cache.iter().filter(|docid| !documents_to_delete.contains(docid)).cloned().collect::<Vec<_>>()
+        }
+        // couldn't find cached documents, try building a new cache from documents in store
+        None => {
+            let mut document_ids = index.main.internal_docids(writer)?.to_vec();
+            super::cache_document_ids_sorted(writer, ranked_map, index, &mut document_ids)?;
+            document_ids
+        }
+    };
+    index.main.put_sorted_document_ids_cache(writer, &new_cache)?;
+    Ok(())
+}

meilisearch-core/src/update/helpers.rs

@@ -0,0 +1,142 @@
+use std::fmt::Write as _;
+
+use indexmap::IndexMap;
+use meilisearch_schema::IndexedPos;
+use meilisearch_types::DocumentId;
+use ordered_float::OrderedFloat;
+use serde_json::Value;
+
+use crate::Number;
+use crate::raw_indexer::RawIndexer;
+use crate::serde::SerializerError;
+use crate::store::DiscoverIds;
+
+/// Returns the number of words indexed or `None` if the type is unindexable.
+pub fn index_value<A: AsRef<[u8]>>(
+    indexer: &mut RawIndexer<A>,
+    document_id: DocumentId,
+    indexed_pos: IndexedPos,
+    value: &Value,
+) -> Option<usize>
+{
+    match value {
+        Value::Null => None,
+        Value::Bool(boolean) => {
+            let text = boolean.to_string();
+            let number_of_words = indexer.index_text(document_id, indexed_pos, &text);
+            Some(number_of_words)
+        },
+        Value::Number(number) => {
+            let text = number.to_string();
+            Some(indexer.index_text(document_id, indexed_pos, &text))
+        },
+        Value::String(string) => {
+            Some(indexer.index_text(document_id, indexed_pos, &string))
+        },
+        Value::Array(_) => {
+            let text = value_to_string(value);
+            Some(indexer.index_text(document_id, indexed_pos, &text))
+        },
+        Value::Object(_) => {
+            let text = value_to_string(value);
+            Some(indexer.index_text(document_id, indexed_pos, &text))
+        },
+    }
+}
+
+/// Transforms the JSON Value type into a String.
+pub fn value_to_string(value: &Value) -> String {
+    fn internal_value_to_string(string: &mut String, value: &Value) {
+        match value {
+            Value::Null => (),
+            Value::Bool(boolean) => { let _ = write!(string, "{}", &boolean); },
+            Value::Number(number) => { let _ = write!(string, "{}", &number); },
+            Value::String(text) => string.push_str(&text),
+            Value::Array(array) => {
+                for value in array {
+                    internal_value_to_string(string, value);
+                    let _ = string.write_str(". ");
+                }
+            },
+            Value::Object(object) => {
+                for (key, value) in object {
+                    string.push_str(key);
+                    let _ = string.write_str(". ");
+                    internal_value_to_string(string, value);
+                    let _ = string.write_str(". ");
+                }
+            },
+        }
+    }
+
+    let mut string = String::new();
+    internal_value_to_string(&mut string, value);
+    string
+}
+
+/// Transforms the JSON Value type into a Number.
+pub fn value_to_number(value: &Value) -> Option<Number> {
+    use std::str::FromStr;
+
+    match value {
+        Value::Null => None,
+        Value::Bool(boolean) => Some(Number::Unsigned(*boolean as u64)),
+        Value::Number(number) => {
+            match (number.as_i64(), number.as_u64(), number.as_f64()) {
+                (Some(n), _, _) => Some(Number::Signed(n)),
+                (_, Some(n), _) => Some(Number::Unsigned(n)),
+                (_, _, Some(n)) => Some(Number::Float(OrderedFloat(n))),
+                (None, None, None) => None,
+            }
+        },
+        Value::String(string) => Number::from_str(string).ok(),
+        Value::Array(_array) => None,
+        Value::Object(_object) => None,
+    }
+}
+
+/// Validates a string representation to be a correct document id and returns
+/// the corresponding id or generate a new one, this is the way we produce documents ids.
+pub fn discover_document_id<F>(
+    docid: &str,
+    external_docids_get: F,
+    available_docids: &mut DiscoverIds<'_>,
+) -> Result<DocumentId, SerializerError>
+where
+    F: FnOnce(&str) -> Option<u32>
+{
+    if docid.chars().all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') {
+        match external_docids_get(docid) {
+            Some(id) => Ok(DocumentId(id)),
+            None => {
+                let internal_id = available_docids.next().expect("no more ids available");
+                Ok(internal_id)
+            },
+        }
+    } else {
+        Err(SerializerError::InvalidDocumentIdFormat)
+    }
+}
+
+/// Extracts and validates the document id of a document.
+pub fn extract_document_id<F>(
+    primary_key: &str,
+    document: &IndexMap<String, Value>,
+    external_docids_get: F,
+    available_docids: &mut DiscoverIds<'_>,
+) -> Result<(DocumentId, String), SerializerError>
+where
+    F: FnOnce(&str) -> Option<u32>
+{
+    match document.get(primary_key) {
+        Some(value) => {
+            let docid = match value {
+                Value::Number(number) => number.to_string(),
+                Value::String(string) => string.clone(),
+                _ => return Err(SerializerError::InvalidDocumentIdFormat),
+            };
+            discover_document_id(&docid, external_docids_get, available_docids).map(|id| (id, docid))
+        }
+        None => Err(SerializerError::DocumentIdNotFound),
+    }
+}

meilisearch-core/src/update/mod.rs

@@ -2,36 +2,34 @@ mod clear_all;
 mod customs_update;
 mod documents_addition;
 mod documents_deletion;
-mod schema_update;
-mod stop_words_addition;
-mod stop_words_deletion;
-mod synonyms_update;
+mod settings_update;
+mod helpers;
 
 pub use self::clear_all::{apply_clear_all, push_clear_all};
 pub use self::customs_update::{apply_customs_update, push_customs_update};
-pub use self::documents_addition::{
-    apply_documents_addition, apply_documents_partial_addition, DocumentsAddition,
-};
+pub use self::documents_addition::{apply_documents_addition, apply_documents_partial_addition, DocumentsAddition};
 pub use self::documents_deletion::{apply_documents_deletion, DocumentsDeletion};
-pub use self::schema_update::{apply_schema_update, push_schema_update};
-pub use self::stop_words_addition::{apply_stop_words_addition, StopWordsAddition};
-pub use self::stop_words_deletion::{apply_stop_words_deletion, StopWordsDeletion};
-pub use self::synonyms_update::{apply_synonyms_update, SynonymsUpdate};
+pub use self::helpers::{index_value, value_to_string, value_to_number, discover_document_id, extract_document_id};
+pub use self::settings_update::{apply_settings_update, push_settings_update};
 
 use std::cmp;
-use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::time::Instant;
 
 use chrono::{DateTime, Utc};
-use heed::Result as ZResult;
-use log::debug;
-use serde::{Deserialize, Serialize};
 use fst::{IntoStreamer, Streamer};
+use heed::Result as ZResult;
+use indexmap::IndexMap;
+use log::debug;
 use sdset::Set;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
 
-use crate::{store, DocumentId, MResult};
+use meilisearch_error::ErrorCode;
+use meilisearch_types::DocumentId;
+
+use crate::{store, MResult, RankedMap};
 use crate::database::{MainT, UpdateT};
-use meilisearch_schema::Schema;
+use crate::settings::SettingsUpdate;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Update {
@@ -47,13 +45,6 @@ impl Update {
         }
     }
 
-    fn schema(data: Schema) -> Update {
-        Update {
-            data: UpdateData::Schema(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
     fn customs(data: Vec<u8>) -> Update {
         Update {
             data: UpdateData::Customs(data),
@@ -61,44 +52,30 @@ impl Update {
         }
     }
 
-    fn documents_addition(data: Vec<HashMap<String, serde_json::Value>>) -> Update {
+    fn documents_addition(primary_key: Option<String>, documents: Vec<IndexMap<String, Value>>) -> Update {
         Update {
-            data: UpdateData::DocumentsAddition(data),
+            data: UpdateData::DocumentsAddition{ documents, primary_key },
             enqueued_at: Utc::now(),
         }
     }
 
-    fn documents_partial(data: Vec<HashMap<String, serde_json::Value>>) -> Update {
+    fn documents_partial(primary_key: Option<String>, documents: Vec<IndexMap<String, Value>>) -> Update {
         Update {
-            data: UpdateData::DocumentsPartial(data),
+            data: UpdateData::DocumentsPartial{ documents, primary_key },
             enqueued_at: Utc::now(),
         }
     }
 
-    fn documents_deletion(data: Vec<DocumentId>) -> Update {
+    fn documents_deletion(data: Vec<String>) -> Update {
         Update {
             data: UpdateData::DocumentsDeletion(data),
             enqueued_at: Utc::now(),
         }
     }
 
-    fn synonyms_update(data: BTreeMap<String, Vec<String>>) -> Update {
+    fn settings(data: SettingsUpdate) -> Update {
         Update {
-            data: UpdateData::SynonymsUpdate(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn stop_words_addition(data: BTreeSet<String>) -> Update {
-        Update {
-            data: UpdateData::StopWordsAddition(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn stop_words_deletion(data: BTreeSet<String>) -> Update {
-        Update {
-            data: UpdateData::StopWordsDeletion(data),
+            data: UpdateData::Settings(Box::new(data)),
             enqueued_at: Utc::now(),
         }
     }
@@ -107,39 +84,36 @@ impl Update {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum UpdateData {
     ClearAll,
-    Schema(Schema),
     Customs(Vec<u8>),
-    DocumentsAddition(Vec<HashMap<String, serde_json::Value>>),
-    DocumentsPartial(Vec<HashMap<String, serde_json::Value>>),
-    DocumentsDeletion(Vec<DocumentId>),
-    SynonymsUpdate(BTreeMap<String, Vec<String>>),
-    StopWordsAddition(BTreeSet<String>),
-    StopWordsDeletion(BTreeSet<String>),
+    // (primary key, documents)
+    DocumentsAddition {
+        primary_key: Option<String>,
+        documents: Vec<IndexMap<String, Value>>
+    },
+    DocumentsPartial {
+        primary_key: Option<String>,
+        documents: Vec<IndexMap<String, Value>>,
+    },
+    DocumentsDeletion(Vec<String>),
+    Settings(Box<SettingsUpdate>)
 }
 
 impl UpdateData {
     pub fn update_type(&self) -> UpdateType {
         match self {
             UpdateData::ClearAll => UpdateType::ClearAll,
-            UpdateData::Schema(_) => UpdateType::Schema,
             UpdateData::Customs(_) => UpdateType::Customs,
-            UpdateData::DocumentsAddition(addition) => UpdateType::DocumentsAddition {
-                number: addition.len(),
+            UpdateData::DocumentsAddition{ documents, .. } => UpdateType::DocumentsAddition {
+                number: documents.len(),
             },
-            UpdateData::DocumentsPartial(addition) => UpdateType::DocumentsPartial {
-                number: addition.len(),
+            UpdateData::DocumentsPartial{ documents, .. } => UpdateType::DocumentsPartial {
+                number: documents.len(),
             },
             UpdateData::DocumentsDeletion(deletion) => UpdateType::DocumentsDeletion {
                 number: deletion.len(),
             },
-            UpdateData::SynonymsUpdate(addition) => UpdateType::SynonymsUpdate {
-                number: addition.len(),
-            },
-            UpdateData::StopWordsAddition(addition) => UpdateType::StopWordsAddition {
-                number: addition.len(),
-            },
-            UpdateData::StopWordsDeletion(deletion) => UpdateType::StopWordsDeletion {
-                number: deletion.len(),
+            UpdateData::Settings(update) => UpdateType::Settings {
+                settings: update.clone(),
             },
         }
     }
@@ -149,14 +123,11 @@ impl UpdateData {
 #[serde(tag = "name")]
 pub enum UpdateType {
     ClearAll,
-    Schema,
     Customs,
     DocumentsAddition { number: usize },
     DocumentsPartial { number: usize },
     DocumentsDeletion { number: usize },
-    SynonymsUpdate { number: usize },
-    StopWordsAddition { number: usize },
-    StopWordsDeletion { number: usize },
+    Settings { settings: Box<SettingsUpdate> },
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -167,6 +138,12 @@ pub struct ProcessedUpdateResult {
     pub update_type: UpdateType,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub error: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error_type: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error_code: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error_link: Option<String>,
     pub duration: f64, // in seconds
     pub enqueued_at: DateTime<Utc>,
     pub processed_at: DateTime<Utc>,
@@ -242,8 +219,8 @@ pub fn next_update_id(
     Ok(new_update_id)
 }
 
-pub fn update_task<'a, 'b>(
-    writer: &'a mut heed::RwTxn<'b, MainT>,
+pub fn update_task(
+    writer: &mut heed::RwTxn<MainT>,
     index: &store::Index,
     update_id: u64,
     update: Update,
@@ -261,14 +238,6 @@ pub fn update_task<'a, 'b>(
 
             (update_type, result, start.elapsed())
         }
-        UpdateData::Schema(schema) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::Schema;
-            let result = apply_schema_update(writer, &schema, index);
-
-            (update_type, result, start.elapsed())
-        }
         UpdateData::Customs(customs) => {
             let start = Instant::now();
 
@@ -277,25 +246,25 @@ pub fn update_task<'a, 'b>(
 
             (update_type, result, start.elapsed())
         }
-        UpdateData::DocumentsAddition(documents) => {
+        UpdateData::DocumentsAddition { documents, primary_key } => {
             let start = Instant::now();
 
             let update_type = UpdateType::DocumentsAddition {
                 number: documents.len(),
             };
 
-            let result = apply_documents_addition(writer, index, documents);
+            let result = apply_documents_addition(writer, index, documents, primary_key);
 
             (update_type, result, start.elapsed())
         }
-        UpdateData::DocumentsPartial(documents) => {
+        UpdateData::DocumentsPartial{ documents, primary_key } => {
             let start = Instant::now();
 
             let update_type = UpdateType::DocumentsPartial {
                 number: documents.len(),
             };
 
-            let result = apply_documents_partial_addition(writer, index, documents);
+            let result = apply_documents_partial_addition(writer, index, documents, primary_key);
 
             (update_type, result, start.elapsed())
         }
@@ -310,37 +279,18 @@ pub fn update_task<'a, 'b>(
 
             (update_type, result, start.elapsed())
         }
-        UpdateData::SynonymsUpdate(synonyms) => {
+        UpdateData::Settings(settings) => {
             let start = Instant::now();
 
-            let update_type = UpdateType::SynonymsUpdate {
-                number: synonyms.len(),
+            let update_type = UpdateType::Settings {
+                settings: settings.clone(),
             };
 
-            let result = apply_synonyms_update(writer, index.main, index.synonyms, synonyms);
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::StopWordsAddition(stop_words) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::StopWordsAddition {
-                number: stop_words.len(),
-            };
-
-            let result =
-                apply_stop_words_addition(writer, index.main, index.postings_lists, stop_words);
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::StopWordsDeletion(stop_words) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::StopWordsDeletion {
-                number: stop_words.len(),
-            };
-
-            let result = apply_stop_words_deletion(writer, index, stop_words);
+            let result = apply_settings_update(
+                writer,
+                index,
+                *settings,
+            );
 
             (update_type, result, start.elapsed())
         }
@@ -354,7 +304,10 @@ pub fn update_task<'a, 'b>(
     let status = ProcessedUpdateResult {
         update_id,
         update_type,
-        error: result.map_err(|e| e.to_string()).err(),
+        error: result.as_ref().map_err(|e| e.to_string()).err(),
+        error_code: result.as_ref().map_err(|e| e.error_name()).err(),
+        error_type: result.as_ref().map_err(|e| e.error_type()).err(),
+        error_link: result.as_ref().map_err(|e| e.error_url()).err(),
         duration: duration.as_secs_f64(),
         enqueued_at,
         processed_at: Utc::now(),
@@ -363,13 +316,13 @@ pub fn update_task<'a, 'b>(
     Ok(status)
 }
 
-fn compute_short_prefixes(writer: &mut heed::RwTxn<MainT>, index: &store::Index) -> MResult<()> {
-    // retrieve the words fst to compute all those prefixes
-    let words_fst = match index.main.words_fst(writer)? {
-        Some(fst) => fst,
-        None => return Ok(()),
-    };
-
+fn compute_short_prefixes<A>(
+    writer: &mut heed::RwTxn<MainT>,
+    words_fst: &fst::Set<A>,
+    index: &store::Index,
+) -> MResult<()>
+where A: AsRef<[u8]>,
+{
     // clear the prefixes
     let pplc_store = index.prefix_postings_lists_cache;
     pplc_store.clear(writer)?;
@@ -426,3 +379,13 @@ fn compute_short_prefixes(writer: &mut heed::RwTxn<MainT>, index: &store::Index)
 
     Ok(())
 }
+
+fn cache_document_ids_sorted(
+    writer: &mut heed::RwTxn<MainT>,
+    ranked_map: &RankedMap,
+    index: &store::Index,
+    document_ids: &mut [DocumentId],
+) -> MResult<()> {
+    crate::bucket_sort::placeholder_document_sort(document_ids, index, writer, ranked_map)?;
+    index.main.put_sorted_document_ids_cache(writer, &document_ids)
+}

meilisearch-core/src/update/schema_update.rs

@@ -1,64 +0,0 @@
-use meilisearch_schema::{Diff, Schema};
-
-use crate::database::{MainT, UpdateT};
-use crate::update::documents_addition::reindex_all_documents;
-use crate::update::{next_update_id, Update};
-use crate::{error::UnsupportedOperation, store, MResult};
-
-pub fn apply_schema_update(
-    writer: &mut heed::RwTxn<MainT>,
-    new_schema: &Schema,
-    index: &store::Index,
-) -> MResult<()> {
-    use UnsupportedOperation::{
-        CanOnlyIntroduceNewSchemaAttributesAtEnd, CannotRemoveSchemaAttribute,
-        CannotReorderSchemaAttribute, CannotUpdateSchemaIdentifier,
-    };
-
-    let mut need_full_reindexing = false;
-
-    if let Some(old_schema) = index.main.schema(writer)? {
-        for diff in meilisearch_schema::diff(&old_schema, new_schema) {
-            match diff {
-                Diff::IdentChange { .. } => return Err(CannotUpdateSchemaIdentifier.into()),
-                Diff::AttrMove { .. } => return Err(CannotReorderSchemaAttribute.into()),
-                Diff::AttrPropsChange { old, new, .. } => {
-                    if new.indexed != old.indexed {
-                        need_full_reindexing = true;
-                    }
-                    if new.ranked != old.ranked {
-                        need_full_reindexing = true;
-                    }
-                }
-                Diff::NewAttr { pos, .. } => {
-                    // new attribute not at the end of the schema
-                    if pos < old_schema.number_of_attributes() {
-                        return Err(CanOnlyIntroduceNewSchemaAttributesAtEnd.into());
-                    }
-                }
-                Diff::RemovedAttr { .. } => return Err(CannotRemoveSchemaAttribute.into()),
-            }
-        }
-    }
-
-    index.main.put_schema(writer, new_schema)?;
-
-    if need_full_reindexing {
-        reindex_all_documents(writer, index)?
-    }
-
-    Ok(())
-}
-
-pub fn push_schema_update(
-    writer: &mut heed::RwTxn<UpdateT>,
-    index: &store::Index,
-    schema: Schema,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, index.updates, index.updates_results)?;
-
-    let update = Update::schema(schema);
-    index.updates.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}

meilisearch-core/src/update/settings_update.rs

@@ -0,0 +1,325 @@
+use std::{borrow::Cow, collections::{BTreeMap, BTreeSet}};
+
+use heed::Result as ZResult;
+use fst::{set::OpBuilder, SetBuilder};
+use sdset::SetBuf;
+use meilisearch_schema::Schema;
+use meilisearch_tokenizer::analyzer::{Analyzer, AnalyzerConfig};
+
+use crate::database::{MainT, UpdateT};
+use crate::settings::{UpdateState, SettingsUpdate, RankingRule};
+use crate::update::documents_addition::reindex_all_documents;
+use crate::update::{next_update_id, Update};
+use crate::{store, MResult, Error};
+
+pub fn push_settings_update(
+    writer: &mut heed::RwTxn<UpdateT>,
+    updates_store: store::Updates,
+    updates_results_store: store::UpdatesResults,
+    settings: SettingsUpdate,
+) -> ZResult<u64> {
+    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
+
+    let update = Update::settings(settings);
+    updates_store.put_update(writer, last_update_id, &update)?;
+
+    Ok(last_update_id)
+}
+
+pub fn apply_settings_update(
+    writer: &mut heed::RwTxn<MainT>,
+    index: &store::Index,
+    settings: SettingsUpdate,
+) -> MResult<()> {
+    let mut must_reindex = false;
+
+    let mut schema = match index.main.schema(writer)? {
+        Some(schema) => schema,
+        None => {
+            match settings.primary_key.clone() {
+                UpdateState::Update(id) => Schema::with_primary_key(&id),
+                _ => return Err(Error::MissingPrimaryKey)
+            }
+        }
+    };
+
+    match settings.ranking_rules {
+        UpdateState::Update(v) => {
+            let ranked_field: Vec<&str> = v.iter().filter_map(RankingRule::field).collect();
+            schema.update_ranked(&ranked_field)?;
+            index.main.put_ranking_rules(writer, &v)?;
+            must_reindex = true;
+        },
+        UpdateState::Clear => {
+            index.main.delete_ranking_rules(writer)?;
+            schema.clear_ranked();
+            must_reindex = true;
+        },
+        UpdateState::Nothing => (),
+    }
+
+    match settings.distinct_attribute {
+        UpdateState::Update(v) => {
+            let field_id = schema.insert(&v)?;
+            index.main.put_distinct_attribute(writer, field_id)?;
+        },
+        UpdateState::Clear => {
+            index.main.delete_distinct_attribute(writer)?;
+        },
+        UpdateState::Nothing => (),
+    }
+
+    match settings.searchable_attributes.clone() {
+        UpdateState::Update(v) => {
+            if v.iter().any(|e| e == "*") || v.is_empty() {
+                schema.set_all_searchable();
+            } else {
+                schema.update_searchable(v)?;
+            }
+            must_reindex = true;
+        },
+        UpdateState::Clear => {
+            schema.set_all_searchable();
+            must_reindex = true;
+        },
+        UpdateState::Nothing => (),
+    }
+    match settings.displayed_attributes.clone() {
+        UpdateState::Update(v) => {
+            if v.contains("*") || v.is_empty() {
+                schema.set_all_displayed();
+            } else {
+                schema.update_displayed(v)?
+            }
+        },
+        UpdateState::Clear => {
+            schema.set_all_displayed();
+        },
+        UpdateState::Nothing => (),
+    }
+
+    match settings.attributes_for_faceting {
+        UpdateState::Update(attrs) => {
+            apply_attributes_for_faceting_update(writer, index, &mut schema, &attrs)?;
+            must_reindex = true;
+        },
+        UpdateState::Clear => {
+            index.main.delete_attributes_for_faceting(writer)?;
+            index.facets.clear(writer)?;
+        },
+        UpdateState::Nothing => (),
+    }
+
+    index.main.put_schema(writer, &schema)?;
+
+    match settings.stop_words {
+        UpdateState::Update(stop_words) => {
+            if apply_stop_words_update(writer, index, stop_words)? {
+                must_reindex = true;
+            }
+        },
+        UpdateState::Clear => {
+            if apply_stop_words_update(writer, index, BTreeSet::new())? {
+                must_reindex = true;
+            }
+        },
+        UpdateState::Nothing => (),
+    }
+
+    match settings.synonyms {
+        UpdateState::Update(synonyms) => apply_synonyms_update(writer, index, synonyms)?,
+        UpdateState::Clear => apply_synonyms_update(writer, index, BTreeMap::new())?,
+        UpdateState::Nothing => (),
+    }
+
+    if must_reindex {
+        reindex_all_documents(writer, index)?;
+    }
+
+    Ok(())
+}
+
+fn apply_attributes_for_faceting_update(
+    writer: &mut heed::RwTxn<MainT>,
+    index: &store::Index,
+    schema: &mut Schema,
+    attributes: &[String]
+    ) -> MResult<()> {
+    let mut attribute_ids = Vec::new();
+    for name in attributes {
+        attribute_ids.push(schema.insert(name)?);
+    }
+    let attributes_for_faceting = SetBuf::from_dirty(attribute_ids);
+    index.main.put_attributes_for_faceting(writer, &attributes_for_faceting)?;
+    Ok(())
+}
+
+pub fn apply_stop_words_update(
+    writer: &mut heed::RwTxn<MainT>,
+    index: &store::Index,
+    stop_words: BTreeSet<String>,
+) -> MResult<bool>
+{
+    let mut must_reindex = false;
+
+    let old_stop_words: BTreeSet<String> = index.main
+        .stop_words_fst(writer)?
+        .stream()
+        .into_strs()?
+        .into_iter()
+        .collect();
+
+    let deletion: BTreeSet<String> = old_stop_words.difference(&stop_words).cloned().collect();
+    let addition: BTreeSet<String> = stop_words.difference(&old_stop_words).cloned().collect();
+
+    if !addition.is_empty() {
+        apply_stop_words_addition(writer, index, addition)?;
+    }
+
+    if !deletion.is_empty() {
+        must_reindex = true;
+        apply_stop_words_deletion(writer, index, deletion)?;
+    }
+
+    let words_fst = index.main.words_fst(writer)?;
+    if !words_fst.is_empty() {
+        let stop_words = fst::Set::from_iter(stop_words)?;
+        let op = OpBuilder::new()
+            .add(&words_fst)
+            .add(&stop_words)
+            .difference();
+
+        let mut builder = fst::SetBuilder::memory();
+        builder.extend_stream(op)?;
+        let words_fst = builder.into_set();
+
+        index.main.put_words_fst(writer, &words_fst)?;
+        index.main.put_stop_words_fst(writer, &stop_words)?;
+    }
+
+    Ok(must_reindex)
+}
+
+fn apply_stop_words_addition(
+    writer: &mut heed::RwTxn<MainT>,
+    index: &store::Index,
+    addition: BTreeSet<String>,
+) -> MResult<()>
+{
+    let main_store = index.main;
+    let postings_lists_store = index.postings_lists;
+
+    let mut stop_words_builder = SetBuilder::memory();
+
+    for word in addition {
+        stop_words_builder.insert(&word)?;
+        // we remove every posting list associated to a new stop word
+        postings_lists_store.del_postings_list(writer, word.as_bytes())?;
+    }
+
+    // create the new delta stop words fst
+    let delta_stop_words = stop_words_builder.into_set();
+
+    // we also need to remove all the stop words from the main fst
+    let words_fst = main_store.words_fst(writer)?;
+    if !words_fst.is_empty() {
+        let op = OpBuilder::new()
+            .add(&words_fst)
+            .add(&delta_stop_words)
+            .difference();
+
+        let mut word_fst_builder = SetBuilder::memory();
+        word_fst_builder.extend_stream(op)?;
+        let word_fst = word_fst_builder.into_set();
+
+        main_store.put_words_fst(writer, &word_fst)?;
+    }
+
+    // now we add all of these stop words from the main store
+    let stop_words_fst = main_store.stop_words_fst(writer)?;
+
+    let op = OpBuilder::new()
+        .add(&stop_words_fst)
+        .add(&delta_stop_words)
+        .r#union();
+
+    let mut stop_words_builder = SetBuilder::memory();
+    stop_words_builder.extend_stream(op)?;
+    let stop_words_fst = stop_words_builder.into_set();
+
+    main_store.put_stop_words_fst(writer, &stop_words_fst)?;
+
+    Ok(())
+}
+
+fn apply_stop_words_deletion(
+    writer: &mut heed::RwTxn<MainT>,
+    index: &store::Index,
+    deletion: BTreeSet<String>,
+) -> MResult<()> {
+
+    let mut stop_words_builder = SetBuilder::memory();
+
+    for word in deletion {
+        stop_words_builder.insert(&word)?;
+    }
+
+    // create the new delta stop words fst
+    let delta_stop_words = stop_words_builder.into_set();
+
+    // now we delete all of these stop words from the main store
+    let stop_words_fst = index.main.stop_words_fst(writer)?;
+
+    let op = OpBuilder::new()
+        .add(&stop_words_fst)
+        .add(&delta_stop_words)
+        .difference();
+
+    let mut stop_words_builder = SetBuilder::memory();
+    stop_words_builder.extend_stream(op)?;
+    let stop_words_fst = stop_words_builder.into_set();
+
+    Ok(index.main.put_stop_words_fst(writer, &stop_words_fst)?)
+}
+
+pub fn apply_synonyms_update(
+    writer: &mut heed::RwTxn<MainT>,
+    index: &store::Index,
+    synonyms: BTreeMap<String, Vec<String>>,
+) -> MResult<()> {
+
+    let main_store = index.main;
+    let synonyms_store = index.synonyms;
+    let stop_words = index.main.stop_words_fst(writer)?.map_data(Cow::into_owned)?;
+    let analyzer = Analyzer::new(AnalyzerConfig::default_with_stopwords(&stop_words));
+
+    fn normalize<T: AsRef<[u8]>>(analyzer: &Analyzer<T>, text: &str) -> String {
+        analyzer.analyze(&text)
+            .tokens()
+            .fold(String::new(), |s, t| s + t.text())
+    }
+
+    let mut synonyms_builder = SetBuilder::memory();
+    synonyms_store.clear(writer)?;
+    for (word, alternatives) in synonyms {
+        let word = normalize(&analyzer, &word);
+
+        synonyms_builder.insert(&word)?;
+
+        let alternatives = {
+            let alternatives = alternatives.iter().map(|text| normalize(&analyzer, &text)).collect();
+            let alternatives = SetBuf::from_dirty(alternatives);
+            let mut alternatives_builder = SetBuilder::memory();
+            alternatives_builder.extend_iter(alternatives)?;
+            alternatives_builder.into_set()
+        };
+
+        synonyms_store.put_synonyms(writer, word.as_bytes(), &alternatives)?;
+    }
+
+    let synonyms_set = synonyms_builder.into_set();
+
+    main_store.put_synonyms_fst(writer, &synonyms_set)?;
+
+    Ok(())
+}

meilisearch-core/src/update/stop_words_addition.rs

@@ -1,118 +0,0 @@
-use std::collections::BTreeSet;
-
-use fst::{set::OpBuilder, SetBuilder};
-
-use crate::database::{MainT, UpdateT};
-use crate::automaton::normalize_str;
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult};
-
-pub struct StopWordsAddition {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: UpdateEventsEmitter,
-    stop_words: BTreeSet<String>,
-}
-
-impl StopWordsAddition {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: UpdateEventsEmitter,
-    ) -> StopWordsAddition {
-        StopWordsAddition {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            stop_words: BTreeSet::new(),
-        }
-    }
-
-    pub fn add_stop_word<S: AsRef<str>>(&mut self, stop_word: S) {
-        let stop_word = normalize_str(stop_word.as_ref());
-        self.stop_words.insert(stop_word);
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        let update_id = push_stop_words_addition(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.stop_words,
-        )?;
-        Ok(update_id)
-    }
-}
-
-pub fn push_stop_words_addition(
-    writer: &mut heed::RwTxn<UpdateT>,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    addition: BTreeSet<String>,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::stop_words_addition(addition);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_stop_words_addition(
-    writer: &mut heed::RwTxn<MainT>,
-    main_store: store::Main,
-    postings_lists_store: store::PostingsLists,
-    addition: BTreeSet<String>,
-) -> MResult<()> {
-    let mut stop_words_builder = SetBuilder::memory();
-
-    for word in addition {
-        stop_words_builder.insert(&word).unwrap();
-        // we remove every posting list associated to a new stop word
-        postings_lists_store.del_postings_list(writer, word.as_bytes())?;
-    }
-
-    // create the new delta stop words fst
-    let delta_stop_words = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    // we also need to remove all the stop words from the main fst
-    if let Some(word_fst) = main_store.words_fst(writer)? {
-        let op = OpBuilder::new()
-            .add(&word_fst)
-            .add(&delta_stop_words)
-            .difference();
-
-        let mut word_fst_builder = SetBuilder::memory();
-        word_fst_builder.extend_stream(op).unwrap();
-        let word_fst = word_fst_builder
-            .into_inner()
-            .and_then(fst::Set::from_bytes)
-            .unwrap();
-
-        main_store.put_words_fst(writer, &word_fst)?;
-    }
-
-    // now we add all of these stop words from the main store
-    let stop_words_fst = main_store.stop_words_fst(writer)?.unwrap_or_default();
-
-    let op = OpBuilder::new()
-        .add(&stop_words_fst)
-        .add(&delta_stop_words)
-        .r#union();
-
-    let mut stop_words_builder = SetBuilder::memory();
-    stop_words_builder.extend_stream(op).unwrap();
-    let stop_words_fst = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    main_store.put_stop_words_fst(writer, &stop_words_fst)?;
-
-    Ok(())
-}

meilisearch-core/src/update/stop_words_deletion.rs

@@ -1,107 +0,0 @@
-use std::collections::BTreeSet;
-
-use fst::{set::OpBuilder, SetBuilder};
-
-use crate::database::{MainT, UpdateT};
-use crate::automaton::normalize_str;
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::update::documents_addition::reindex_all_documents;
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult};
-
-pub struct StopWordsDeletion {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: UpdateEventsEmitter,
-    stop_words: BTreeSet<String>,
-}
-
-impl StopWordsDeletion {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: UpdateEventsEmitter,
-    ) -> StopWordsDeletion {
-        StopWordsDeletion {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            stop_words: BTreeSet::new(),
-        }
-    }
-
-    pub fn delete_stop_word<S: AsRef<str>>(&mut self, stop_word: S) {
-        let stop_word = normalize_str(stop_word.as_ref());
-        self.stop_words.insert(stop_word);
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        let update_id = push_stop_words_deletion(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.stop_words,
-        )?;
-        Ok(update_id)
-    }
-}
-
-pub fn push_stop_words_deletion(
-    writer: &mut heed::RwTxn<UpdateT>,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    deletion: BTreeSet<String>,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::stop_words_deletion(deletion);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_stop_words_deletion(
-    writer: &mut heed::RwTxn<MainT>,
-    index: &store::Index,
-    deletion: BTreeSet<String>,
-) -> MResult<()> {
-    let mut stop_words_builder = SetBuilder::memory();
-
-    for word in deletion {
-        stop_words_builder.insert(&word).unwrap();
-    }
-
-    // create the new delta stop words fst
-    let delta_stop_words = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    // now we delete all of these stop words from the main store
-    let stop_words_fst = index.main.stop_words_fst(writer)?.unwrap_or_default();
-
-    let op = OpBuilder::new()
-        .add(&stop_words_fst)
-        .add(&delta_stop_words)
-        .difference();
-
-    let mut stop_words_builder = SetBuilder::memory();
-    stop_words_builder.extend_stream(op).unwrap();
-    let stop_words_fst = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    index.main.put_stop_words_fst(writer, &stop_words_fst)?;
-
-    // now that we have setup the stop words
-    // lets reindex everything...
-    if let Ok(number) = index.main.number_of_documents(writer) {
-        if number > 0 {
-            reindex_all_documents(writer, index)?;
-        }
-    }
-
-    Ok(())
-}

meilisearch-core/src/update/synonyms_update.rs

@@ -1,103 +0,0 @@
-use std::collections::BTreeMap;
-
-use fst::SetBuilder;
-use sdset::SetBuf;
-
-use crate::database::{MainT, UpdateT};
-use crate::automaton::normalize_str;
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult};
-
-pub struct SynonymsUpdate {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: UpdateEventsEmitter,
-    synonyms: BTreeMap<String, Vec<String>>,
-}
-
-impl SynonymsUpdate {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: UpdateEventsEmitter,
-    ) -> SynonymsUpdate {
-        SynonymsUpdate {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            synonyms: BTreeMap::new(),
-        }
-    }
-
-    pub fn add_synonym<S, T, I>(&mut self, synonym: S, alternatives: I)
-    where
-        S: AsRef<str>,
-        T: AsRef<str>,
-        I: IntoIterator<Item = T>,
-    {
-        let synonym = normalize_str(synonym.as_ref());
-        let alternatives = alternatives.into_iter().map(|s| s.as_ref().to_lowercase());
-        self.synonyms
-            .entry(synonym)
-            .or_insert_with(Vec::new)
-            .extend(alternatives);
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        let update_id = push_synonyms_update(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.synonyms,
-        )?;
-        Ok(update_id)
-    }
-}
-
-pub fn push_synonyms_update(
-    writer: &mut heed::RwTxn<UpdateT>,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    addition: BTreeMap<String, Vec<String>>,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::synonyms_update(addition);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_synonyms_update(
-    writer: &mut heed::RwTxn<MainT>,
-    main_store: store::Main,
-    synonyms_store: store::Synonyms,
-    addition: BTreeMap<String, Vec<String>>,
-) -> MResult<()> {
-    let mut synonyms_builder = SetBuilder::memory();
-    synonyms_store.clear(writer)?;
-    for (word, alternatives) in addition {
-        synonyms_builder.insert(&word).unwrap();
-
-        let alternatives = {
-            let alternatives = SetBuf::from_dirty(alternatives);
-            let mut alternatives_builder = SetBuilder::memory();
-            alternatives_builder.extend_iter(alternatives).unwrap();
-            let bytes = alternatives_builder.into_inner().unwrap();
-            fst::Set::from_bytes(bytes).unwrap()
-        };
-
-        synonyms_store.put_synonyms(writer, word.as_bytes(), &alternatives)?;
-    }
-
-    let synonyms = synonyms_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    main_store.put_synonyms_fst(writer, &synonyms)?;
-
-    Ok(())
-}

meilisearch-error/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "meilisearch-error"
+version = "0.19.0"
+authors = ["marin <postma.marin@protonmail.com>"]
+edition = "2018"
+
+[dependencies]
+actix-http = "2.2.0"

meilisearch-error/src/lib.rs

@@ -0,0 +1,185 @@
+use std::fmt;
+
+use actix_http::http::StatusCode;
+
+pub trait ErrorCode: std::error::Error {
+    fn error_code(&self) -> Code;
+
+    /// returns the HTTP status code ascociated with the error
+    fn http_status(&self) -> StatusCode {
+        self.error_code().http()
+    }
+
+    /// returns the doc url ascociated with the error
+    fn error_url(&self) -> String {
+        self.error_code().url()
+    }
+
+    /// returns error name, used as error code
+    fn error_name(&self) -> String {
+        self.error_code().name()
+    }
+
+    /// return the error type
+    fn error_type(&self) -> String {
+        self.error_code().type_()
+    }
+}
+
+#[allow(clippy::enum_variant_names)]
+enum ErrorType {
+    InternalError,
+    InvalidRequestError,
+    AuthenticationError,
+}
+
+impl fmt::Display for ErrorType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use ErrorType::*;
+
+        match self {
+            InternalError => write!(f, "internal_error"),
+            InvalidRequestError => write!(f, "invalid_request_error"),
+            AuthenticationError => write!(f, "authentication_error"),
+        }
+    }
+}
+
+pub enum Code {
+    // index related error
+    CreateIndex,
+    IndexAlreadyExists,
+    IndexNotFound,
+    InvalidIndexUid,
+    OpenIndex,
+
+    // invalid state error
+    InvalidState,
+    MissingPrimaryKey,
+    PrimaryKeyAlreadyPresent,
+
+    MaxFieldsLimitExceeded,
+    MissingDocumentId,
+
+    Facet,
+    Filter,
+
+    BadParameter,
+    BadRequest,
+    DocumentNotFound,
+    Internal,
+    InvalidToken,
+    MissingAuthorizationHeader,
+    NotFound,
+    PayloadTooLarge,
+    RetrieveDocument,
+    SearchDocuments,
+    UnsupportedMediaType,
+
+    DumpAlreadyInProgress,
+    DumpProcessFailed,
+}
+
+impl Code {
+
+    /// ascociate a `Code` variant to the actual ErrCode
+    fn err_code(&self) -> ErrCode {
+        use Code::*;
+
+        match self {
+            // index related errors
+            // create index is thrown on internal error while creating an index.
+            CreateIndex => ErrCode::internal("index_creation_failed", StatusCode::BAD_REQUEST),
+            IndexAlreadyExists => ErrCode::invalid("index_already_exists", StatusCode::BAD_REQUEST),
+            // thrown when requesting an unexisting index
+            IndexNotFound => ErrCode::invalid("index_not_found", StatusCode::NOT_FOUND),
+            InvalidIndexUid => ErrCode::invalid("invalid_index_uid", StatusCode::BAD_REQUEST),
+            OpenIndex => ErrCode::internal("index_not_accessible", StatusCode::INTERNAL_SERVER_ERROR),
+
+            // invalid state error
+            InvalidState => ErrCode::internal("invalid_state", StatusCode::INTERNAL_SERVER_ERROR),
+            // thrown when no primary key has been set
+            MissingPrimaryKey => ErrCode::invalid("missing_primary_key", StatusCode::BAD_REQUEST),
+            // error thrown when trying to set an already existing primary key
+            PrimaryKeyAlreadyPresent => ErrCode::invalid("primary_key_already_present", StatusCode::BAD_REQUEST),
+
+            // invalid document
+            MaxFieldsLimitExceeded => ErrCode::invalid("max_fields_limit_exceeded", StatusCode::BAD_REQUEST),
+            MissingDocumentId => ErrCode::invalid("missing_document_id", StatusCode::BAD_REQUEST),
+
+            // error related to facets
+            Facet => ErrCode::invalid("invalid_facet", StatusCode::BAD_REQUEST),
+            // error related to filters
+            Filter => ErrCode::invalid("invalid_filter", StatusCode::BAD_REQUEST),
+
+            BadParameter => ErrCode::invalid("bad_parameter", StatusCode::BAD_REQUEST),
+            BadRequest => ErrCode::invalid("bad_request", StatusCode::BAD_REQUEST),
+            DocumentNotFound => ErrCode::invalid("document_not_found", StatusCode::NOT_FOUND),
+            Internal => ErrCode::internal("internal", StatusCode::INTERNAL_SERVER_ERROR),
+            InvalidToken => ErrCode::authentication("invalid_token", StatusCode::FORBIDDEN),
+            MissingAuthorizationHeader => ErrCode::authentication("missing_authorization_header", StatusCode::UNAUTHORIZED),
+            NotFound => ErrCode::invalid("not_found", StatusCode::NOT_FOUND),
+            PayloadTooLarge => ErrCode::invalid("payload_too_large", StatusCode::PAYLOAD_TOO_LARGE),
+            RetrieveDocument => ErrCode::internal("unretrievable_document", StatusCode::BAD_REQUEST),
+            SearchDocuments => ErrCode::internal("search_error", StatusCode::BAD_REQUEST),
+            UnsupportedMediaType => ErrCode::invalid("unsupported_media_type", StatusCode::UNSUPPORTED_MEDIA_TYPE),
+
+            // error related to dump
+            DumpAlreadyInProgress => ErrCode::invalid("dump_already_in_progress", StatusCode::CONFLICT),
+            DumpProcessFailed => ErrCode::internal("dump_process_failed", StatusCode::INTERNAL_SERVER_ERROR),
+        }
+    }
+
+    /// return the HTTP status code ascociated with the `Code`
+    fn http(&self) -> StatusCode {
+        self.err_code().status_code
+    }
+
+    /// return error name, used as error code
+    fn name(&self) -> String {
+        self.err_code().error_name.to_string()
+    }
+
+    /// return the error type
+    fn type_(&self) -> String {
+        self.err_code().error_type.to_string()
+    }
+
+    /// return the doc url ascociated with the error
+    fn url(&self) -> String {
+        format!("https://docs.meilisearch.com/errors#{}", self.name())
+    }
+}
+
+/// Internal structure providing a convenient way to create error codes
+struct ErrCode {
+    status_code: StatusCode,
+    error_type: ErrorType,
+    error_name: &'static str,
+}
+
+impl ErrCode {
+    fn authentication(error_name: &'static str, status_code: StatusCode) -> ErrCode {
+        ErrCode {
+            status_code,
+            error_name,
+            error_type: ErrorType::AuthenticationError,
+        }
+    }
+
+    fn internal(error_name: &'static str, status_code: StatusCode) -> ErrCode {
+        ErrCode {
+            status_code,
+            error_name,
+            error_type: ErrorType::InternalError,
+        }
+    }
+
+    fn invalid(error_name: &'static str, status_code: StatusCode) -> ErrCode {
+        ErrCode {
+            status_code,
+            error_name,
+            error_type: ErrorType::InvalidRequestError,
+        }
+    }
+}

meilisearch-http/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "meilisearch-http"
 description = "MeiliSearch HTTP server"
-version = "0.8.4"
+version = "0.19.0"
 license = "MIT"
 authors = [
     "Quentin de Quelen <quentin@dequelen.me>",
@@ -13,54 +13,74 @@ edition = "2018"
 name = "meilisearch"
 path = "src/main.rs"
 
+[features]
+default = ["sentry"]
+
 [dependencies]
-bincode = "1.2.0"
-chrono = { version = "0.4.9", features = ["serde"] }
-crossbeam-channel = "0.4.0"
-env_logger = "0.7.1"
-heed = "0.6.1"
-http = "0.1.19"
-indexmap = { version = "1.3.0", features = ["serde-1"] }
-log = "0.4.8"
-main_error = "0.1.0"
-meilisearch-core = { path = "../meilisearch-core", version = "0.8.4" }
-meilisearch-schema = { path = "../meilisearch-schema", version = "0.8.4" }
-pretty-bytes = "0.2.2"
-rand = "0.7.2"
-rayon = "1.2.0"
-serde = { version = "1.0.101", features = ["derive"] }
-serde_json = { version = "1.0.41", features = ["preserve_order"] }
-serde_qs = "0.5.1"
-siphasher = "0.3.1"
-structopt = "0.3.3"
-sysinfo = "0.9.5"
-ureq = { version = "0.11.2", features = ["tls"], default-features = false }
-walkdir = "2.2.9"
-whoami = "0.6"
+actix-cors = "0.5.4"
+actix-http = "2.2.0"
+actix-rt = "1.1.1"
+actix-service = "1.0.6"
+actix-web = { version = "3.3.2", features = ["rustls"] }
+bytes = "1.0.0"
+chrono = { version = "0.4.19", features = ["serde"] }
+crossbeam-channel = "0.5.0"
+env_logger = "0.8.2"
+flate2 = "1.0.19"
+futures = "0.3.8"
+http = "0.2.2"
+indexmap = { version = "1.6.1", features = ["serde-1"] }
+log = "0.4.11"
+main_error = "0.1.1"
+meilisearch-core = { path = "../meilisearch-core", version = "0.19.0" }
+meilisearch-error = { path = "../meilisearch-error", version = "0.19.0" }
+meilisearch-schema = { path = "../meilisearch-schema", version = "0.19.0" }
+mime = "0.3.16"
+once_cell = "1.5.2"
+rand = "0.8.1"
+regex = "1.4.2"
+rustls = "0.18.0"
+serde = { version = "1.0.118", features = ["derive"] }
+serde_json = { version = "1.0.61", features = ["preserve_order"] }
+serde_qs = "0.8.2"
+sha2 = "0.9.2"
+siphasher = "0.3.3"
+slice-group-by = "0.2.6"
+structopt = "0.3.21"
+tar = "0.4.30"
+tempfile = "3.1.0"
+tokio = { version = "0.2", features = ["macros"] }
+ureq = { version = "2.0.0", features = ["tls"], default-features = false }
+uuid = "0.8"
+walkdir = "2.3.1"
+whoami = "1.0.3"
 
-[dependencies.async-compression]
+[dependencies.sentry]
+version = "0.18.1"
 default-features = false
-features = ["stream", "gzip", "zlib", "brotli", "zstd"]
-version = "=0.1.0-alpha.7"
+features = [
+    "with_client_implementation",
+    "with_panic",
+    "with_failure",
+    "with_device_info",
+    "with_rust_info",
+    "with_reqwest_transport",
+    "with_rustls",
+    "with_env_logger"
+]
+optional = true
 
-[dependencies.tide]
-git = "https://github.com/rustasync/tide"
-rev = "e77709370bb24cf776fe6da902467c35131535b1"
+[dev-dependencies]
+serde_url_params = "0.2.0"
+tempdir = "0.3.7"
+tokio = { version = "0.2", features = ["macros", "time"] }
 
-[dependencies.tide-log]
-git = "https://github.com/rustasync/tide"
-rev = "e77709370bb24cf776fe6da902467c35131535b1"
-
-[dependencies.tide-slog]
-git = "https://github.com/rustasync/tide"
-rev = "e77709370bb24cf776fe6da902467c35131535b1"
-
-[dependencies.tide-compression]
-git = "https://github.com/rustasync/tide"
-rev = "e77709370bb24cf776fe6da902467c35131535b1"
+[dev-dependencies.assert-json-diff]
+git = "https://github.com/qdequele/assert-json-diff"
+branch = "master"
 
 [build-dependencies]
-vergen = "3.0.4"
+vergen = "3.1.0"
 
 [target.'cfg(unix)'.dependencies]
 jemallocator = "0.3.2"

meilisearch-http/public/interface.html

@@ -3,7 +3,7 @@
   <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <link rel="stylesheet" href="/bulma.min.css">
+    <link rel="stylesheet" href="bulma.min.css">
     <title>MeiliSearch</title>
     <style>
       em {
@@ -28,8 +28,6 @@
       }
 
       .document {
-        padding: 20px 20px;
-        background-color: #f5f5f5;
         border-radius: 4px;
         margin-bottom: 20px;
         display: flex;
@@ -40,120 +38,182 @@
         max-width: 75%;
         padding: 0;
         margin: 0;
+        list-style-type: none;
       }
 
+      .document ol li {
+        list-style: none;
+      }
+
+
       .document .image {
-        max-width: 25%;
-        flex: 0 0 25%;
-        padding-left: 30px;
+        max-width: 50%;
+        margin: 0 auto;
         box-sizing: border-box;
       }
 
+      @media screen and (min-width: 770px) {
+        .document .image {
+          max-width: 25%;
+          flex: 0 0 25%;
+          margin: 0;
+          padding-left: 30px;
+          box-sizing: border-box;
+        }
+      }
+
       .document .image img {
         width: 100%;
       }
 
-      .field {
-        list-style-type: none;
-        display: flex;
-        flex-wrap: wrap;
-      }
-
-      .field:not(:last-child) {
-        margin-bottom: 7px;
-      }
-
       .attribute {
-        flex: 0 0 25%;
-        max-width: 25%;
-        text-align: right;
-        padding-right: 10px;
+        text-align: center;
         box-sizing: border-box;
         text-transform: uppercase;
+        font-weight: bold;
         color: rgba(0,0,0,.7);
       }
 
+      @media screen and (min-width: 770px) {
+        .attribute {
+          flex: 0 0 25%;
+          max-width: 25%;
+          text-align: right;
+          padding-right: 10px;
+          font-weight: normal;
+          box-sizing: border-box;
+        }
+      }
+      @media screen and (max-width: 770px) {
+        .attribute {
+          padding-bottom: 0;
+        }
+      }
+
       .content {
-        max-width: 75%;
         flex: 0 0 75%;
         box-sizing: border-box;
-        padding-left: 10px;
         color: rgba(0,0,0,.9);
+        overflow-wrap: anywhere;
+      }
+
+      .hero-foot {
+        padding-bottom: 3rem;
+      }
+
+      @media screen and (max-width: 770px) {
+        .align-on-mobile {
+          text-align: center;
+        }
       }
     </style>
   </head>
   <body>
 
     <section class="hero is-light">
+      
       <div class="hero-body">
         <div class="container">
-          <h1 class="title">
-            Welcome to MeiliSearch
-          </h1>
-          <h2 class="subtitle">
-            This dashboard will help you check the search results with ease.
-          </h2>
+          <div class="content is-medium align-on-mobile">
+            <h1 class="title is-1 is-spaced">
+              Welcome to MeiliSearch
+            </h1>
+            <p class="subtitle is-4">
+              This dashboard will help you check the search results with ease.
+            </p>
+          </div>
+          <div class="columns">
+            <div class="column is-4">
+              <div class="field">
+                  <!-- API Key -->
+                    <label class="label" for="apiKey">API Key (optional)</label>
+                    <div class="control">
+                      <input id="apiKey" class="input is-small" type="password" placeholder="Enter your API key">
+                    </div>
+                    <p class="help">At least a private API key is required for the dashboard to access the indexes list.</p>
+              </div>
+            </div>
+          </div>
+          <div class="columns">
+            <div class="column is-8">
+                <label class="label" for="search">Search something</label>
+                <div class="field has-addons">
+                  <div class="control">
+                    <span class="select">
+                      <select role="listbox" id="index" aria-label="Select the index you want to search on">
+                        <!-- indexes names -->
+                      </select>
+                    </span>
+                  </div>
+                  <div class="control is-expanded">
+                    <input id="search" class="input" type="search" autofocus placeholder="e.g. George Clooney" aria-label="Search through your documents">
+                  </div>
+                </div>
+            </div>
+            <div class="column is-4">
+              <div class="columns">
+                <div class="column is-6 has-text-centered">
+                  <p class="heading">Documents</p>
+                  <p id="count" class="title">0</p>
+                </div>
+                <div class="column is-6 has-text-centered">
+                  <p class="heading">Time Spent</p>
+                  <p id="time" class="title">N/A</p>
+                </div>
+              </div>
+            </div>
+          </div>
         </div>
       </div>
     </section>
 
-    <section class="hero container">
-        <div class="notification" style="border-radius: 0 0 4px 4px;">
-
-          <nav class="level">
-            <!-- Left side -->
-            <div class="level-left">
-              <div class="level-item">
-                <div class="field has-addons has-addons-right">
-                  <p class="control">
-                    <span class="select">
-                      <select id="index">
-                        <!-- indexes names -->
-                      </select>
-                    </span>
-                  </p>
-                  <p class="control">
-                    <input id="search" class="input" type="text" autofocus placeholder="e.g. George Clooney">
-                  </p>
-                </div>
-              </div>
-            </div>
-
-            <!-- Right side -->
-            <nav class="level-right">
-              <div class="level-item has-text-centered">
-                <div>
-                  <p class="heading">Documents</p>
-                  <p id="count" class="title">25</p>
-                </div>
-              </div>
-              <div class="level-item has-text-centered">
-                <div>
-                  <p class="heading">Time Spent</p>
-                  <p id="time" class="title">4ms</p>
-                </div>
-              </div>
-            </nav>
-          </nav>
-
-        </div>
-    </section>
-
     <section>
-      <ol id="results" class="content">
-        <!-- documents matching resquests -->
-      </ol>
+      <div class="container">
+        <ol id="results" class="content">
+          <!-- documents matching resquests -->
+        </ol>
+      </div>
     </section>
   </body>
 
   <script>
-    function httpGet(theUrl) {
+    function sanitizeHTMLEntities(str) {
+        if (str && typeof str === 'string') {
+            str = str.replace(/</g,"&lt;");
+            str = str.replace(/>/g,"&gt;");
+            str = str.replace(/&lt;em&gt;/g,"<em>");
+            str = str.replace(/&lt;\/em&gt;/g,"<\/em>");
+        }
+        return str;
+    }
+
+    function httpGet(theUrl, apiKey) {
         var xmlHttp = new XMLHttpRequest();
         xmlHttp.open("GET", theUrl, false); // false for synchronous request
+        if (apiKey) {
+          xmlHttp.setRequestHeader("x-Meili-API-Key", apiKey);
+        }
         xmlHttp.send(null);
         return xmlHttp.responseText;
     }
 
+    function refreshIndexList() {
+        // TODO we must not block here
+        let result = JSON.parse(httpGet(`${baseUrl}/indexes`, localStorage.getItem('apiKey')));
+
+        if (!Array.isArray(result)) { return }
+
+        let select = document.getElementById("index");
+        select.innerHTML = '';
+
+        for (index of result) {
+            const option = document.createElement('option');
+            option.value = index.uid;
+            option.innerHTML = index.name;
+            select.appendChild(option);
+        }
+    }
+
     let lastRequest = undefined;
 
     function triggerSearch() {
@@ -161,19 +221,25 @@
         if (e.selectedIndex == -1) { return }
         var index = e.options[e.selectedIndex].value;
 
-        let theUrl = `${baseUrl}/indexes/${index}/search?q=${search.value}&attributesToHighlight=*`;
+        let theUrl = `${baseUrl}/indexes/${index}/search?q=${encodeURIComponent(search.value)}&attributesToHighlight=*`;
 
         if (lastRequest) { lastRequest.abort() }
         lastRequest = new XMLHttpRequest();
 
         lastRequest.open("GET", theUrl, true);
+
+        if (localStorage.getItem('apiKey')) {
+          lastRequest.setRequestHeader("x-Meili-API-Key", localStorage.getItem('apiKey'));
+        }
+
         lastRequest.onload = function (e) {
             if (lastRequest.readyState === 4 && lastRequest.status === 200) {
-                let httpResults = JSON.parse(lastRequest.responseText);
+                let sanitizedResponseText = sanitizeHTMLEntities(lastRequest.responseText);
+                let httpResults = JSON.parse(sanitizedResponseText);
                 results.innerHTML = '';
 
                 let processingTimeMs = httpResults.processingTimeMs;
-                let numberOfDocuments = httpResults.hits.length;
+                let numberOfDocuments = httpResults.nbHits;
                 time.innerHTML = `${processingTimeMs}ms`;
                 count.innerHTML = `${numberOfDocuments}`;
 
@@ -182,9 +248,12 @@
                     delete element._formatted;
 
                     const elem = document.createElement('li');
-                    elem.classList.add("document");
+                    elem.classList.add("document","box");
 
-                    const ol = document.createElement('ol');
+                    const div = document.createElement('div');
+                    div.classList.add("columns","is-desktop","is-tablet");
+                    const info = document.createElement('div');
+                    info.classList.add("column","align-on-mobile");
                     let image = undefined;
 
                     for (const prop in element) {
@@ -195,33 +264,41 @@
                             }
                         }
 
-                        const field = document.createElement('li');
-                        field.classList.add("field");
+                        const field = document.createElement('div');
+                        field.classList.add("columns");
 
                         const attribute = document.createElement('div');
-                        attribute.classList.add("attribute");
+                        attribute.classList.add("attribute", "column");
                         attribute.innerHTML = prop;
 
                         const content = document.createElement('div');
-                        content.classList.add("content");
-                        content.innerHTML = element[prop];
+                        content.classList.add("content", "column");
+                        
+                        if (typeof (element[prop]) === "object") {
+                          content.innerHTML = JSON.stringify(element[prop]);
+                        } else {
+                          content.innerHTML = element[prop];
+                        }
 
                         field.appendChild(attribute);
                         field.appendChild(content);
 
-                        ol.appendChild(field);
+                        info.appendChild(field);
                     }
 
-                    elem.appendChild(ol);
+                    div.appendChild(info);
+                    elem.appendChild(div);
 
                     if (image != undefined) {
-                        const div = document.createElement('div');
-                        div.classList.add("image");
-
+                        const divImage = document.createElement('div');
+                        divImage.classList.add("image","column","align-on-mobile");
+                        
                         const img = document.createElement('img');
                         img.src = image;
+                        img.setAttribute("alt","Item illustration");
 
-                        div.appendChild(img);
+                        divImage.appendChild(img);
+                        div.appendChild(divImage);
                         elem.appendChild(div);
                     }
 
@@ -234,19 +311,21 @@
         lastRequest.send(null);
     }
 
-    let baseUrl = window.location.origin;
-    // TODO we must not block here
-    let result = JSON.parse(httpGet(`${baseUrl}/indexes`));
-
-    let select = document.getElementById("index");
-    for (index of result) {
-        const option = document.createElement('option');
-        option.value = index.uid;
-        option.innerHTML = index.name;
-        select.appendChild(option);
+    if (!apiKey.value) {
+        apiKey.value = localStorage.getItem('apiKey');
     }
 
+    apiKey.addEventListener('input', function(e) {
+        localStorage.setItem('apiKey', apiKey.value);
+        refreshIndexList();
+    }, false);
+
+    let baseUrl = window.location.origin;
+    refreshIndexList();
+
     search.oninput = triggerSearch;
+    
+    let select = document.getElementById("index");
     select.onchange = triggerSearch;
 
     triggerSearch();

meilisearch-http/src/analytics.rs

@@ -1,20 +1,72 @@
 use std::hash::{Hash, Hasher};
-use std::thread;
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use std::{error, thread};
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
 
 use log::error;
 use serde::Serialize;
 use serde_qs as qs;
 use siphasher::sip::SipHasher;
+use walkdir::WalkDir;
+
+use crate::Data;
+use crate::Opt;
 
 const AMPLITUDE_API_KEY: &str = "f7fba398780e06d8fe6666a9be7e3d47";
 
+#[derive(Debug, Serialize)]
+struct EventProperties {
+    database_size: u64,
+    last_update_timestamp: Option<i64>, //timestamp
+    number_of_documents: Vec<u64>,
+}
+
+impl EventProperties {
+    fn from(data: Data) -> Result<EventProperties, Box<dyn error::Error>> {
+        let mut index_list = Vec::new();
+
+        let reader = data.db.main_read_txn()?;
+
+        for index_uid in data.db.indexes_uids() {
+            if let Some(index) = data.db.open_index(&index_uid) {
+                let number_of_documents = index.main.number_of_documents(&reader)?;
+                index_list.push(number_of_documents);
+            }
+        }
+
+        let database_size = WalkDir::new(&data.db_path)
+            .into_iter()
+            .filter_map(|entry| entry.ok())
+            .filter_map(|entry| entry.metadata().ok())
+            .filter(|metadata| metadata.is_file())
+            .fold(0, |acc, m| acc + m.len());
+
+        let last_update_timestamp = data.db.last_update(&reader)?.map(|u| u.timestamp());
+
+        Ok(EventProperties {
+            database_size,
+            last_update_timestamp,
+            number_of_documents: index_list,
+        })
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct UserProperties<'a> {
+    env: &'a str,
+    start_since_days: u64,
+    user_email: Option<String>,
+    server_provider: Option<String>,
+}
+
 #[derive(Debug, Serialize)]
 struct Event<'a> {
     user_id: &'a str,
     event_type: &'a str,
     device_id: &'a str,
     time: u64,
+    app_version: &'a str,
+    user_properties: UserProperties<'a>,
+    event_properties: Option<EventProperties>,
 }
 
 #[derive(Debug, Serialize)]
@@ -23,7 +75,7 @@ struct AmplitudeRequest<'a> {
     event: &'a str,
 }
 
-pub fn analytics_sender() {
+pub fn analytics_sender(data: Data, opt: Opt) {
     let username = whoami::username();
     let hostname = whoami::hostname();
     let platform = whoami::platform();
@@ -36,6 +88,7 @@ pub fn analytics_sender() {
 
     let uid = format!("{:X}", hash);
     let platform = platform.to_string();
+    let first_start = Instant::now();
 
     loop {
         let n = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
@@ -43,12 +96,27 @@ pub fn analytics_sender() {
         let device_id = &platform;
         let time = n.as_secs();
         let event_type = "runtime_tick";
+        let elapsed_since_start = first_start.elapsed().as_secs() / 86_400; // One day
+        let event_properties = EventProperties::from(data.clone()).ok();
+        let app_version = env!("CARGO_PKG_VERSION").to_string();
+        let app_version = app_version.as_str();
+        let user_email = std::env::var("MEILI_USER_EMAIL").ok();
+        let server_provider = std::env::var("MEILI_SERVER_PROVIDER").ok();
+        let user_properties = UserProperties {
+            env: &opt.env,
+            start_since_days: elapsed_since_start,
+            user_email,
+            server_provider,
+        };
 
         let event = Event {
             user_id,
             event_type,
             device_id,
             time,
+            app_version,
+            user_properties,
+            event_properties
         };
         let event = serde_json::to_string(&event).unwrap();
 
@@ -59,11 +127,16 @@ pub fn analytics_sender() {
 
         let body = qs::to_string(&request).unwrap();
         let response = ureq::post("https://api.amplitude.com/httpapi").send_string(&body);
-        if !response.ok() {
-            let body = response.into_string().unwrap();
-            error!("Unsuccessful call to Amplitude: {}", body);
+        match response {
+            Err(ureq::Error::Status(_ , response)) => {
+                error!("Unsuccessful call to Amplitude: {}", response.into_string().unwrap_or_default());
+            }
+            Err(e) => {
+                error!("Unsuccessful call to Amplitude: {}", e);
+            }
+            _ => (),
         }
 
-        thread::sleep(Duration::from_secs(86_400)) // one day
+        thread::sleep(Duration::from_secs(3600)) // one hour
     }
 }

meilisearch-http/src/data.rs

@@ -1,19 +1,15 @@
-use std::collections::HashMap;
+use std::error::Error;
 use std::ops::Deref;
-use std::sync::Arc;
+use std::path::PathBuf;
+use std::sync::{Arc, Mutex};
 
-use chrono::{DateTime, Utc};
-use heed::types::{SerdeBincode, Str};
-use log::error;
-use meilisearch_core::{Database, MainT, UpdateT, Error as MError, MResult};
-use sysinfo::Pid;
+use meilisearch_core::{Database, DatabaseOptions, Index};
+use sha2::Digest;
 
+use crate::error::{Error as MSError, ResponseError};
+use crate::index_update_callback;
 use crate::option::Opt;
-use crate::routes::index::index_update_callback;
-
-const LAST_UPDATE_KEY: &str = "last-update";
-
-type SerdeDatetime = SerdeBincode<DateTime<Utc>>;
+use crate::dump::DumpInfo;
 
 #[derive(Clone)]
 pub struct Data {
@@ -32,87 +28,73 @@ impl Deref for Data {
 pub struct DataInner {
     pub db: Arc<Database>,
     pub db_path: String,
-    pub api_key: Option<String>,
-    pub server_pid: Pid,
+    pub dumps_dir: PathBuf,
+    pub dump_batch_size: usize,
+    pub api_keys: ApiKeys,
+    pub server_pid: u32,
+    pub http_payload_size_limit: usize,
+    pub current_dump: Arc<Mutex<Option<DumpInfo>>>,
 }
 
-impl DataInner {
-    pub fn is_indexing(&self, reader: &heed::RoTxn<UpdateT>, index: &str) -> MResult<Option<bool>> {
-        match self.db.open_index(&index) {
-            Some(index) => index.current_update_id(&reader).map(|u| Some(u.is_some())),
-            None => Ok(None),
-        }
-    }
+#[derive(Clone)]
+pub struct ApiKeys {
+    pub public: Option<String>,
+    pub private: Option<String>,
+    pub master: Option<String>,
+}
 
-    pub fn last_update(&self, reader: &heed::RoTxn<MainT>) -> MResult<Option<DateTime<Utc>>> {
-        match self
-            .db
-            .common_store()
-            .get::<_, Str, SerdeDatetime>(reader, LAST_UPDATE_KEY)?
-        {
-            Some(datetime) => Ok(Some(datetime)),
-            None => Ok(None),
-        }
-    }
-
-    pub fn set_last_update(&self, writer: &mut heed::RwTxn<MainT>) -> MResult<()> {
-        self.db
-            .common_store()
-            .put::<_, Str, SerdeDatetime>(writer, LAST_UPDATE_KEY, &Utc::now())
-            .map_err(Into::into)
-    }
-
-    pub fn compute_stats(&self, writer: &mut heed::RwTxn<MainT>, index_uid: &str) -> MResult<()> {
-        let index = match self.db.open_index(&index_uid) {
-            Some(index) => index,
-            None => {
-                error!("Impossible to retrieve index {}", index_uid);
-                return Ok(());
+impl ApiKeys {
+    pub fn generate_missing_api_keys(&mut self) {
+        if let Some(master_key) = &self.master {
+            if self.private.is_none() {
+                let key = format!("{}-private", master_key);
+                let sha = sha2::Sha256::digest(key.as_bytes());
+                self.private = Some(format!("{:x}", sha));
+            }
+            if self.public.is_none() {
+                let key = format!("{}-public", master_key);
+                let sha = sha2::Sha256::digest(key.as_bytes());
+                self.public = Some(format!("{:x}", sha));
             }
-        };
-
-        let schema = match index.main.schema(&writer)? {
-            Some(schema) => schema,
-            None => return Ok(()),
-        };
-
-        let all_documents_fields = index
-            .documents_fields_counts
-            .all_documents_fields_counts(&writer)?;
-
-        // count fields frequencies
-        let mut fields_frequency = HashMap::<_, usize>::new();
-        for result in all_documents_fields {
-            let (_, attr, _) = result?;
-            *fields_frequency.entry(attr).or_default() += 1;
         }
-
-        // convert attributes to their names
-        let frequency: HashMap<_, _> = fields_frequency
-            .into_iter()
-            .map(|(a, c)| (schema.attribute_name(a).to_owned(), c))
-            .collect();
-
-        index
-            .main
-            .put_fields_frequency(writer, &frequency)
-            .map_err(MError::Zlmdb)
     }
 }
 
 impl Data {
-    pub fn new(opt: Opt) -> Data {
+    pub fn new(opt: Opt) -> Result<Data, Box<dyn Error>> {
         let db_path = opt.db_path.clone();
-        let api_key = opt.api_key.clone();
-        let server_pid = sysinfo::get_current_pid().unwrap();
+        let dumps_dir = opt.dumps_dir.clone();
+        let dump_batch_size = opt.dump_batch_size;
+        let server_pid = std::process::id();
 
-        let db = Arc::new(Database::open_or_create(opt.db_path.clone()).unwrap());
+        let db_opt = DatabaseOptions {
+            main_map_size: opt.max_mdb_size,
+            update_map_size: opt.max_udb_size,
+        };
+
+        let http_payload_size_limit = opt.http_payload_size_limit;
+
+        let db = Arc::new(Database::open_or_create(opt.db_path, db_opt)?);
+
+        let mut api_keys = ApiKeys {
+            master: opt.master_key,
+            private: None,
+            public: None,
+        };
+
+        api_keys.generate_missing_api_keys();
+
+        let current_dump = Arc::new(Mutex::new(None));
 
         let inner_data = DataInner {
             db: db.clone(),
             db_path,
-            api_key,
+            dumps_dir,
+            dump_batch_size,
+            api_keys,
             server_pid,
+            http_payload_size_limit,
+            current_dump,
         };
 
         let data = Data {
@@ -124,6 +106,70 @@ impl Data {
             index_update_callback(&index_uid, &callback_context, status);
         }));
 
-        data
+        Ok(data)
+    }
+
+    fn create_index(&self, uid: &str) -> Result<Index, ResponseError> {
+        if !uid
+            .chars()
+            .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_')
+        {
+            return Err(MSError::InvalidIndexUid.into());
+        }
+
+        let created_index = self.db.create_index(&uid).map_err(|e| match e {
+            meilisearch_core::Error::IndexAlreadyExists => e.into(),
+            _ => ResponseError::from(MSError::create_index(e)),
+        })?;
+
+        self.db.main_write::<_, _, ResponseError>(|mut writer| {
+            created_index.main.put_name(&mut writer, uid)?;
+
+            created_index
+                .main
+                .created_at(&writer)?
+                .ok_or(MSError::internal("Impossible to read created at"))?;
+
+            created_index
+                .main
+                .updated_at(&writer)?
+                .ok_or(MSError::internal("Impossible to read updated at"))?;
+            Ok(())
+        })?;
+
+        Ok(created_index)
+    }
+
+    pub fn get_current_dump_info(&self) -> Option<DumpInfo> {
+        self.current_dump.lock().unwrap().clone()
+    }
+
+    pub fn set_current_dump_info(&self, dump_info: DumpInfo) {
+        self.current_dump.lock().unwrap().replace(dump_info);
+    }
+
+    pub fn get_or_create_index<F, R>(&self, uid: &str, f: F) -> Result<R, ResponseError>
+    where
+        F: FnOnce(&Index) -> Result<R, ResponseError>,
+    {
+        let mut index_has_been_created = false;
+
+        let index = match self.db.open_index(&uid) {
+            Some(index) => index,
+            None => {
+                index_has_been_created = true;
+                self.create_index(&uid)?
+            }
+        };
+
+        match f(&index) {
+            Ok(r) => Ok(r),
+            Err(err) => {
+                if index_has_been_created {
+                    let _ = self.db.delete_index(&uid);
+                }
+                Err(err)
+            }
+        }
     }
 }

meilisearch-http/src/dump.rs

@@ -0,0 +1,412 @@
+use std::fs::{create_dir_all, File};
+use std::io::prelude::*;
+use std::path::{Path, PathBuf};
+use std::thread;
+
+use actix_web::web;
+use chrono::offset::Utc;
+use indexmap::IndexMap;
+use log::{error, info};
+use meilisearch_core::{MainWriter, MainReader, UpdateReader};
+use meilisearch_core::settings::Settings;
+use meilisearch_core::update::{apply_settings_update, apply_documents_addition};
+use serde::{Deserialize, Serialize};
+use serde_json::json;
+use tempfile::TempDir;
+
+use crate::Data;
+use crate::error::{Error, ResponseError};
+use crate::helpers::compression;
+use crate::routes::index;
+use crate::routes::index::IndexResponse;
+
+#[derive(Debug, Serialize, Deserialize, Copy, Clone)]
+enum DumpVersion {
+    V1,
+}
+
+impl DumpVersion {
+    const CURRENT: Self = Self::V1;
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct DumpMetadata {
+    indexes: Vec<crate::routes::index::IndexResponse>,
+    db_version: String,
+    dump_version: DumpVersion,
+}
+
+impl DumpMetadata {
+    /// Create a DumpMetadata with the current dump version of meilisearch.
+    pub fn new(indexes: Vec<crate::routes::index::IndexResponse>, db_version: String) -> Self {
+        DumpMetadata {
+            indexes,
+            db_version,
+            dump_version: DumpVersion::CURRENT,
+        }
+    }
+
+    /// Extract DumpMetadata from `metadata.json` file present at provided `dir_path`
+    fn from_path(dir_path: &Path) -> Result<Self, Error> {
+        let path = dir_path.join("metadata.json");
+        let file = File::open(path)?;
+        let reader = std::io::BufReader::new(file);
+        let metadata = serde_json::from_reader(reader)?;
+
+        Ok(metadata)
+    }
+
+    /// Write DumpMetadata in `metadata.json` file at provided `dir_path`
+    fn to_path(&self, dir_path: &Path) -> Result<(), Error> {
+        let path = dir_path.join("metadata.json");
+        let file = File::create(path)?;
+
+        serde_json::to_writer(file, &self)?;
+
+        Ok(())
+    }
+}
+
+/// Extract Settings from `settings.json` file present at provided `dir_path`
+fn settings_from_path(dir_path: &Path) -> Result<Settings, Error> {
+    let path = dir_path.join("settings.json");
+    let file = File::open(path)?;
+    let reader = std::io::BufReader::new(file);
+    let metadata = serde_json::from_reader(reader)?;
+
+    Ok(metadata)
+}
+
+/// Write Settings in `settings.json` file at provided `dir_path`
+fn settings_to_path(settings: &Settings, dir_path: &Path) -> Result<(), Error> {
+    let path = dir_path.join("settings.json");
+    let file = File::create(path)?;
+
+    serde_json::to_writer(file, settings)?;
+
+    Ok(())
+}
+
+/// Import settings and documents of a dump with version `DumpVersion::V1` in specified index.
+fn import_index_v1(
+    data: &Data,
+    dumps_dir: &Path,
+    index_uid: &str,
+    document_batch_size: usize,
+    write_txn: &mut MainWriter,
+) -> Result<(), Error> {
+
+    // open index
+    let index = data
+        .db
+        .open_index(index_uid)
+        .ok_or(Error::index_not_found(index_uid))?;
+
+    // index dir path in  dump dir
+    let index_path = &dumps_dir.join(index_uid);
+
+    // extract `settings.json` file and import content
+    let settings = settings_from_path(&index_path)?;
+    let settings = settings.to_update().map_err(|e| Error::dump_failed(format!("importing settings for index {}; {}", index_uid, e)))?;
+    apply_settings_update(write_txn, &index, settings)?;
+
+    // create iterator over documents in `documents.jsonl` to make batch importation
+    // create iterator over documents in `documents.jsonl` to make batch importation
+    let documents = {
+        let file = File::open(&index_path.join("documents.jsonl"))?;
+        let reader = std::io::BufReader::new(file);
+        let deserializer = serde_json::Deserializer::from_reader(reader);
+        deserializer.into_iter::<IndexMap<String, serde_json::Value>>()
+    };
+
+    // batch import document every `document_batch_size`:
+    // create a Vec to bufferize documents
+    let mut values = Vec::with_capacity(document_batch_size);
+    // iterate over documents
+    for document in documents {
+        // push document in buffer
+        values.push(document?);
+        // if buffer is full, create and apply a batch, and clean buffer
+        if values.len() == document_batch_size {
+            let batch = std::mem::replace(&mut values, Vec::with_capacity(document_batch_size));
+            apply_documents_addition(write_txn, &index, batch, None)?;
+        }
+    }
+
+    // apply documents remaining in the buffer
+    if !values.is_empty() {
+        apply_documents_addition(write_txn, &index, values, None)?;
+    }
+
+    // sync index information: stats, updated_at, last_update
+    if let Err(e) = crate::index_update_callback_txn(index, index_uid, data, write_txn) {
+        return Err(Error::Internal(e));
+    }
+
+    Ok(())
+}
+
+/// Import dump from `dump_path` in database.
+pub fn import_dump(
+    data: &Data,
+    dump_path: &Path,
+    document_batch_size: usize,
+) -> Result<(), Error> {
+    info!("Importing dump from {:?}...", dump_path);
+
+    // create a temporary directory
+    let tmp_dir = TempDir::new()?;
+    let tmp_dir_path = tmp_dir.path();
+
+    // extract dump in temporary directory
+    compression::from_tar_gz(dump_path, tmp_dir_path)?;
+
+    // read dump metadata
+    let metadata = DumpMetadata::from_path(&tmp_dir_path)?;
+
+    // choose importation function from DumpVersion of metadata
+    let import_index = match metadata.dump_version {
+        DumpVersion::V1 => import_index_v1,
+    };
+
+    // remove indexes which have same `uid` than indexes to import and create empty indexes
+    let existing_index_uids = data.db.indexes_uids();
+    for index in metadata.indexes.iter() {
+        if existing_index_uids.contains(&index.uid) {
+            data.db.delete_index(index.uid.clone())?;
+        }
+        index::create_index_sync(&data.db, index.uid.clone(), index.name.clone(), index.primary_key.clone())?;
+    }
+
+    // import each indexes content
+    data.db.main_write::<_, _, Error>(|mut writer| {
+        for index in metadata.indexes {
+            import_index(&data, tmp_dir_path, &index.uid, document_batch_size, &mut writer)?;
+        }
+        Ok(())
+    })?;
+
+    info!("Dump importation from {:?} succeed", dump_path);
+    Ok(())
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+#[serde(rename_all = "snake_case")]
+pub enum DumpStatus {
+    Done,
+    InProgress,
+    Failed,
+}
+
+#[derive(Debug, Serialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct DumpInfo {
+    pub uid: String,
+    pub status: DumpStatus,
+    #[serde(skip_serializing_if = "Option::is_none", flatten)]
+    pub error: Option<serde_json::Value>,
+
+}
+
+impl DumpInfo {
+    pub fn new(uid: String, status: DumpStatus) -> Self {
+        Self { uid, status, error: None }
+    }
+
+    pub fn with_error(mut self, error: ResponseError) -> Self {
+        self.status = DumpStatus::Failed;
+        self.error = Some(json!(error));
+
+        self
+    }
+
+    pub fn dump_already_in_progress(&self) -> bool {
+        self.status == DumpStatus::InProgress
+    }
+}
+
+/// Generate uid from creation date
+fn generate_uid() -> String {
+    Utc::now().format("%Y%m%d-%H%M%S%3f").to_string()
+}
+
+/// Infer dumps_dir from dump_uid
+pub fn compressed_dumps_dir(dumps_dir: &Path, dump_uid: &str) -> PathBuf {
+    dumps_dir.join(format!("{}.dump", dump_uid))
+}
+
+/// Write metadata in dump
+fn dump_metadata(data: &web::Data<Data>, dir_path: &Path, indexes: Vec<IndexResponse>) -> Result<(), Error> {
+    let (db_major, db_minor, db_patch) = data.db.version();
+    let metadata = DumpMetadata::new(indexes, format!("{}.{}.{}", db_major, db_minor, db_patch));
+
+    metadata.to_path(dir_path)
+}
+
+/// Export settings of provided index in dump
+fn dump_index_settings(data: &web::Data<Data>, reader: &MainReader, dir_path: &Path, index_uid: &str) -> Result<(), Error> {
+    let settings = crate::routes::setting::get_all_sync(data, reader, index_uid)?;
+
+    settings_to_path(&settings, dir_path)
+}
+
+/// Export updates of provided index in dump
+fn dump_index_updates(data: &web::Data<Data>, reader: &UpdateReader, dir_path: &Path, index_uid: &str) -> Result<(), Error> {
+    let updates_path = dir_path.join("updates.jsonl");
+    let updates = crate::routes::index::get_all_updates_status_sync(data, reader, index_uid)?;
+
+    let file = File::create(updates_path)?;
+
+    for update in updates {
+        serde_json::to_writer(&file, &update)?;
+        writeln!(&file)?;
+    }
+
+    Ok(())
+}
+
+/// Export documents of provided index in dump
+fn dump_index_documents(data: &web::Data<Data>, reader: &MainReader, dir_path: &Path, index_uid: &str) -> Result<(), Error> {
+    let documents_path = dir_path.join("documents.jsonl");
+    let file = File::create(documents_path)?;
+    let dump_batch_size = data.dump_batch_size;
+
+    let mut offset = 0;
+    loop {
+        let documents = crate::routes::document::get_all_documents_sync(data, reader, index_uid, offset, dump_batch_size, None)?;
+        if documents.is_empty() { break; } else { offset += dump_batch_size; }
+
+        for document in documents {
+            serde_json::to_writer(&file, &document)?;
+            writeln!(&file)?;
+        }
+    }
+
+    Ok(())
+}
+
+/// Write error with a context.
+fn fail_dump_process<E: std::error::Error>(data: &web::Data<Data>, dump_info: DumpInfo, context: &str, error: E) {
+        let error_message = format!("{}; {}", context, error);
+        error!("Something went wrong during dump process: {}", &error_message);
+        data.set_current_dump_info(dump_info.with_error(Error::dump_failed(error_message).into()))
+}
+
+/// Main function of dump.
+fn dump_process(data: web::Data<Data>, dumps_dir: PathBuf, dump_info: DumpInfo) {
+    // open read transaction on Update
+    let update_reader = match data.db.update_read_txn() {
+        Ok(r) => r,
+        Err(e) => {
+            fail_dump_process(&data, dump_info, "creating RO transaction on updates", e);
+            return ;
+        }
+    };
+
+    // open read transaction on Main
+    let main_reader = match data.db.main_read_txn() {
+        Ok(r) => r,
+        Err(e) => {
+            fail_dump_process(&data, dump_info, "creating RO transaction on main", e);
+            return ;
+        }
+    };
+
+    // create a temporary directory
+    let tmp_dir = match TempDir::new() {
+        Ok(tmp_dir) => tmp_dir,
+        Err(e) => {
+            fail_dump_process(&data, dump_info, "creating temporary directory", e);
+            return ;
+        }
+    };
+    let tmp_dir_path = tmp_dir.path();
+
+    // fetch indexes
+    let indexes = match crate::routes::index::list_indexes_sync(&data, &main_reader) {
+        Ok(indexes) => indexes,
+        Err(e) => {
+            fail_dump_process(&data, dump_info, "listing indexes", e);
+            return ;
+        }
+    };
+
+    // create metadata
+    if let Err(e) = dump_metadata(&data, &tmp_dir_path, indexes.clone()) {
+        fail_dump_process(&data, dump_info, "generating metadata", e);
+        return ;
+    }
+
+    // export settings, updates and documents for each indexes
+    for index in indexes {
+        let index_path = tmp_dir_path.join(&index.uid);
+
+        // create index sub-dircetory
+        if let Err(e) = create_dir_all(&index_path) {
+            fail_dump_process(&data, dump_info, &format!("creating directory for index {}", &index.uid), e);
+            return ;
+        }
+
+        // export settings
+        if let Err(e) = dump_index_settings(&data, &main_reader, &index_path, &index.uid) {
+            fail_dump_process(&data, dump_info, &format!("generating settings for index {}", &index.uid), e);
+            return ;
+        }
+
+        // export documents
+        if let Err(e) = dump_index_documents(&data, &main_reader, &index_path, &index.uid) {
+            fail_dump_process(&data, dump_info, &format!("generating documents for index {}", &index.uid), e);
+            return ;
+        }
+
+        // export updates
+        if let Err(e) = dump_index_updates(&data, &update_reader, &index_path, &index.uid) {
+            fail_dump_process(&data, dump_info, &format!("generating updates for index {}", &index.uid), e);
+            return ;
+        }
+    }
+
+    // compress dump in a file named `{dump_uid}.dump` in `dumps_dir`
+    if let Err(e) = crate::helpers::compression::to_tar_gz(&tmp_dir_path, &compressed_dumps_dir(&dumps_dir, &dump_info.uid)) {
+        fail_dump_process(&data, dump_info, "compressing dump", e);
+        return ;
+    }
+
+    // update dump info to `done`
+    let resume = DumpInfo::new(
+        dump_info.uid,
+        DumpStatus::Done
+    );
+
+    data.set_current_dump_info(resume);
+}
+
+pub fn init_dump_process(data: &web::Data<Data>, dumps_dir: &Path) -> Result<DumpInfo, Error> {
+    create_dir_all(dumps_dir).map_err(|e| Error::dump_failed(format!("creating temporary directory {}", e)))?;
+
+    // check if a dump is already in progress
+    if let Some(resume) = data.get_current_dump_info() {
+        if resume.dump_already_in_progress() {
+            return Err(Error::dump_conflict())
+        }
+    }
+
+    // generate a new dump info
+    let info = DumpInfo::new(
+        generate_uid(),
+        DumpStatus::InProgress
+    );
+
+    data.set_current_dump_info(info.clone());
+
+    let data = data.clone();
+    let dumps_dir = dumps_dir.to_path_buf();
+    let info_cloned = info.clone();
+    // run dump process in a new thread
+    thread::spawn(move ||
+        dump_process(data, dumps_dir, info_cloned)
+    );
+
+    Ok(info)
+}

meilisearch-http/src/error.rs

@@ -1,126 +1,307 @@
-use std::fmt::Display;
+use std::error;
+use std::fmt;
 
-use http::status::StatusCode;
-use log::{error, warn};
-use serde::{Deserialize, Serialize};
-use tide::response::IntoResponse;
-use tide::Response;
+use actix_http::ResponseBuilder;
+use actix_web as aweb;
+use actix_web::error::{JsonPayloadError, QueryPayloadError};
+use actix_web::http::StatusCode;
+use serde::ser::{Serialize, Serializer, SerializeStruct};
 
-pub type SResult<T> = Result<T, ResponseError>;
+use meilisearch_error::{ErrorCode, Code};
 
-pub enum ResponseError {
-    Internal(String),
-    BadRequest(String),
-    InvalidToken(String),
-    NotFound(String),
-    IndexNotFound(String),
-    DocumentNotFound(String),
-    MissingHeader(String),
+#[derive(Debug)]
+pub struct ResponseError {
+    inner: Box<dyn ErrorCode>,
+}
+
+impl error::Error for ResponseError {}
+
+impl ErrorCode for ResponseError {
+    fn error_code(&self) -> Code {
+        self.inner.error_code()
+    }
+}
+
+impl fmt::Display for ResponseError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.inner.fmt(f)
+    }
+}
+
+impl From<Error> for ResponseError {
+    fn from(error: Error) -> ResponseError {
+        ResponseError { inner: Box::new(error) }
+    }
+}
+
+impl From<meilisearch_core::Error> for ResponseError {
+    fn from(err: meilisearch_core::Error) -> ResponseError {
+        ResponseError { inner: Box::new(err) }
+    }
+}
+
+impl From<meilisearch_schema::Error> for ResponseError {
+    fn from(err: meilisearch_schema::Error) -> ResponseError {
+        ResponseError { inner: Box::new(err) }
+    }
+}
+
+impl From<FacetCountError> for ResponseError {
+    fn from(err: FacetCountError) -> ResponseError {
+        ResponseError { inner: Box::new(err) }
+    }
+}
+
+impl Serialize for ResponseError {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let struct_name = "ResponseError";
+        let field_count = 4;
+
+        let mut state = serializer.serialize_struct(struct_name, field_count)?;
+        state.serialize_field("message", &self.to_string())?;
+        state.serialize_field("errorCode", &self.error_name())?;
+        state.serialize_field("errorType", &self.error_type())?;
+        state.serialize_field("errorLink", &self.error_url())?;
+        state.end()
+    }
+}
+
+impl aweb::error::ResponseError for ResponseError {
+    fn error_response(&self) -> aweb::HttpResponse {
+        ResponseBuilder::new(self.status_code()).json(&self)
+    }
+
+    fn status_code(&self) -> StatusCode {
+        self.http_status()
+    }
+}
+
+#[derive(Debug)]
+pub enum Error {
     BadParameter(String, String),
-    OpenIndex(String),
+    BadRequest(String),
     CreateIndex(String),
-    Maintenance,
+    DocumentNotFound(String),
+    IndexNotFound(String),
+    IndexAlreadyExists(String),
+    Internal(String),
+    InvalidIndexUid,
+    InvalidToken(String),
+    MissingAuthorizationHeader,
+    NotFound(String),
+    OpenIndex(String),
+    RetrieveDocument(u32, String),
+    SearchDocuments(String),
+    PayloadTooLarge,
+    UnsupportedMediaType,
+    DumpAlreadyInProgress,
+    DumpProcessFailed(String),
 }
 
-impl ResponseError {
-    pub fn internal(message: impl Display) -> ResponseError {
-        ResponseError::Internal(message.to_string())
-    }
+impl error::Error for Error {}
 
-    pub fn bad_request(message: impl Display) -> ResponseError {
-        ResponseError::BadRequest(message.to_string())
-    }
-
-    pub fn invalid_token(message: impl Display) -> ResponseError {
-        ResponseError::InvalidToken(message.to_string())
-    }
-
-    pub fn not_found(message: impl Display) -> ResponseError {
-        ResponseError::NotFound(message.to_string())
-    }
-
-    pub fn index_not_found(message: impl Display) -> ResponseError {
-        ResponseError::IndexNotFound(message.to_string())
-    }
-
-    pub fn document_not_found(message: impl Display) -> ResponseError {
-        ResponseError::DocumentNotFound(message.to_string())
-    }
-
-    pub fn missing_header(message: impl Display) -> ResponseError {
-        ResponseError::MissingHeader(message.to_string())
-    }
-
-    pub fn bad_parameter(name: impl Display, message: impl Display) -> ResponseError {
-        ResponseError::BadParameter(name.to_string(), message.to_string())
-    }
-
-    pub fn open_index(message: impl Display) -> ResponseError {
-        ResponseError::OpenIndex(message.to_string())
-    }
-
-    pub fn create_index(message: impl Display) -> ResponseError {
-        ResponseError::CreateIndex(message.to_string())
-    }
-}
-
-impl IntoResponse for ResponseError {
-    fn into_response(self) -> Response {
+impl ErrorCode for Error {
+    fn error_code(&self) -> Code {
+        use Error::*;
         match self {
-            ResponseError::Internal(err) => {
-                error!("internal server error: {}", err);
-                error(
-                    String::from("Internal server error"),
-                    StatusCode::INTERNAL_SERVER_ERROR,
-                )
-            }
-            ResponseError::BadRequest(err) => {
-                warn!("bad request: {}", err);
-                error(err, StatusCode::BAD_REQUEST)
-            }
-            ResponseError::InvalidToken(err) => {
-                error(format!("Invalid API key: {}", err), StatusCode::FORBIDDEN)
-            }
-            ResponseError::NotFound(err) => error(err, StatusCode::NOT_FOUND),
-            ResponseError::IndexNotFound(index) => {
-                error(format!("Index {} not found", index), StatusCode::NOT_FOUND)
-            }
-            ResponseError::DocumentNotFound(id) => error(
-                format!("Document with id {} not found", id),
-                StatusCode::NOT_FOUND,
-            ),
-            ResponseError::MissingHeader(header) => error(
-                format!("Header {} is missing", header),
-                StatusCode::UNAUTHORIZED,
-            ),
-            ResponseError::BadParameter(param, e) => error(
-                format!("Url parameter {} error: {}", param, e),
-                StatusCode::BAD_REQUEST,
-            ),
-            ResponseError::CreateIndex(err) => error(
-                format!("Impossible to create index; {}", err),
-                StatusCode::BAD_REQUEST,
-            ),
-            ResponseError::OpenIndex(err) => error(
-                format!("Impossible to open index; {}", err),
-                StatusCode::BAD_REQUEST,
-            ),
-            ResponseError::Maintenance => error(
-                String::from("Server is in maintenance, please try again later"),
-                StatusCode::SERVICE_UNAVAILABLE,
-            ),
+            BadParameter(_, _) => Code::BadParameter,
+            BadRequest(_) => Code::BadRequest,
+            CreateIndex(_) => Code::CreateIndex,
+            DocumentNotFound(_) => Code::DocumentNotFound,
+            IndexNotFound(_) => Code::IndexNotFound,
+            IndexAlreadyExists(_) => Code::IndexAlreadyExists,
+            Internal(_) => Code::Internal,
+            InvalidIndexUid => Code::InvalidIndexUid,
+            InvalidToken(_) => Code::InvalidToken,
+            MissingAuthorizationHeader => Code::MissingAuthorizationHeader,
+            NotFound(_) => Code::NotFound,
+            OpenIndex(_) => Code::OpenIndex,
+            RetrieveDocument(_, _) => Code::RetrieveDocument,
+            SearchDocuments(_) => Code::SearchDocuments,
+            PayloadTooLarge => Code::PayloadTooLarge,
+            UnsupportedMediaType => Code::UnsupportedMediaType,
+            DumpAlreadyInProgress => Code::DumpAlreadyInProgress,
+            DumpProcessFailed(_) => Code::DumpProcessFailed,
         }
     }
 }
 
-#[derive(Serialize, Deserialize)]
-struct ErrorMessage {
-    message: String,
+#[derive(Debug)]
+pub enum FacetCountError {
+    AttributeNotSet(String),
+    SyntaxError(String),
+    UnexpectedToken { found: String, expected: &'static [&'static str] },
+    NoFacetSet,
 }
 
-fn error(message: String, status: StatusCode) -> Response {
-    let message = ErrorMessage { message };
-    tide::response::json(message)
-        .with_status(status)
-        .into_response()
+impl error::Error for FacetCountError {}
+
+impl ErrorCode for FacetCountError {
+    fn error_code(&self) -> Code {
+        Code::BadRequest
+    }
+}
+
+impl FacetCountError {
+    pub fn unexpected_token(found: impl ToString, expected: &'static [&'static str]) -> FacetCountError {
+        let found = found.to_string();
+        FacetCountError::UnexpectedToken { expected, found }
+    }
+}
+
+impl From<serde_json::error::Error> for FacetCountError {
+    fn from(other: serde_json::error::Error) -> FacetCountError {
+        FacetCountError::SyntaxError(other.to_string())
+    }
+}
+
+impl fmt::Display for FacetCountError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use FacetCountError::*;
+
+        match self {
+            AttributeNotSet(attr) => write!(f, "Attribute {} is not set as facet", attr),
+            SyntaxError(msg) => write!(f, "Syntax error: {}", msg),
+            UnexpectedToken { expected, found } => write!(f, "Unexpected {} found, expected {:?}", found, expected),
+            NoFacetSet => write!(f, "Can't perform facet count, as no facet is set"),
+        }
+    }
+}
+
+impl Error {
+    pub fn internal(err: impl fmt::Display) -> Error {
+        Error::Internal(err.to_string())
+    }
+
+    pub fn bad_request(err: impl fmt::Display) -> Error {
+        Error::BadRequest(err.to_string())
+    }
+
+    pub fn missing_authorization_header() -> Error {
+        Error::MissingAuthorizationHeader
+    }
+
+    pub fn invalid_token(err: impl fmt::Display) -> Error {
+        Error::InvalidToken(err.to_string())
+    }
+
+    pub fn not_found(err: impl fmt::Display) -> Error {
+        Error::NotFound(err.to_string())
+    }
+
+    pub fn index_not_found(err: impl fmt::Display) -> Error {
+        Error::IndexNotFound(err.to_string())
+    }
+
+    pub fn document_not_found(err: impl fmt::Display) -> Error {
+        Error::DocumentNotFound(err.to_string())
+    }
+
+    pub fn bad_parameter(param: impl fmt::Display, err: impl fmt::Display) -> Error {
+        Error::BadParameter(param.to_string(), err.to_string())
+    }
+
+    pub fn open_index(err: impl fmt::Display) -> Error {
+        Error::OpenIndex(err.to_string())
+    }
+
+    pub fn create_index(err: impl fmt::Display) -> Error {
+        Error::CreateIndex(err.to_string())
+    }
+
+    pub fn invalid_index_uid() -> Error {
+        Error::InvalidIndexUid
+    }
+
+    pub fn retrieve_document(doc_id: u32, err: impl fmt::Display) -> Error {
+        Error::RetrieveDocument(doc_id, err.to_string())
+    }
+
+    pub fn search_documents(err: impl fmt::Display) -> Error {
+        Error::SearchDocuments(err.to_string())
+    }
+
+    pub fn dump_conflict() -> Error {
+        Error::DumpAlreadyInProgress
+    }
+
+    pub fn dump_failed(message: String) -> Error {
+        Error::DumpProcessFailed(message)
+    }
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::BadParameter(param, err) => write!(f, "Url parameter {} error: {}", param, err),
+            Self::BadRequest(err) => f.write_str(err),
+            Self::CreateIndex(err) => write!(f, "Impossible to create index; {}", err),
+            Self::DocumentNotFound(document_id) => write!(f, "Document with id {} not found", document_id),
+            Self::IndexNotFound(index_uid) => write!(f, "Index {} not found", index_uid),
+            Self::IndexAlreadyExists(index_uid) => write!(f, "Index {} already exists", index_uid),
+            Self::Internal(err) => f.write_str(err),
+            Self::InvalidIndexUid => f.write_str("Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."),
+            Self::InvalidToken(err) => write!(f, "Invalid API key: {}", err),
+            Self::MissingAuthorizationHeader => f.write_str("You must have an authorization token"),
+            Self::NotFound(err) => write!(f, "{} not found", err),
+            Self::OpenIndex(err) => write!(f, "Impossible to open index; {}", err),
+            Self::RetrieveDocument(id, err) => write!(f, "Impossible to retrieve the document with id: {}; {}", id, err),
+            Self::SearchDocuments(err) => write!(f, "Impossible to search documents; {}", err),
+            Self::PayloadTooLarge => f.write_str("Payload too large"),
+            Self::UnsupportedMediaType => f.write_str("Unsupported media type"),
+            Self::DumpAlreadyInProgress => f.write_str("Another dump is already in progress"),
+            Self::DumpProcessFailed(message) => write!(f, "Dump process failed: {}", message),
+        }
+    }
+}
+
+impl From<std::io::Error> for Error {
+    fn from(err: std::io::Error) -> Error {
+        Error::Internal(err.to_string())
+    }
+}
+
+impl From<actix_http::Error> for Error {
+    fn from(err: actix_http::Error) -> Error {
+        Error::Internal(err.to_string())
+    }
+}
+
+impl From<meilisearch_core::Error> for Error {
+    fn from(err: meilisearch_core::Error) -> Error {
+        Error::Internal(err.to_string())
+    }
+}
+
+impl From<serde_json::error::Error> for Error {
+    fn from(err: serde_json::error::Error) -> Error {
+        Error::Internal(err.to_string())
+    }
+}
+
+impl From<JsonPayloadError> for Error {
+    fn from(err: JsonPayloadError) -> Error {
+        match err {
+            JsonPayloadError::Deserialize(err) => Error::BadRequest(format!("Invalid JSON: {}", err)),
+            JsonPayloadError::Overflow => Error::PayloadTooLarge,
+            JsonPayloadError::ContentType => Error::UnsupportedMediaType,
+            JsonPayloadError::Payload(err) => Error::BadRequest(format!("Problem while decoding the request: {}", err)),
+        }
+    }
+}
+
+impl From<QueryPayloadError> for Error {
+    fn from(err: QueryPayloadError) -> Error {
+        match err {
+            QueryPayloadError::Deserialize(err) => Error::BadRequest(format!("Invalid query parameters: {}", err)),
+        }
+    }
+}
+
+pub fn payload_error_handler<E: Into<Error>>(err: E) -> ResponseError {
+    let error: Error = err.into();
+    error.into()
 }

meilisearch-http/src/helpers/authentication.rs

@@ -0,0 +1,107 @@
+use std::cell::RefCell;
+use std::pin::Pin;
+use std::rc::Rc;
+use std::task::{Context, Poll};
+
+use actix_service::{Service, Transform};
+use actix_web::{dev::ServiceRequest, dev::ServiceResponse, web};
+use futures::future::{err, ok, Future, Ready};
+use actix_web::error::ResponseError as _;
+use actix_web::dev::Body;
+
+use crate::error::{Error, ResponseError};
+use crate::Data;
+
+#[derive(Clone)]
+pub enum Authentication {
+    Public,
+    Private,
+    Admin,
+}
+
+impl<S: 'static> Transform<S> for Authentication
+where
+    S: Service<Request = ServiceRequest, Response = ServiceResponse<Body>, Error = actix_web::Error>,
+    S::Future: 'static,
+{
+    type Request = ServiceRequest;
+    type Response = ServiceResponse<Body>;
+    type Error = actix_web::Error;
+    type InitError = ();
+    type Transform = LoggingMiddleware<S>;
+    type Future = Ready<Result<Self::Transform, Self::InitError>>;
+
+    fn new_transform(&self, service: S) -> Self::Future {
+        ok(LoggingMiddleware {
+            acl: self.clone(),
+            service: Rc::new(RefCell::new(service)),
+        })
+    }
+}
+
+pub struct LoggingMiddleware<S> {
+    acl: Authentication,
+    service: Rc<RefCell<S>>,
+}
+
+#[allow(clippy::type_complexity)]
+impl<S> Service for LoggingMiddleware<S>
+where
+    S: Service<Request = ServiceRequest, Response = ServiceResponse<Body>, Error = actix_web::Error> + 'static,
+    S::Future: 'static,
+{
+    type Request = ServiceRequest;
+    type Response = ServiceResponse<Body>;
+    type Error = actix_web::Error;
+    type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>>>>;
+
+    fn poll_ready(&mut self, cx: &mut Context) -> Poll<Result<(), Self::Error>> {
+        self.service.poll_ready(cx)
+    }
+
+    fn call(&mut self, req: ServiceRequest) -> Self::Future {
+        let mut svc = self.service.clone();
+        // This unwrap is left because this error should never appear. If that's the case, then
+        // it means that actix-web has an issue or someone changes the type `Data`.
+        let data = req.app_data::<web::Data<Data>>().unwrap();
+
+        if data.api_keys.master.is_none() {
+            return Box::pin(svc.call(req));
+        }
+
+        let auth_header = match req.headers().get("X-Meili-API-Key") {
+            Some(auth) => match auth.to_str() {
+                Ok(auth) => auth,
+                Err(_) => {
+                    let error = ResponseError::from(Error::MissingAuthorizationHeader).error_response();
+                    let (request, _) = req.into_parts();
+                    return Box::pin(ok(ServiceResponse::new(request, error)))
+                }
+            },
+            None => {
+                return Box::pin(err(ResponseError::from(Error::MissingAuthorizationHeader).into()));
+            }
+        };
+
+        let authenticated = match self.acl {
+            Authentication::Admin => data.api_keys.master.as_deref() == Some(auth_header),
+            Authentication::Private => {
+                data.api_keys.master.as_deref() == Some(auth_header)
+                    || data.api_keys.private.as_deref() == Some(auth_header)
+            }
+            Authentication::Public => {
+                data.api_keys.master.as_deref() == Some(auth_header)
+                    || data.api_keys.private.as_deref() == Some(auth_header)
+                    || data.api_keys.public.as_deref() == Some(auth_header)
+            }
+        };
+
+        if authenticated {
+            Box::pin(svc.call(req))
+        } else {
+            let error = ResponseError::from(Error::InvalidToken(auth_header.to_string())).error_response();
+            let (request, _) = req.into_parts();
+            return Box::pin(ok(ServiceResponse::new(request, error)))
+        }
+    }
+}

meilisearch-http/src/helpers/compression.rs

@@ -0,0 +1,35 @@
+use flate2::Compression;
+use flate2::read::GzDecoder;
+use flate2::write::GzEncoder;
+use std::fs::{create_dir_all, rename, File};
+use std::path::Path;
+use tar::{Builder, Archive};
+use uuid::Uuid;
+
+use crate::error::Error;
+
+pub fn to_tar_gz(src: &Path, dest: &Path) -> Result<(), Error> {
+    let file_name = format!(".{}", Uuid::new_v4().to_urn());
+    let p = dest.with_file_name(file_name);
+    let tmp_dest = p.as_path();
+
+    let f = File::create(tmp_dest)?;
+    let gz_encoder = GzEncoder::new(f, Compression::default());
+    let mut tar_encoder = Builder::new(gz_encoder);
+    tar_encoder.append_dir_all(".", src)?;
+    let gz_encoder = tar_encoder.into_inner()?;
+    gz_encoder.finish()?;
+
+    rename(tmp_dest, dest)?;
+
+    Ok(())
+}
+
+pub fn from_tar_gz(src: &Path, dest: &Path) -> Result<(), Error> {
+    let f = File::open(src)?;
+    let gz = GzDecoder::new(f);
+    let mut ar = Archive::new(gz);
+    create_dir_all(dest)?;
+    ar.unpack(dest)?;
+    Ok(())
+}

meilisearch-http/src/helpers/meilisearch.rs

@@ -1,75 +1,29 @@
-use crate::routes::setting::{RankingOrdering, Setting};
-use indexmap::IndexMap;
-use log::{error, warn};
-use meilisearch_core::criterion::*;
-use meilisearch_core::Highlight;
-use meilisearch_core::{Index, RankedMap};
-use meilisearch_core::MainT;
-use meilisearch_schema::{Schema, SchemaAttr};
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
 use std::cmp::Ordering;
 use std::collections::{HashMap, HashSet};
-use std::convert::From;
-use std::error;
-use std::fmt;
-use std::time::{Duration, Instant};
+use std::hash::{Hash, Hasher};
+use std::time::Instant;
 
-#[derive(Debug)]
-pub enum Error {
-    SearchDocuments(String),
-    RetrieveDocument(u64, String),
-    DocumentNotFound(u64),
-    CropFieldWrongType(String),
-    AttributeNotFoundOnDocument(String),
-    AttributeNotFoundOnSchema(String),
-    MissingFilterValue,
-    UnknownFilteredAttribute,
-    Internal(String),
-}
+use indexmap::IndexMap;
+use log::error;
+use meilisearch_core::{Filter, MainReader};
+use meilisearch_core::facets::FacetFilter;
+use meilisearch_core::criterion::*;
+use meilisearch_core::settings::RankingRule;
+use meilisearch_core::{Highlight, Index, RankedMap};
+use meilisearch_schema::{FieldId, Schema};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use siphasher::sip::SipHasher;
+use slice_group_by::GroupBy;
 
-impl error::Error for Error {}
-
-impl fmt::Display for Error {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        use Error::*;
-
-        match self {
-            SearchDocuments(err) => write!(f, "impossible to search documents; {}", err),
-            RetrieveDocument(id, err) => write!(
-                f,
-                "impossible to retrieve the document with id: {}; {}",
-                id, err
-            ),
-            DocumentNotFound(id) => write!(f, "document {} not found", id),
-            CropFieldWrongType(field) => {
-                write!(f, "the field {} cannot be cropped it's not a string", field)
-            }
-            AttributeNotFoundOnDocument(field) => {
-                write!(f, "field {} is not found on document", field)
-            }
-            AttributeNotFoundOnSchema(field) => write!(f, "field {} is not found on schema", field),
-            MissingFilterValue => f.write_str("a filter doesn't have a value to compare it with"),
-            UnknownFilteredAttribute => {
-                f.write_str("a filter is specifying an unknown schema attribute")
-            }
-            Internal(err) => write!(f, "internal error; {}", err),
-        }
-    }
-}
-
-impl From<meilisearch_core::Error> for Error {
-    fn from(error: meilisearch_core::Error) -> Self {
-        Error::Internal(error.to_string())
-    }
-}
+use crate::error::{Error, ResponseError};
 
 pub trait IndexSearchExt {
-    fn new_search(&self, query: String) -> SearchBuilder;
+    fn new_search(&self, query: Option<String>) -> SearchBuilder;
 }
 
 impl IndexSearchExt for Index {
-    fn new_search(&self, query: String) -> SearchBuilder {
+    fn new_search(&self, query: Option<String>) -> SearchBuilder {
         SearchBuilder {
             index: self,
             query,
@@ -77,27 +31,27 @@ impl IndexSearchExt for Index {
             limit: 20,
             attributes_to_crop: None,
             attributes_to_retrieve: None,
-            attributes_to_search_in: None,
             attributes_to_highlight: None,
             filters: None,
-            timeout: Duration::from_millis(30),
             matches: false,
+            facet_filters: None,
+            facets: None,
         }
     }
 }
 
 pub struct SearchBuilder<'a> {
     index: &'a Index,
-    query: String,
+    query: Option<String>,
     offset: usize,
     limit: usize,
     attributes_to_crop: Option<HashMap<String, usize>>,
     attributes_to_retrieve: Option<HashSet<String>>,
-    attributes_to_search_in: Option<HashSet<String>>,
     attributes_to_highlight: Option<HashSet<String>>,
     filters: Option<String>,
-    timeout: Duration,
     matches: bool,
+    facet_filters: Option<FacetFilter>,
+    facets: Option<Vec<(FieldId, String)>>
 }
 
 impl<'a> SearchBuilder<'a> {
@@ -127,48 +81,39 @@ impl<'a> SearchBuilder<'a> {
         self
     }
 
-    pub fn attributes_to_search_in(&mut self, value: HashSet<String>) -> &SearchBuilder {
-        self.attributes_to_search_in = Some(value);
-        self
-    }
-
-    pub fn add_attribute_to_search_in(&mut self, value: String) -> &SearchBuilder {
-        let attributes_to_search_in = self.attributes_to_search_in.get_or_insert(HashSet::new());
-        attributes_to_search_in.insert(value);
-        self
-    }
-
     pub fn attributes_to_highlight(&mut self, value: HashSet<String>) -> &SearchBuilder {
         self.attributes_to_highlight = Some(value);
         self
     }
 
+    pub fn add_facet_filters(&mut self, filters: FacetFilter) -> &SearchBuilder {
+        self.facet_filters = Some(filters);
+        self
+    }
+
     pub fn filters(&mut self, value: String) -> &SearchBuilder {
         self.filters = Some(value);
         self
     }
 
-    pub fn timeout(&mut self, value: Duration) -> &SearchBuilder {
-        self.timeout = value;
-        self
-    }
-
     pub fn get_matches(&mut self) -> &SearchBuilder {
         self.matches = true;
         self
     }
 
-    pub fn search(&self, reader: &heed::RoTxn<MainT>) -> Result<SearchResult, Error> {
-        let schema = self.index.main.schema(reader);
-        let schema = schema.map_err(|e| Error::Internal(e.to_string()))?;
-        let schema = match schema {
-            Some(schema) => schema,
-            None => return Err(Error::Internal(String::from("missing schema"))),
-        };
+    pub fn add_facets(&mut self, facets: Vec<(FieldId, String)>) -> &SearchBuilder {
+        self.facets = Some(facets);
+        self
+    }
 
-        let ranked_map = self.index.main.ranked_map(reader);
-        let ranked_map = ranked_map.map_err(|e| Error::Internal(e.to_string()))?;
-        let ranked_map = ranked_map.unwrap_or_default();
+    pub fn search(self, reader: &MainReader) -> Result<SearchResult, ResponseError> {
+        let schema = self
+            .index
+            .main
+            .schema(reader)?
+            .ok_or(Error::internal("missing schema"))?;
+
+        let ranked_map = self.index.main.ranked_map(reader)?.unwrap_or_default();
 
         // Change criteria
         let mut query_builder = match self.get_criteria(reader, &ranked_map, &schema)? {
@@ -176,80 +121,84 @@ impl<'a> SearchBuilder<'a> {
             None => self.index.query_builder(),
         };
 
-        // Filter searchable fields
-        if let Some(fields) = &self.attributes_to_search_in {
-            for attribute in fields.iter().filter_map(|f| schema.attribute(f)) {
-                query_builder.add_searchable_attribute(attribute.0);
-            }
-        }
-
-        if let Some(filters) = &self.filters {
-            let mut split = filters.split(':');
-            match (split.next(), split.next()) {
-                (Some(_), None) | (Some(_), Some("")) => return Err(Error::MissingFilterValue),
-                (Some(attr), Some(value)) => {
-                    let ref_reader = reader;
-                    let ref_index = &self.index;
-                    let value = value.trim().to_lowercase();
-
-                    let attr = match schema.attribute(attr) {
-                        Some(attr) => attr,
-                        None => return Err(Error::UnknownFilteredAttribute),
-                    };
-
-                    query_builder.with_filter(move |id| {
-                        let attr = attr;
-                        let index = ref_index;
-                        let reader = ref_reader;
-
-                        match index.document_attribute::<Value>(reader, id, attr) {
-                            Ok(Some(Value::String(s))) => s.to_lowercase() == value,
-                            Ok(Some(Value::Bool(b))) => {
-                                (value == "true" && b) || (value == "false" && !b)
-                            }
-                            Ok(Some(Value::Array(a))) => {
-                                a.into_iter().any(|s| s.as_str() == Some(&value))
-                            }
-                            _ => false,
-                        }
-                    });
+        if let Some(filter_expression) = &self.filters {
+            let filter = Filter::parse(filter_expression, &schema)?;
+            let index = &self.index;
+            query_builder.with_filter(move |id| {
+                let reader = &reader;
+                let filter = &filter;
+                match filter.test(reader, index, id) {
+                    Ok(res) => res,
+                    Err(e) => {
+                        log::warn!("unexpected error during filtering: {}", e);
+                        false
+                    }
                 }
-                (_, _) => (),
-            }
+            });
         }
 
-        query_builder.with_fetch_timeout(self.timeout);
+        if let Some(field) = self.index.main.distinct_attribute(reader)? {
+            let index = &self.index;
+            query_builder.with_distinct(1, move |id| {
+                match index.document_attribute_bytes(reader, id, field) {
+                    Ok(Some(bytes)) => {
+                        let mut s = SipHasher::new();
+                        bytes.hash(&mut s);
+                        Some(s.finish())
+                    }
+                    _ => None,
+                }
+            });
+        }
+
+        query_builder.set_facet_filter(self.facet_filters);
+        query_builder.set_facets(self.facets);
 
         let start = Instant::now();
-        let docs = query_builder.query(reader, &self.query, self.offset..(self.offset + self.limit));
+        let result = query_builder.query(reader, self.query.as_deref(), self.offset..(self.offset + self.limit));
+        let search_result = result.map_err(Error::search_documents)?;
         let time_ms = start.elapsed().as_millis() as usize;
 
-        let mut hits = Vec::with_capacity(self.limit);
-        for doc in docs.map_err(|e| Error::SearchDocuments(e.to_string()))? {
-            // retrieve the content of document in kv store
-            let mut fields: Option<HashSet<&str>> = None;
-            if let Some(attributes_to_retrieve) = &self.attributes_to_retrieve {
-                let mut set = HashSet::new();
-                for field in attributes_to_retrieve {
-                    set.insert(field.as_str());
+        let mut all_attributes: HashSet<&str> = HashSet::new();
+        let mut all_formatted: HashSet<&str> = HashSet::new();
+
+        match &self.attributes_to_retrieve {
+            Some(to_retrieve) => {
+                all_attributes.extend(to_retrieve.iter().map(String::as_str));
+
+                if let Some(to_highlight) = &self.attributes_to_highlight {
+                    all_formatted.extend(to_highlight.iter().map(String::as_str));
                 }
-                fields = Some(set);
-            }
 
-            let document: IndexMap<String, Value> = self
+                if let Some(to_crop) = &self.attributes_to_crop {
+                    all_formatted.extend(to_crop.keys().map(String::as_str));
+                }
+
+                all_attributes.extend(&all_formatted);
+            },
+            None => {
+                all_attributes.extend(schema.displayed_names());
+                // If we specified at least one attribute to highlight or crop then
+                // all available attributes will be returned in the _formatted field.
+                if self.attributes_to_highlight.is_some() || self.attributes_to_crop.is_some() {
+                    all_formatted.extend(all_attributes.iter().cloned());
+                }
+            },
+        }
+
+        let mut hits = Vec::with_capacity(self.limit);
+        for doc in search_result.documents {
+            let mut document: IndexMap<String, Value> = self
                 .index
-                .document(reader, fields.as_ref(), doc.id)
-                .map_err(|e| Error::RetrieveDocument(doc.id.0, e.to_string()))?
-                .ok_or(Error::DocumentNotFound(doc.id.0))?;
+                .document(reader, Some(&all_attributes), doc.id)
+                .map_err(|e| Error::retrieve_document(doc.id.0, e))?
+                .unwrap_or_default();
 
-            let has_attributes_to_highlight = self.attributes_to_highlight.is_some();
-            let has_attributes_to_crop = self.attributes_to_crop.is_some();
+            let mut formatted = document.iter()
+                .filter(|(key, _)| all_formatted.contains(key.as_str()))
+                .map(|(k, v)| (k.clone(), v.clone()))
+                .collect();
 
-            let mut formatted = if has_attributes_to_highlight || has_attributes_to_crop {
-                document.clone()
-            } else {
-                IndexMap::new()
-            };
             let mut matches = doc.highlights.clone();
 
             // Crops fields if needed
@@ -258,15 +207,24 @@ impl<'a> SearchBuilder<'a> {
             }
 
             // Transform to readable matches
-            let matches = calculate_matches(matches, self.attributes_to_retrieve.clone(), &schema);
-
-            if !self.matches {
-                if let Some(attributes_to_highlight) = &self.attributes_to_highlight {
-                    formatted = calculate_highlights(&formatted, &matches, attributes_to_highlight);
-                }
+            if let Some(attributes_to_highlight) = &self.attributes_to_highlight {
+                let matches = calculate_matches(
+                    &matches,
+                    self.attributes_to_highlight.clone(),
+                    &schema,
+                );
+                formatted = calculate_highlights(&formatted, &matches, attributes_to_highlight);
             }
 
-            let matches_info = if self.matches { Some(matches) } else { None };
+            let matches_info = if self.matches {
+                Some(calculate_matches(&matches, self.attributes_to_retrieve.clone(), &schema))
+            } else {
+                None
+            };
+
+            if let Some(attributes_to_retrieve) = &self.attributes_to_retrieve {
+                document.retain(|key, _| attributes_to_retrieve.contains(&key.to_string()))
+            }
 
             let hit = SearchHit {
                 document,
@@ -281,8 +239,12 @@ impl<'a> SearchBuilder<'a> {
             hits,
             offset: self.offset,
             limit: self.limit,
+            nb_hits: search_result.nb_hits,
+            exhaustive_nb_hits: search_result.exhaustive_nb_hit,
             processing_time_ms: time_ms,
-            query: self.query.to_string(),
+            query: self.query.unwrap_or_default(),
+            facets_distribution: search_result.facets,
+            exhaustive_facets_count: search_result.exhaustive_facets_count,
         };
 
         Ok(results)
@@ -290,91 +252,56 @@ impl<'a> SearchBuilder<'a> {
 
     pub fn get_criteria(
         &self,
-        reader: &heed::RoTxn<MainT>,
+        reader: &MainReader,
         ranked_map: &'a RankedMap,
         schema: &Schema,
-    ) -> Result<Option<Criteria<'a>>, Error> {
-        let current_settings = match self.index.main.customs(reader).unwrap() {
-            Some(bytes) => bincode::deserialize(bytes).unwrap(),
-            None => Setting::default(),
-        };
-
-        let ranking_rules = &current_settings.ranking_rules;
-        let ranking_order = &current_settings.ranking_order;
+    ) -> Result<Option<Criteria<'a>>, ResponseError> {
+        let ranking_rules = self.index.main.ranking_rules(reader)?;
 
         if let Some(ranking_rules) = ranking_rules {
             let mut builder = CriteriaBuilder::with_capacity(7 + ranking_rules.len());
-            if let Some(ranking_rules_order) = ranking_order {
-                for rule in ranking_rules_order {
-                    match rule.as_str() {
-                        "_typo" => builder.push(Typo),
-                        "_words" => builder.push(Words),
-                        "_proximity" => builder.push(Proximity),
-                        "_attribute" => builder.push(Attribute),
-                        "_words_position" => builder.push(WordsPosition),
-                        "_exact" => builder.push(Exact),
-                        _ => {
-                            let order = match ranking_rules.get(rule.as_str()) {
-                                Some(o) => o,
-                                None => continue,
-                            };
-
-                            let custom_ranking = match order {
-                                RankingOrdering::Asc => {
-                                    SortByAttr::lower_is_better(&ranked_map, &schema, &rule)
-                                        .unwrap()
-                                }
-                                RankingOrdering::Dsc => {
-                                    SortByAttr::higher_is_better(&ranked_map, &schema, &rule)
-                                        .unwrap()
-                                }
-                            };
-
-                            builder.push(custom_ranking);
+            for rule in ranking_rules {
+                match rule {
+                    RankingRule::Typo => builder.push(Typo),
+                    RankingRule::Words => builder.push(Words),
+                    RankingRule::Proximity => builder.push(Proximity),
+                    RankingRule::Attribute => builder.push(Attribute),
+                    RankingRule::WordsPosition => builder.push(WordsPosition),
+                    RankingRule::Exactness => builder.push(Exactness),
+                    RankingRule::Asc(field) => {
+                        match SortByAttr::lower_is_better(&ranked_map, &schema, &field) {
+                            Ok(rule) => builder.push(rule),
+                            Err(err) => error!("Error during criteria builder; {:?}", err),
+                        }
+                    }
+                    RankingRule::Desc(field) => {
+                        match SortByAttr::higher_is_better(&ranked_map, &schema, &field) {
+                            Ok(rule) => builder.push(rule),
+                            Err(err) => error!("Error during criteria builder; {:?}", err),
                         }
                     }
                 }
-                builder.push(DocumentId);
-                return Ok(Some(builder.build()));
-            } else {
-                builder.push(Typo);
-                builder.push(Words);
-                builder.push(Proximity);
-                builder.push(Attribute);
-                builder.push(WordsPosition);
-                builder.push(Exact);
-                for (rule, order) in ranking_rules.iter() {
-                    let custom_ranking = match order {
-                        RankingOrdering::Asc => {
-                            SortByAttr::lower_is_better(&ranked_map, &schema, &rule)
-                        }
-                        RankingOrdering::Dsc => {
-                            SortByAttr::higher_is_better(&ranked_map, &schema, &rule)
-                        }
-                    };
-                    if let Ok(custom_ranking) = custom_ranking {
-                        builder.push(custom_ranking);
-                    } else {
-                        // TODO push this warning to a log tree
-                        warn!("Custom ranking cannot be added; Attribute {} not registered for ranking", rule)
-                    }
-
-                }
-                builder.push(DocumentId);
-                return Ok(Some(builder.build()));
             }
+            builder.push(DocumentId);
+            return Ok(Some(builder.build()));
         }
 
         Ok(None)
     }
 }
 
-#[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Serialize, Deserialize)]
+#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
 pub struct MatchPosition {
     pub start: usize,
     pub length: usize,
 }
 
+impl PartialOrd for MatchPosition {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
 impl Ord for MatchPosition {
     fn cmp(&self, other: &Self) -> Ordering {
         match self.start.cmp(&other.start) {
@@ -398,16 +325,48 @@ pub struct SearchHit {
     pub matches_info: Option<MatchesInfos>,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct SearchResult {
     pub hits: Vec<SearchHit>,
     pub offset: usize,
     pub limit: usize,
+    pub nb_hits: usize,
+    pub exhaustive_nb_hits: bool,
     pub processing_time_ms: usize,
     pub query: String,
-    // pub parsed_query: String,
-    // pub params: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub facets_distribution: Option<HashMap<String, HashMap<String, usize>>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub exhaustive_facets_count: Option<bool>,
+}
+
+/// returns the start index and the length on the crop.
+fn aligned_crop(text: &str, match_index: usize, context: usize) -> (usize, usize) {
+    let is_word_component = |c: &char| c.is_alphanumeric() && !super::is_cjk(*c);
+
+    let word_end_index = |mut index| {
+        if text.chars().nth(index - 1).map_or(false, |c| is_word_component(&c)) {
+            index += text.chars().skip(index).take_while(is_word_component).count();
+        }
+        index
+    };
+
+    if context == 0 {
+        // count need to be at least 1 for cjk queries to return something
+        return (match_index, 1 + text.chars().skip(match_index).take_while(is_word_component).count());
+    }
+    let start = match match_index.saturating_sub(context) {
+        0 => 0,
+        n => {
+            let word_end_index = word_end_index(n);
+            // skip whitespaces if any
+            word_end_index + text.chars().skip(word_end_index).take_while(char::is_ascii_whitespace).count()
+        }
+    };
+    let end = word_end_index(match_index + context);
+
+    (start, end - start)
 }
 
 fn crop_text(
@@ -418,14 +377,23 @@ fn crop_text(
     let mut matches = matches.into_iter().peekable();
 
     let char_index = matches.peek().map(|m| m.char_index as usize).unwrap_or(0);
-    let start = char_index.saturating_sub(context);
-    let text = text.chars().skip(start).take(context * 2).collect();
+    let (start, count) = aligned_crop(text, char_index, context);
 
+    // TODO do something about double allocation
+    let text = text
+        .chars()
+        .skip(start)
+        .take(count)
+        .collect::<String>()
+        .trim()
+        .to_string();
+
+    // update matches index to match the new cropped text
     let matches = matches
-        .take_while(|m| (m.char_index as usize) + (m.char_length as usize) <= start + (context * 2))
-        .map(|match_| Highlight {
-            char_index: match_.char_index - start as u16,
-            ..match_
+        .take_while(|m| (m.char_index as usize) + (m.char_length as usize) <= start + count)
+        .map(|m| Highlight {
+            char_index: m.char_index - start as u16,
+            ..m
         })
         .collect();
 
@@ -441,14 +409,14 @@ fn crop_document(
     matches.sort_unstable_by_key(|m| (m.char_index, m.char_length));
 
     for (field, length) in fields {
-        let attribute = match schema.attribute(field) {
+        let attribute = match schema.id(field) {
             Some(attribute) => attribute,
             None => continue,
         };
 
         let selected_matches = matches
             .iter()
-            .filter(|m| SchemaAttr::new(m.attribute) == attribute)
+            .filter(|m| FieldId::new(m.attribute) == attribute)
             .cloned();
 
         if let Some(Value::String(ref mut original_text)) = document.get_mut(field) {
@@ -457,39 +425,41 @@ fn crop_document(
 
             *original_text = cropped_text;
 
-            matches.retain(|m| SchemaAttr::new(m.attribute) != attribute);
+            matches.retain(|m| FieldId::new(m.attribute) != attribute);
             matches.extend_from_slice(&cropped_matches);
         }
     }
 }
 
 fn calculate_matches(
-    matches: Vec<Highlight>,
+    matches: &[Highlight],
     attributes_to_retrieve: Option<HashSet<String>>,
     schema: &Schema,
 ) -> MatchesInfos {
     let mut matches_result: HashMap<String, Vec<MatchPosition>> = HashMap::new();
     for m in matches.iter() {
-        let attribute = schema
-            .attribute_name(SchemaAttr::new(m.attribute))
-            .to_string();
-        if let Some(attributes_to_retrieve) = attributes_to_retrieve.clone() {
-            if !attributes_to_retrieve.contains(attribute.as_str()) {
+        if let Some(attribute) = schema.name(FieldId::new(m.attribute)) {
+            if let Some(ref attributes_to_retrieve) = attributes_to_retrieve {
+                if !attributes_to_retrieve.contains(attribute) {
+                    continue;
+                }
+            }
+            if !schema.displayed_names().contains(&attribute) {
                 continue;
             }
-        };
-        if let Some(pos) = matches_result.get_mut(&attribute) {
-            pos.push(MatchPosition {
-                start: m.char_index as usize,
-                length: m.char_length as usize,
-            });
-        } else {
-            let mut positions = Vec::new();
-            positions.push(MatchPosition {
-                start: m.char_index as usize,
-                length: m.char_length as usize,
-            });
-            matches_result.insert(attribute, positions);
+            if let Some(pos) = matches_result.get_mut(attribute) {
+                pos.push(MatchPosition {
+                    start: m.char_index as usize,
+                    length: m.char_length as usize,
+                });
+            } else {
+                let mut positions = Vec::new();
+                positions.push(MatchPosition {
+                    start: m.char_index as usize,
+                    length: m.char_length as usize,
+                });
+                matches_result.insert(attribute.to_string(), positions);
+            }
         }
     }
     for (_, val) in matches_result.iter_mut() {
@@ -509,30 +479,33 @@ fn calculate_highlights(
     for (attribute, matches) in matches.iter() {
         if attributes_to_highlight.contains(attribute) {
             if let Some(Value::String(value)) = document.get(attribute) {
-                let value: Vec<_> = value.chars().collect();
+                let value = value;
                 let mut highlighted_value = String::new();
                 let mut index = 0;
-                for m in matches {
-                    if m.start >= index {
-                        let before = value.get(index..m.start);
-                        let highlighted = value.get(m.start..(m.start + m.length));
-                        if let (Some(before), Some(highlighted)) = (before, highlighted) {
-                            highlighted_value.extend(before);
-                            highlighted_value.push_str("<em>");
-                            highlighted_value.extend(highlighted);
-                            highlighted_value.push_str("</em>");
-                            index = m.start + m.length;
-                        } else {
-                            error!("value: {:?}; index: {:?}, match: {:?}", value, index, m);
-                        }
+
+                let longest_matches = matches
+                    .linear_group_by_key(|m| m.start)
+                    .map(|group| group.last().unwrap())
+                    .filter(move |m| m.start >= index);
+
+                for m in longest_matches {
+                    let before = value.get(index..m.start);
+                    let highlighted = value.get(m.start..(m.start + m.length));
+                    if let (Some(before), Some(highlighted)) = (before, highlighted) {
+                        highlighted_value.push_str(before);
+                        highlighted_value.push_str("<em>");
+                        highlighted_value.push_str(highlighted);
+                        highlighted_value.push_str("</em>");
+                        index = m.start + m.length;
+                    } else {
+                        error!("value: {:?}; index: {:?}, match: {:?}", value, index, m);
                     }
                 }
-                highlighted_value.extend(value[index..].iter());
+                highlighted_value.push_str(&value[index..]);
                 highlight_result.insert(attribute.to_string(), Value::String(highlighted_value));
             };
         }
     }
-
     highlight_result
 }
 
@@ -540,6 +513,67 @@ fn calculate_highlights(
 mod tests {
     use super::*;
 
+    #[test]
+    fn aligned_crops() {
+        let text = r#"En ce début de trentième millénaire, l'Empire n'a jamais été aussi puissant, aussi étendu à travers toute la galaxie. C'est dans sa capitale, Trantor, que l'éminent savant Hari Seldon invente la psychohistoire, une science toute nouvelle, à base de psychologie et de mathématiques, qui lui permet de prédire l'avenir... C'est-à-dire l'effondrement de l'Empire d'ici cinq siècles et au-delà, trente mille années de chaos et de ténèbres. Pour empêcher cette catastrophe et sauver la civilisation, Seldon crée la Fondation."#;
+
+        // simple test
+        let (start, length) = aligned_crop(&text, 6, 2);
+        let cropped =  text.chars().skip(start).take(length).collect::<String>().trim().to_string();
+        assert_eq!("début", cropped);
+
+        // first word test
+        let (start, length) = aligned_crop(&text, 0, 1);
+        let cropped =  text.chars().skip(start).take(length).collect::<String>().trim().to_string();
+        assert_eq!("En", cropped);
+        // last word test
+        let (start, length) = aligned_crop(&text, 510, 2);
+        let cropped =  text.chars().skip(start).take(length).collect::<String>().trim().to_string();
+        assert_eq!("Fondation", cropped);
+
+        // CJK tests
+        let text = "this isのス foo myタイリ test";
+
+        // mixed charset
+        let (start, length) = aligned_crop(&text, 5, 3);
+        let cropped =  text.chars().skip(start).take(length).collect::<String>().trim().to_string();
+        assert_eq!("isの", cropped);
+
+        // split regular word / CJK word, no space
+        let (start, length) = aligned_crop(&text, 7, 1);
+        let cropped =  text.chars().skip(start).take(length).collect::<String>().trim().to_string();
+        assert_eq!("の", cropped);
+    }
+
+    #[test]
+    fn calculate_matches() {
+        let mut matches = Vec::new();
+        matches.push(Highlight { attribute: 0, char_index: 0, char_length: 3});
+        matches.push(Highlight { attribute: 0, char_index: 0, char_length: 2});
+
+        let mut attributes_to_retrieve: HashSet<String> = HashSet::new();
+        attributes_to_retrieve.insert("title".to_string());
+
+        let schema = Schema::with_primary_key("title");
+
+        let matches_result = super::calculate_matches(&matches, Some(attributes_to_retrieve), &schema);
+
+        let mut matches_result_expected: HashMap<String, Vec<MatchPosition>> = HashMap::new();
+
+        let mut positions = Vec::new();
+        positions.push(MatchPosition {
+            start: 0,
+            length: 2,
+        });
+        positions.push(MatchPosition {
+            start: 0,
+            length: 3,
+        });
+        matches_result_expected.insert("title".to_string(), positions);
+
+        assert_eq!(matches_result, matches_result_expected);
+    }
+
     #[test]
     fn calculate_highlights() {
         let data = r#"{
@@ -563,7 +597,7 @@ mod tests {
 
         let mut m = Vec::new();
         m.push(MatchPosition {
-            start: 510,
+            start: 529,
             length: 9,
         });
         matches.insert("description".to_string(), m);
@@ -578,4 +612,38 @@ mod tests {
 
         assert_eq!(result, result_expected);
     }
+
+    #[test]
+    fn highlight_longest_match() {
+        let data = r#"{
+            "title": "Ice"
+        }"#;
+
+        let document: IndexMap<String, Value> = serde_json::from_str(data).unwrap();
+        let mut attributes_to_highlight = HashSet::new();
+        attributes_to_highlight.insert("title".to_string());
+
+        let mut matches = HashMap::new();
+
+        let mut m = Vec::new();
+        m.push(MatchPosition {
+            start: 0,
+            length: 2,
+        });
+        m.push(MatchPosition {
+            start: 0,
+            length: 3,
+        });
+        matches.insert("title".to_string(), m);
+
+        let result = super::calculate_highlights(&document, &matches, &attributes_to_highlight);
+
+        let mut result_expected = IndexMap::new();
+        result_expected.insert(
+            "title".to_string(),
+            Value::String("<em>Ice</em>".to_string()),
+        );
+
+        assert_eq!(result, result_expected);
+    }
 }

meilisearch-http/src/helpers/mod.rs

@@ -1,2 +1,26 @@
+pub mod authentication;
 pub mod meilisearch;
-pub mod tide;
+pub mod normalize_path;
+pub mod compression;
+
+pub use authentication::Authentication;
+pub use normalize_path::NormalizePath;
+
+pub fn is_cjk(c: char) -> bool {
+    ('\u{1100}'..'\u{11ff}').contains(&c)  // Hangul Jamo
+        || ('\u{2e80}'..'\u{2eff}').contains(&c)  // CJK Radicals Supplement
+        || ('\u{2f00}'..'\u{2fdf}').contains(&c) // Kangxi radical
+        || ('\u{3000}'..'\u{303f}').contains(&c) // Japanese-style punctuation
+        || ('\u{3040}'..'\u{309f}').contains(&c) // Japanese Hiragana
+        || ('\u{30a0}'..'\u{30ff}').contains(&c) // Japanese Katakana
+        || ('\u{3100}'..'\u{312f}').contains(&c)
+        || ('\u{3130}'..'\u{318F}').contains(&c) // Hangul Compatibility Jamo
+        || ('\u{3200}'..'\u{32ff}').contains(&c) // Enclosed CJK Letters and Months
+        || ('\u{3400}'..'\u{4dbf}').contains(&c) // CJK Unified Ideographs Extension A
+        || ('\u{4e00}'..'\u{9fff}').contains(&c) // CJK Unified Ideographs
+        || ('\u{a960}'..'\u{a97f}').contains(&c) // Hangul Jamo Extended-A
+        || ('\u{ac00}'..'\u{d7a3}').contains(&c) // Hangul Syllables
+        || ('\u{d7b0}'..'\u{d7ff}').contains(&c) // Hangul Jamo Extended-B
+        || ('\u{f900}'..'\u{faff}').contains(&c) // CJK Compatibility Ideographs
+        || ('\u{ff00}'..'\u{ffef}').contains(&c) // Full-width roman characters and half-width katakana
+}

meilisearch-http/src/helpers/normalize_path.rs

@@ -0,0 +1,86 @@
+/// From https://docs.rs/actix-web/3.0.0-alpha.2/src/actix_web/middleware/normalize.rs.html#34
+use actix_http::Error;
+use actix_service::{Service, Transform};
+use actix_web::{
+    dev::ServiceRequest,
+    dev::ServiceResponse,
+    http::uri::{PathAndQuery, Uri},
+};
+use futures::future::{ok, Ready};
+use regex::Regex;
+use std::task::{Context, Poll};
+pub struct NormalizePath;
+
+impl<S, B> Transform<S> for NormalizePath
+where
+    S: Service<Request = ServiceRequest, Response = ServiceResponse<B>, Error = Error>,
+    S::Future: 'static,
+{
+    type Request = ServiceRequest;
+    type Response = ServiceResponse<B>;
+    type Error = Error;
+    type InitError = ();
+    type Transform = NormalizePathNormalization<S>;
+    type Future = Ready<Result<Self::Transform, Self::InitError>>;
+
+    fn new_transform(&self, service: S) -> Self::Future {
+        ok(NormalizePathNormalization {
+            service,
+            merge_slash: Regex::new("//+").unwrap(),
+        })
+    }
+}
+
+pub struct NormalizePathNormalization<S> {
+    service: S,
+    merge_slash: Regex,
+}
+
+impl<S, B> Service for NormalizePathNormalization<S>
+where
+    S: Service<Request = ServiceRequest, Response = ServiceResponse<B>, Error = Error>,
+    S::Future: 'static,
+{
+    type Request = ServiceRequest;
+    type Response = ServiceResponse<B>;
+    type Error = Error;
+    type Future = S::Future;
+
+    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        self.service.poll_ready(cx)
+    }
+
+    fn call(&mut self, mut req: ServiceRequest) -> Self::Future {
+        let head = req.head_mut();
+
+        // always add trailing slash, might be an extra one
+        let path = head.uri.path().to_string() + "/";
+
+        if self.merge_slash.find(&path).is_some() {
+            // normalize multiple /'s to one /
+            let path = self.merge_slash.replace_all(&path, "/");
+
+            let path = if path.len() > 1 {
+                path.trim_end_matches('/')
+            } else {
+                &path
+            };
+
+            let mut parts = head.uri.clone().into_parts();
+            let pq = parts.path_and_query.as_ref().unwrap();
+
+            let path = if let Some(q) = pq.query() {
+                bytes::Bytes::from(format!("{}?{}", path, q))
+            } else {
+                bytes::Bytes::copy_from_slice(path.as_bytes())
+            };
+            parts.path_and_query = Some(PathAndQuery::from_maybe_shared(path).unwrap());
+
+            let uri = Uri::from_parts(parts).unwrap();
+            req.match_info_mut().get_mut().update(&uri);
+            req.head_mut().uri = uri;
+        }
+
+        self.service.call(req)
+    }
+}

meilisearch-http/src/helpers/tide.rs

@@ -1,110 +0,0 @@
-use crate::error::{ResponseError, SResult};
-use crate::models::token::*;
-use crate::Data;
-use chrono::Utc;
-use heed::types::{SerdeBincode, Str};
-use meilisearch_core::Index;
-use tide::Context;
-
-pub trait ContextExt {
-    fn is_allowed(&self, acl: ACL) -> SResult<()>;
-    fn header(&self, name: &str) -> Result<String, ResponseError>;
-    fn url_param(&self, name: &str) -> Result<String, ResponseError>;
-    fn index(&self) -> Result<Index, ResponseError>;
-    fn identifier(&self) -> Result<String, ResponseError>;
-}
-
-impl ContextExt for Context<Data> {
-    fn is_allowed(&self, acl: ACL) -> SResult<()> {
-        let api_key = match &self.state().api_key {
-            Some(api_key) => api_key,
-            None => return Ok(()),
-        };
-
-        let user_api_key = self.header("X-Meili-API-Key")?;
-        if user_api_key == *api_key {
-            return Ok(());
-        }
-        let request_index: Option<String> = None; //self.param::<String>("index").ok();
-
-        let db = &self.state().db;
-        let reader = db.main_read_txn().map_err(ResponseError::internal)?;
-
-        let token_key = format!("{}{}", TOKEN_PREFIX_KEY, user_api_key);
-
-        let token_config = db
-            .common_store()
-            .get::<_, Str, SerdeBincode<Token>>(&reader, &token_key)
-            .map_err(ResponseError::internal)?
-            .ok_or(ResponseError::invalid_token(format!(
-                "Api key does not exist: {}",
-                user_api_key
-            )))?;
-
-        if token_config.revoked {
-            return Err(ResponseError::invalid_token("token revoked"));
-        }
-
-        if let Some(index) = request_index {
-            if !token_config
-                .indexes
-                .iter()
-                .any(|r| match_wildcard(&r, &index))
-            {
-                return Err(ResponseError::invalid_token(
-                    "token is not allowed to access to this index",
-                ));
-            }
-        }
-
-        if token_config.expires_at < Utc::now() {
-            return Err(ResponseError::invalid_token("token expired"));
-        }
-
-        if token_config.acl.contains(&ACL::All) {
-            return Ok(());
-        }
-
-        if !token_config.acl.contains(&acl) {
-            return Err(ResponseError::invalid_token("no permission"));
-        }
-
-        Ok(())
-    }
-
-    fn header(&self, name: &str) -> Result<String, ResponseError> {
-        let header = self
-            .headers()
-            .get(name)
-            .ok_or(ResponseError::missing_header(name))?
-            .to_str()
-            .map_err(|_| ResponseError::missing_header("X-Meili-API-Key"))?
-            .to_string();
-        Ok(header)
-    }
-
-    fn url_param(&self, name: &str) -> Result<String, ResponseError> {
-        let param = self
-            .param::<String>(name)
-            .map_err(|e| ResponseError::bad_parameter(name, e))?;
-        Ok(param)
-    }
-
-    fn index(&self) -> Result<Index, ResponseError> {
-        let index_uid = self.url_param("index")?;
-        let index = self
-            .state()
-            .db
-            .open_index(&index_uid)
-            .ok_or(ResponseError::index_not_found(index_uid))?;
-        Ok(index)
-    }
-
-    fn identifier(&self) -> Result<String, ResponseError> {
-        let name = self
-            .param::<String>("identifier")
-            .map_err(|e| ResponseError::bad_parameter("identifier", e))?;
-
-        Ok(name)
-    }
-}

meilisearch-http/src/lib.rs

@@ -1,8 +1,104 @@
+#![allow(clippy::or_fun_call)]
+
 pub mod data;
 pub mod error;
 pub mod helpers;
 pub mod models;
 pub mod option;
 pub mod routes;
+pub mod analytics;
+pub mod snapshot;
+pub mod dump;
 
+use actix_http::Error;
+use actix_service::ServiceFactory;
+use actix_web::{dev, web, App};
+use chrono::Utc;
+use log::error;
+
+use meilisearch_core::{Index, MainWriter, ProcessedUpdateResult};
+
+pub use option::Opt;
 pub use self::data::Data;
+use self::error::{payload_error_handler, ResponseError};
+
+pub fn create_app(
+    data: &Data,
+    enable_frontend: bool,
+) -> App<
+    impl ServiceFactory<
+        Config = (),
+        Request = dev::ServiceRequest,
+        Response = dev::ServiceResponse<actix_http::body::Body>,
+        Error = Error,
+        InitError = (),
+    >,
+    actix_http::body::Body,
+> {
+    let app = App::new()
+        .data(data.clone())
+        .app_data(
+            web::JsonConfig::default()
+                .limit(data.http_payload_size_limit)
+                .content_type(|_mime| true) // Accept all mime types
+                .error_handler(|err, _req| payload_error_handler(err).into()),
+        )
+        .app_data(
+            web::QueryConfig::default()
+            .error_handler(|err, _req| payload_error_handler(err).into())
+        )
+        .configure(routes::document::services)
+        .configure(routes::index::services)
+        .configure(routes::search::services)
+        .configure(routes::setting::services)
+        .configure(routes::stop_words::services)
+        .configure(routes::synonym::services)
+        .configure(routes::health::services)
+        .configure(routes::stats::services)
+        .configure(routes::key::services)
+        .configure(routes::dump::services);
+    if enable_frontend {
+        app
+            .service(routes::load_html)
+            .service(routes::load_css)
+    } else {
+        app
+    }
+}
+
+pub fn index_update_callback_txn(index: Index, index_uid: &str, data: &Data, mut writer: &mut MainWriter) -> Result<(), String> {
+    if let Err(e) = data.db.compute_stats(&mut writer, index_uid) {
+        return Err(format!("Impossible to compute stats; {}", e));
+    }
+
+    if let Err(e) = data.db.set_last_update(&mut writer, &Utc::now()) {
+        return Err(format!("Impossible to update last_update; {}", e));
+    }
+
+    if let Err(e) = index.main.put_updated_at(&mut writer) {
+        return Err(format!("Impossible to update updated_at; {}", e));
+    }
+
+    Ok(())
+}
+
+pub fn index_update_callback(index_uid: &str, data: &Data, status: ProcessedUpdateResult) {
+    if status.error.is_some() {
+        return;
+    }
+
+    if let Some(index) = data.db.open_index(index_uid) {
+        let db = &data.db;
+        let res = db.main_write::<_, _, ResponseError>(|mut writer| {
+            if let Err(e) = index_update_callback_txn(index, index_uid, data, &mut writer) {
+                error!("{}", e);
+            }
+
+            Ok(())
+        });
+        match res {
+            Ok(_) => (),
+            Err(e) => error!("{}", e),
+        }
+    }
+}