Merge pull request #604 from meilisearch/personal-token-binaries

Use a personal access token to publish release binaries

.github/workflows/README.md

@@ -7,5 +7,12 @@
 ## Workflow
 
 - On each pull request, we are triggering `cargo test`.
-- On each commit on master, we are building the latest docker image.
-- On each tag, we are building the tagged docker image and the binaries for MacOS & Ubuntu.
+- On each tag, we are building:
+    - the tagged docker image
+    - the binaries for MacOS, Ubuntu, and Windows
+    - the debian package
+- On each stable release, we are build the latest docker image.
+
+## Problems
+
+- We do not test on Windows because we are unable to make it work, there is a disk space problem.

.github/workflows/check-updated-changelog.yml

@@ -0,0 +1,16 @@
+name: Check if the CHANGELOG.md has been updated
+
+on: [pull_request]
+
+jobs:
+  check:
+    name: Test on ${{ matrix.os }}
+    if: ${{ !contains(github.event.pull_request.labels.*.name, 'ignore-changelog') }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Checking the CHANGELOG.md has been updated in this PR
+      run: |
+        set -e
+        git fetch origin ${{ github.base_ref }}
+        git diff --name-only origin/${{ github.base_ref }} | grep -q CHANGELOG.md

.github/workflows/publish-binaries.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
         include:
           - os: ubuntu-latest
             artifact_name: meilisearch
@@ -19,6 +19,9 @@ jobs:
           - os: macos-latest
             artifact_name: meilisearch
             asset_name: meilisearch-macos-amd64
+          - os: windows-latest
+            artifact_name: meilisearch
+            asset_name: meilisearch-windows-amd64
 
     steps:
     - uses: hecrj/setup-rust-action@master
@@ -30,7 +33,55 @@ jobs:
     - name: Upload binaries to release
       uses: svenstaro/upload-release-action@v1-release
       with:
-        repo_token: ${{ secrets.GITHUB_TOKEN }}
+        repo_token: ${{ secrets.PUBLISH_TOKEN }}
         file: target/release/${{ matrix.artifact_name }}
         asset_name: ${{ matrix.asset_name }}
         tag: ${{ github.ref }}
+
+  publish-armv7:
+    name: Publish for ARMv7
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v1.0.0
+      - uses: uraimo/run-on-arch-action@v1.0.7
+        id: runcmd
+        with:
+          architecture: armv7
+          distribution: ubuntu18.04
+          run: |
+            apt update
+            apt install -y curl gcc make
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
+            source $HOME/.cargo/env
+            cargo build --release --locked
+      - name: Upload the binary to release
+        uses: svenstaro/upload-release-action@v1-release
+        with:
+          repo_token: ${{ secrets.PUBLISH_TOKEN }}
+          file: target/release/meilisearch
+          asset_name: meilisearch-linux-armv7
+          tag: ${{ github.ref }}
+
+  publish-armv8:
+    name: Publish for ARMv8
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v1.0.0
+      - uses: uraimo/run-on-arch-action@v1.0.7
+        id: runcmd
+        with:
+          architecture: aarch64 # aka ARMv8
+          distribution: ubuntu18.04
+          run: |
+            apt update
+            apt install -y curl gcc make
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
+            source $HOME/.cargo/env
+            cargo build --release --locked
+      - name: Upload the binary to release
+        uses: svenstaro/upload-release-action@v1-release
+        with:
+          repo_token: ${{ secrets.PUBLISH_TOKEN }}
+          file: target/release/meilisearch
+          asset_name: meilisearch-linux-armv8
+          tag: ${{ github.ref }}

.github/workflows/publish-deb-brew-pkg.yml

@@ -0,0 +1,39 @@
+name: Publish deb pkg to GitHub release & apt repository & Homebrew
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+  debian:
+    name: Publish debian packagge
+    runs-on: ubuntu-latest
+    steps:
+    - uses: hecrj/setup-rust-action@master
+      with:
+        rust-version: stable
+    - name: Install cargo-deb
+      run: cargo install cargo-deb
+    - uses: actions/checkout@v1
+    - name: Build deb package
+      run: cargo deb -p meilisearch-http -o target/debian/meilisearch.deb
+    - name: Upload debian pkg to release
+      uses: svenstaro/upload-release-action@v1-release
+      with:
+        repo_token: ${{ secrets.GITHUB_TOKEN }}
+        file: target/debian/meilisearch.deb
+        asset_name: meilisearch.deb
+        tag: ${{ github.ref }}
+    - name: Upload debian pkg to apt repository
+      run: curl -F package=@target/debian/meilisearch.deb https://${{ secrets.GEMFURY_PUSH_TOKEN }}@push.fury.io/meilisearch/
+
+  homebrew:
+    name: Bump Homebrew formula
+    runs-on: ubuntu-latest
+    steps:
+      - uses: mislav/bump-homebrew-formula-action@v1
+        with:
+          formula-name: meilisearch
+        env:
+          COMMITTER_TOKEN: ${{ secrets.HOMEBREW_COMMITTER_TOKEN }}

.github/workflows/publish-docker-latest.yml

@@ -1,8 +1,8 @@
 ---
 on:
   push:
-    branches:
-      - master
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
 
 name: Publish latest image to Docker Hub
 

.github/workflows/test.yml

@@ -1,21 +1,25 @@
 ---
 on: [pull_request]
 
-name: Cargo test
+name: Test binaries with cargo test
 
 jobs:
   check:
-    name: MeiliSearch
-    runs-on: ubuntu-latest
+    name: Test on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+
     steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - name: Run cargo test
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --locked
+    - uses: actions/checkout@v1
+    - uses: actions-rs/toolchain@v1
+      with:
+        profile: minimal
+        toolchain: stable
+        override: true
+    - name: Run cargo test
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --locked --release

.gitignore

@@ -1,6 +1,8 @@
 /target
+meilisearch-core/target
 **/*.csv
 **/*.json_lines
 **/*.rs.bk
 /*.mdb
 /query-history.txt
+/data.ms

CHANGELOG.md

@@ -0,0 +1,13 @@
+## v0.10
+
+  - Refined filtering (#592)
+  - Add the number of hits in search result (#541)
+  - Add support for aligned crop in search result (#543)
+  - Sanitize the content displayed in the web interface (#539)
+  - Add support of nested null, boolean and seq values (#571 and #568, #574)
+  - Fixed the core benchmark (#576)
+  - Publish an ARMv7 and ARMv8 binaries on releases (#540 and #581)
+  - Fixed a bug where the result of the update status after the first update was empty (#542)
+  - Fixed a bug where stop words were not handled correctly (#594)
+  - Fix CORS issues (#602)
+  - Support wildcard on attributes to retrieve, highlight, and crop (#549, #565, and #598)

Dockerfile

@@ -4,7 +4,6 @@ FROM    alpine:3.10 AS compiler
 RUN     apk update --quiet
 RUN     apk add curl
 RUN     apk add build-base
-RUN     apk add libressl-dev
 
 RUN     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
 
@@ -20,10 +19,9 @@ RUN     $HOME/.cargo/bin/cargo build --release
 FROM    alpine:3.10
 
 RUN     apk update --quiet
-RUN     apk add libressl
-RUN     apk add build-base
+RUN     apk add libgcc
 
 COPY    --from=compiler /meilisearch/target/release/meilisearch .
 
-ENV     MEILI_HTTP_ADDR 0.0.0.0:8080
+ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
 CMD     ./meilisearch

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) [year] [fullname]
+Copyright (c) 2019-2020 Meili SAS
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,165 +1,217 @@
-# MeiliSearch
+<p align="center">
+  <img src="assets/logo.svg" alt="MeiliSearch" width="200" height="200" />
+</p>
 
-[![Build Status](https://github.com/meilisearch/MeiliSearch/workflows/Cargo%20test/badge.svg)](https://github.com/meilisearch/MeiliSearch/actions)
-[![dependency status](https://deps.rs/repo/github/meilisearch/MeiliSearch/status.svg)](https://deps.rs/repo/github/meilisearch/MeiliSearch)
-[![License](https://img.shields.io/badge/license-commons%20clause-lightgrey)](https://commonsclause.com/)
+<h1 align="center">MeiliSearch</h1>
 
-Ultra relevant and instant full-text search API.
+<h4 align="center">
+  <a href="https://www.meilisearch.com">Website</a> | 
+  <a href="https://blog.meilisearch.com">Blog</a> | 
+  <a href="https://fr.linkedin.com/company/meilisearch">LinkedIn</a> | 
+  <a href="https://twitter.com/meilisearch">Twitter</a> | 
+  <a href="https://docs.meilisearch.com">Documentation</a> | 
+  <a href="https://docs.meilisearch.com/resources/faq.html">FAQ</a>
+</h4>
 
-MeiliSearch is a powerful, fast, open-source, easy to use and deploy search engine. The search and indexation are fully customizable and handles features like typo-tolerance, filters, and ranking.
+<p align="center">
+  <a href="https://github.com/meilisearch/MeiliSearch/actions"><img src="https://github.com/meilisearch/MeiliSearch/workflows/Cargo%20test/badge.svg" alt="Build Status"></a>
+  <a href="https://deps.rs/repo/github/meilisearch/MeiliSearch"><img src="https://deps.rs/repo/github/meilisearch/MeiliSearch/status.svg" alt="Dependency status"></a>
+  <a href="https://github.com/meilisearch/MeiliSearch/blob/master/LICENSE"><img src="https://img.shields.io/badge/license-MIT-informational" alt="License"></a>
+  <a href="https://slack.meilisearch.com"><img src="https://img.shields.io/badge/slack-MeiliSearch-blue.svg?logo=slack" alt="Slack"></a>
+</p>
 
-## Features
+<p align="center">⚡ Lightning Fast, Ultra Relevant, and Typo-Tolerant Search Engine 🔍</p>
 
-- Provides [6 default ranking criteria](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-core/src/criterion/mod.rs#L107-L113) used to [bucket sort](https://en.wikipedia.org/wiki/Bucket_sort) documents
-- Accepts [custom criteria](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-core/src/criterion/mod.rs#L24-L33) and can apply them in any custom order
-- Support [ranged queries](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-core/src/query_builder.rs#L283), useful for paginating results
-- Can [distinct](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-core/src/query_builder.rs#L265-L270) and [filter](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-core/src/query_builder.rs#L246-L259) returned documents based on context defined rules
-- Searches for [concatenated](https://github.com/meilisearch/MeiliSearch/pull/164) and [splitted query words](https://github.com/meilisearch/MeiliSearch/pull/232) to improve the search quality.
-- Can store complete documents or only [user schema specified fields](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-schema/src/lib.rs#L265-L279)
-- The [default tokenizer](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-tokenizer/src/lib.rs) can index latin and kanji based languages
-- Returns [the matching text areas](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-core/src/lib.rs#L66-L88), useful to highlight matched words in results
-- Accepts query time search config like the [searchable attributes](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-core/src/query_builder.rs#L272-L275)
-- Supports [runtime incremental indexing](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-core/src/store/mod.rs#L143-L173)
+**MeiliSearch** is a powerful, fast, open-source, easy to use and deploy search engine. Both searching and indexing are highly customizable. Features such as typo-tolerance, filters, and synonyms are provided out-of-the-box.
+For more information about features go to [our documentation](https://docs.meilisearch.com/).
 
-
-
-It uses [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database) as the internal key-value store. The key-value store allows us to handle updates and queries with small memory and CPU overheads. The whole ranking system is [data oriented](https://github.com/meilisearch/MeiliSearch/issues/82) and provides great performances.
-
-You can [read the deep dive](deep-dive.md) if you want more information on the engine, it describes the whole process of generating updates and handling queries or you can take a look at the [typos and ranking rules](typos-ranking-rules.md) if you want to know the default rules used to sort the documents.
-
-We will be glad if you submit issues and pull requests. You can help to grow this project and start contributing by checking [issues tagged "good-first-issue"](https://github.com/meilisearch/MeiliSearch/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22). It is a good start!
-
-[![crates.io demo gif](misc/crates-io-demo.gif)](https://crates.meilisearch.com)
+<p align="center">
+  <a href="https://crates.meilisearch.com"><img src="assets/crates-io-demo.gif" alt="crates.io demo gif" /></a>
+</p>
 
 > Meili helps the Rust community find crates on [crates.meilisearch.com](https://crates.meilisearch.com)
 
+## Features
+* Search as-you-type experience (answers < 50 milliseconds)
+* Full-text search
+* Typo tolerant (understands typos and miss-spelling)
+* Supports Kanji
+* Supports Synonym
+* Easy to install, deploy, and maintain
+* Whole documents are returned
+* Highly customizable
+* RESTful API
 
-
-## Quick Start
-
-You can deploy your own instant, relevant and typo-tolerant MeiliSearch search engine by yourself too.
-Something similar to the demo above can be achieve by following these little three steps first.
-You will need to create your own web front display to make it pretty though.
+## Get started
 
 ### Deploy the Server
 
-If you have not installed Rust and its package manager `cargo` yet, go to [the installation page](https://www.rust-lang.org/tools/install).<br/>
-You can deploy the server on your own machine, it will listen to HTTP requests on the 8080 port by default.
+#### Run it using Docker
+
+```bash
+docker run -p 7700:7700 -v $(pwd)/data.ms:/data.ms getmeili/meilisearch
+```
+
+#### Installing with Homebrew
+
+```bash
+brew update && brew install meilisearch
+meilisearch
+```
+
+#### Installing with APT
+
+```bash
+echo "deb [trusted=yes] https://apt.fury.io/meilisearch/ /" > /etc/apt/sources.list.d/fury.list
+apt update && apt install meilisearch-http
+meilisearch
+```
+
+#### Download the binary
+
+```bash
+curl -L https://install.meilisearch.com | sh
+./meilisearch
+```
+
+#### Compile and run it from sources
+
+If you have the Rust toolchain already installed on your local system, clone the repository and change it to your working directory.
+
+```bash
+git clone https://github.com/meilisearch/MeiliSearch.git
+cd MeiliSearch
+```
+
+In the cloned repository, compile MeiliSearch.
 
 ```bash
 cargo run --release
 ```
 
-For more logs during the execution, run:
-```bash
-RUST_LOG=info cargo run --release
-```
-
 ### Create an Index and Upload Some Documents
 
-MeiliSearch can serve multiple indexes, with different kinds of documents,
-therefore, it is required to create the index before sending documents to it.
+Let's create an index! If you need a sample dataset, use [this movie database](https://www.notion.so/meilisearch/A-movies-dataset-to-test-Meili-1cbf7c9cfa4247249c40edfa22d7ca87#b5ae399b81834705ba5420ac70358a65). You can also find it in the `datasets/` directory.
 
 ```bash
-curl -i -X POST 'http://127.0.0.1:8080/indexes' --data '{ "name": "Movies", "uid": "movies" }'
+curl -L 'https://bit.ly/2PAcw9l' -o movies.json
 ```
 
-Now that the server knows about our brand new index, we can send it data.
-We provided you a little dataset, it is available in the `datasets/` directory.
+MeiliSearch can serve multiple indexes, with different kinds of documents.
+It is required to create an index before sending documents to it.
 
 ```bash
-curl -i -X POST 'http://127.0.0.1:8080/indexes/movies/documents' \
+curl -i -X POST 'http://127.0.0.1:7700/indexes' --data '{ "name": "Movies", "uid": "movies" }'
+```
+
+Now that the server knows about your brand new index, you're ready to send it some data.
+
+```bash
+curl -i -X POST 'http://127.0.0.1:7700/indexes/movies/documents' \
   --header 'content-type: application/json' \
-  --data @datasets/movies/movies.json
+  --data-binary @movies.json
 ```
 
 ### Search for Documents
 
-The search engine is now aware of our documents and can serve those via our HTTP server again.
-The [`jq` command line tool](https://stedolan.github.io/jq/) can greatly help you read the server responses.
+#### In command line
+
+The search engine is now aware of your documents and can serve those via an HTTP server. 
+
+The [`jq` command-line tool](https://stedolan.github.io/jq/) can greatly help you read the server responses.
 
 ```bash
-curl 'http://127.0.0.1:8080/indexes/movies/search?q=botman'
+curl 'http://127.0.0.1:7700/indexes/movies/search?q=botman+robin&limit=2' | jq
 ```
 
 ```json
 {
   "hits": [
     {
-      "id": "29751",
-      "title": "Batman Unmasked: The Psychology of the Dark Knight",
-      "poster": "https://image.tmdb.org/t/p/w1280/jjHu128XLARc2k4cJrblAvZe0HE.jpg",
-      "overview": "Delve into the world of Batman and the vigilante justice tha",
-      "release_date": "2008-07-15"
+      "id": "415",
+      "title": "Batman & Robin",
+      "poster": "https://image.tmdb.org/t/p/w1280/79AYCcxw3kSKbhGpx1LiqaCAbwo.jpg",
+      "overview": "Along with crime-fighting partner Robin and new recruit Batgirl...",
+      "release_date": "1997-06-20",
     },
     {
-      "id": "471474",
-      "title": "Batman: Gotham by Gaslight",
-      "poster": "https://image.tmdb.org/t/p/w1280/7souLi5zqQCnpZVghaXv0Wowi0y.jpg",
-      "overview": "ve Victorian Age Gotham City, Batman begins his war on crime",
-      "release_date": "2018-01-12"
+      "id": "411736",
+      "title": "Batman: Return of the Caped Crusaders",
+      "poster": "https://image.tmdb.org/t/p/w1280/GW3IyMW5Xgl0cgCN8wu96IlNpD.jpg",
+      "overview": "Adam West and Burt Ward returns to their iconic roles of Batman and Robin...",
+      "release_date": "2016-10-08",
     }
   ],
   "offset": 0,
   "limit": 2,
   "processingTimeMs": 1,
-  "query": "botman"
+  "query": "botman robin"
 }
 ```
 
+#### Use the Web Interface
 
+We also deliver an **out-of-the-box web interface** in which you can test MeiliSearch interactively.
 
-## Performances
+You can access the web interface in your web browser at the root of the server. The default URL is [http://127.0.0.1:7700](http://127.0.0.1:7700). All you need to do is open your web browser and enter MeiliSearch’s address to visit it. This will lead you to a web page with a search bar that will allow you to search in the selected index.
 
-With a dataset composed of _100 353_ documents with _352_ attributes each and _3_ of them indexed.
-So more than _300 000_ fields indexed for _35 million_ stored we can handle more than _2.8k req/sec_ with an average response time of _9 ms_ on an Intel i7-7700 (8) @ 4.2GHz.
+<p align="center">
+  <img src="assets/movies-web-demo.gif" alt="Web interface gif" />
+</p>
 
-Requests are made using [wrk](https://github.com/wg/wrk) and scripted to simulate real users queries.
+### Documentation
+
+Now that your MeiliSearch server is up and running, you can learn more about how to tune your search engine in [the documentation](https://docs.meilisearch.com).
+
+### Technical features
+
+- Provides [6 default ranking criteria](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/criterion/mod.rs#L106-L111) used to [bucket sort](https://en.wikipedia.org/wiki/Bucket_sort) documents
+- Accepts [custom criteria](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/criterion/mod.rs#L20-L29) and can apply them in any custom order
+- Supports [ranged queries](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/query_builder.rs#L342), useful for paginating results
+- Can [distinct](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/query_builder.rs#L324-L329) and [filter](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/query_builder.rs#L313-L318) returned documents based on context defined rules
+- Searches for [concatenated](https://github.com/meilisearch/MeiliSearch/pull/164) and [splitted query words](https://github.com/meilisearch/MeiliSearch/pull/232) to improve the search quality.
+- Can store complete documents or only [user schema specified fields](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/datasets/movies/schema.toml)
+- The [default tokenizer](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-tokenizer/src/lib.rs) can index Latin and Kanji based languages
+- Returns [the matching text areas](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-types/src/lib.rs#L49-L65), useful to highlight matched words in results
+- Accepts query time search config like the [searchable attributes](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/query_builder.rs#L331-L336)
+- Supports [runtime incremental indexing](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/store/mod.rs#L143-L212)
+
+## Performance
+
+When processing a dataset composed of 5M books, each with their own titles and authors, MeiliSearch is able to carry out more than 553 req/sec with an average response time of 21 ms on an Intel i7-7700 (8) @ 4.2GHz.
+
+Requests are made using [wrk](https://github.com/wg/wrk) and scripted to simulate real users' queries.
 
 ```
-Running 10s test @ http://localhost:2230
-  2 threads and 25 connections
+Running 10s test @ http://1.2.3.4:7700
+  2 threads and 10 connections
   Thread Stats   Avg      Stdev     Max   +/- Stdev
-    Latency     9.52ms    7.61ms  99.25ms   84.58%
-    Req/Sec     1.41k   119.11     1.78k    64.50%
-  28080 requests in 10.01s, 7.42MB read
-Requests/sec:   2806.46
-Transfer/sec:    759.17KB
+    Latency    21.45ms   15.64ms 214.10ms   85.95%
+    Req/Sec   256.48     37.66   330.00     69.50%
+  5132 requests in 10.05s, 2.31MB read
+Requests/sec:    510.46
+Transfer/sec:    234.77KB
 ```
 
-We also indexed a dataset containing something like _12 millions_ cities names in _24 minutes_ on a machine with _8 cores_, _64 GB of RAM_ and a _300 GB NMVe_ SSD.<br/>
-The resulting database was _16 GB_ and search results were between _30 ms_ and _4 seconds_ for short prefix queries.
+We also indexed a dataset containing about _12 millions_ cities names in _24 minutes_ on a _8 cores_, _64 GB of RAM_, and a _300 GB NMVe_ SSD machine.<br/>
+The size of the resulting database reached _16 GB_ and search results were presented between _30 ms_ and _4 seconds_ for short prefix queries.
 
-### Notes
+## Contributing
 
-With Rust 1.32 the allocator has been [changed to use the system allocator](https://blog.rust-lang.org/2019/01/17/Rust-1.32.0.html#jemalloc-is-removed-by-default).
-We have seen much better performances when [using jemalloc as the global allocator](https://github.com/alexcrichton/jemallocator#documentation).
-
-## Usage and Examples
-
-MeiliSearch also provides an example binary that is mostly used for features testing.
-Notice that the example binary is faster to index data as it does read direct CSV files and not JSON HTTP payloads.
-
-The _index_ subcommand has been made to create an index and inject documents into it. Using the command line below, the index will be named _movies_ and the _19 700_ movies of the `datasets/` will be injected in MeiliSearch.
-
-```bash
-cargo run --release --example from_file -- \
-    index example.mdb datasets/movies/movies.csv \
-    --schema datasets/movies/schema.toml
-```
-
-Once the first command is done, you can query the freshly created _movies_ index using the _search_ subcomand. In this example we filtered the dataset to only show _non-adult_ movies using the non-definitive `!adult` syntax filter.
-
-```bash
-cargo run --release --example from_file -- \
-    search example.mdb \
-    --number-results 4 \
-    --filter '!adult' \
-    id popularity adult original_title
-```
+Hey! We're glad you're thinking about contributing to MeiliSearch! If you think something is missing or could be improved, please open issues and pull requests. If you'd like to help this project grow, we'd love to have you! To start contributing, checking [issues tagged as "good-first-issue"](https://github.com/meilisearch/MeiliSearch/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) is a good start!
 
 ### Analytic Events
 
-We send events to our Amplitude instance to be aware of the number of people who use MeiliSearch.<br/>
-We only send the platform on which the server runs once by day. No other information is sent.<br/>
-If you do not want us to send events, you can disable these analytics by using the `MEILI_NO_ANALYTICS` env variable.
+Once a day, events are being sent to our Amplitude instance so we can know how many people are using MeiliSearch.<br/>
+Only information about the platform on which the server runs is stored. No other information is being sent.<br/>
+If this doesn't suit you, you can disable these analytics by using the `MEILI_NO_ANALYTICS` env variable.
+
+## Contact
+
+Feel free to contact us about any questions you may have:
+* At [bonjour@meilisearch.com](mailto:bonjour@meilisearch.com): English or French is welcome! 🇬🇧 🇫🇷
+* Via the chat box available on every page of [our documentation](https://docs.meilisearch.com/) and on [our landing page](https://www.meilisearch.com/).
+* Join our [Slack community](https://slack.meilisearch.com/).
+* By opening an issue.
+
+Any suggestion or feedback is highly appreciated. Thank you for your support!

assets/logo.svg

@@ -0,0 +1,17 @@
+<svg width="360" height="360" viewBox="0 0 360 360" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g id="logo_main">
+<rect id="Rectangle" x="107.333" y="0.150146" width="274.315" height="274.315" rx="98.8334" transform="rotate(23 107.333 0.150146)" fill="url(#paint0_linear)"/>
+<path id="Rectangle_2" fill-rule="evenodd" clip-rule="evenodd" d="M61.3296 230.199C46.2224 194.608 38.6688 176.813 38.208 160.329C37.5286 136.025 47.0175 112.539 64.3891 95.5282C76.1718 83.9904 93.9669 76.4368 129.557 61.3296C165.147 46.2224 182.943 38.6688 199.427 38.208C223.731 37.5286 247.217 47.0175 264.228 64.3891C275.766 76.1718 283.319 93.9669 298.426 129.557C313.534 165.147 321.087 182.943 321.548 199.427C322.227 223.731 312.738 247.217 295.367 264.228C283.584 275.766 265.789 283.319 230.199 298.426C194.608 313.534 176.813 321.087 160.329 321.548C136.025 322.227 112.539 312.738 95.5282 295.367C83.9903 283.584 76.4368 265.789 61.3296 230.199Z" fill="url(#paint1_linear)"/>
+<path id="m" fill-rule="evenodd" clip-rule="evenodd" d="M219.568 130.748C242.363 130.748 259.263 147.451 259.263 174.569V229.001H227.232V179.678C227.232 166.119 220.747 159.634 210.136 159.634C205.223 159.634 200.311 161.796 195.595 167.494C195.791 169.852 195.988 172.21 195.988 174.569V229.001H164.154V179.678C164.154 166.119 157.472 159.634 147.057 159.634C142.145 159.634 137.429 161.992 132.712 168.084V229.001H100.878V133.695H132.712V139.394C139.197 133.892 145.878 130.748 156.49 130.748C168.477 130.748 178.695 135.267 185.769 143.52C195.791 134.678 205.42 130.748 219.568 130.748Z" fill="white"/>
+</g>
+<defs>
+<linearGradient id="paint0_linear" x1="-13.6248" y1="129.208" x2="244.49" y2="403.522" gradientUnits="userSpaceOnUse">
+<stop stop-color="#E41359"/>
+<stop offset="1" stop-color="#F23C79"/>
+</linearGradient>
+<linearGradient id="paint1_linear" x1="11.0088" y1="111.65" x2="111.65" y2="348.747" gradientUnits="userSpaceOnUse">
+<stop stop-color="#24222F"/>
+<stop offset="1" stop-color="#2B2937"/>
+</linearGradient>
+</defs>
+</svg>

datasets/movies/movies.json

@@ -15001,7 +15001,6 @@
 {"id":"84354","title":"Young & Wild","poster":"https://image.tmdb.org/t/p/w1280/4PXOIrtZfj38ACyvbv8DI3kEGym.jpg","overview":"Daniela, raised in the bosom of a strict Evangelical family and recently unmasked as a fornicator by her shocked parents, struggles to find her own path to spiritual harmony.","release_date":1332982800},
 {"id":"222230","title":"Angelique","poster":"https://image.tmdb.org/t/p/w1280/ilxsOXHDsDykyfYrOU8VTioIvrL.jpg","overview":"The incredible destiny of Angélique: a beautiful girl who found in her love for Joffrey Peyrac the strength to fight injustice and submission in a century plagued by power struggles, inequality and the oppression.","release_date":1387324800},
 {"id":"480434","title":"Dreamland","poster":"https://image.tmdb.org/t/p/w1280/2a3MTUtH7yuta6SPRGjx4sUCeMB.jpg","overview":"Amid the dust storms and economic depression of Dustbowl Era Oklahoma, Eugene Evans finds his family farm on the brink of foreclosure. His last bet to save the farm is the bounty on the head of fugitive bank robber Allison Wells.","release_date":1556240400},
-{"id":"279664","title":"Green Chair 2013 - Love Conceptually","poster":"https://image.tmdb.org/t/p/w1280/9H8vACDBSBs8NXO6NfLH3i9ZsYq.jpg","overview":"Moon-hee is an attractive woman who runs an art academy after returning from the States. She lives apart from her husband who won't divorce her and enjoys a free relationship with In-gyu, her lover of long time and Professor Yoon whom she's been privately involved with for a long time. On the other hand, her young student Joo Won only draws her face during class. He fell in love with her when he saw Moon-hee as a bride at the wedding his grandma took him too. He started art because of her and started attending the academy. He thinks it's destiny. Moon-hee is attracted to Joo Won's pure heart until they share love in in a studio where it's just them....","release_date":1383177600},
 {"id":"39862","title":"Crocodile 2: Death Swamp","poster":"https://image.tmdb.org/t/p/w1280/5NKUMW6qbNna8yF7RPnkHkuq6bi.jpg","overview":"After a bank robbery, four criminals escape to Mexico, but a storm causes an accident which takes down the plane where several die in the crash. The criminals take it into their own hands to continue when one survivor is attacked and eaten by a crocodile. The criminals kill it, but from then on the mother Crocodile is on a killing spree with a goal to kill each survivor. But that is not the only worry, because they're trapped within it's world, and if it doesn't kill them, the criminals will.","release_date":1009843200},
 {"id":"80188","title":"Boy Wonder","poster":"https://image.tmdb.org/t/p/w1280/nVdZgKJZyn7nmPjhfVKnsMktudy.jpg","overview":"A young Brooklyn boy witnesses the brutal murder of his mother and grows up obsessed with finding her killer. Thus begins his life as a quiet, straight-A student by day and a self-appointed hero at night. But what is a real hero? And who decides what is right or wrong? As the boundaries blur, Sean's dual life wears on his psyche and his two worlds careen dangerously close to colliding. Like a graphic novel you can't put down, Boy Wonder challenges morality, distorting perceptions of what is right and what is justified, as it races to its shocking conclusion.","release_date":1281488400},
 {"id":"554142","title":"Cosa fai a Capodanno?","poster":"https://image.tmdb.org/t/p/w1280/aTnQp96wPRwezLpKt2dp5jvE2Vd.jpg","overview":"","release_date":1542240000},

datasets/movies/schema.toml

@@ -1,21 +0,0 @@
-# This schema has been generated ...
-# The order in which the attributes are declared is important,
-# it specify the attribute xxx...
-identifier = "id"
-
-[attributes.id]
-displayed = true
-
-[attributes.title]
-displayed = true
-indexed = true
-
-[attributes.overview]
-displayed = true
-indexed = true
-
-[attributes.release_date]
-displayed = true
-
-[attributes.poster]
-displayed = true

datasets/movies/settings.json

@@ -0,0 +1,10 @@
+{
+    "searchableAttributes": ["title", "overview"],
+    "displayedAttributes": [
+        "id",
+        "title",
+        "overview",
+        "release_date",
+        "poster"
+    ]
+}

deep-dive.md

@@ -1,95 +0,0 @@
-# A deep dive in MeiliSearch
-
-On the 15 of May 2019.
-
-MeiliSearch is a full text search engine based on a final state transducer named [fst](https://github.com/BurntSushi/fst) and a key-value store named [sled](https://github.com/spacejam/sled). The goal of a search engine is to store data and to respond to queries as accurate and fast as possible. To achieve this it must save the matching words in an [inverted index](https://en.wikipedia.org/wiki/Inverted_index).
-
-<!-- MarkdownTOC autolink="true" -->
-
-- [Where is the data stored?](#where-is-the-data-stored)
-- [What does the key-value store contains?](#what-does-the-key-value-store-contains)
-    - [The inverted word index](#the-inverted-word-index)
-        - [A final state transducer](#a-final-state-transducer)
-        - [Document indexes](#document-indexes)
-    - [The schema](#the-schema)
-    - [Document attributes](#document-attributes)
-- [How is a request processed?](#how-is-a-request-processed)
-    - [Query lexemes](#query-lexemes)
-    - [Automatons and query index](#automatons-and-query-index)
-    - [Sort by criteria](#sort-by-criteria)
-
-<!-- /MarkdownTOC -->
-
-## Where is the data stored?
-
-MeiliSearch is entirely backed by a key-value store like any good database (i.e. Postgres, MySQL). This brings a great flexibility in the way documents can be stored and updates handled along time.
-
-[sled will brings some](https://github.com/spacejam/sled/tree/434533332a3f485e6d2e467023be0a0b55d3a1af#plans) of the [A.C.I.D. properties](https://en.wikipedia.org/wiki/ACID_(computer_science)) to help us be sure the saved data is consistent.
-
-
-
-## What does the key-value store contains?
-
-It contain the inverted word index, the schema and the documents fields.
-
-### The inverted word index
-
-[The inverted word index](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/words_index.rs) is a sled Tree dedicated to store and give access to all documents that contains a specific word. The information stored under the word is simply a big ordered array of where in the document the word has been found. In other word, a big list of [`DocIndex`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/lib.rs#L35-L51).
-
-#### A final state transducer
-
-_...also abbreviated fst_
-
-This is the first entry point of the engine, you can read more about how it work with the beautiful blog post of @BurntSushi, [Index 1,600,000,000 Keys with Automata and Rust](https://blog.burntsushi.net/transducers/).
-
-To make it short it is a powerful way to store all the words that are present in the indexed documents. You construct it by giving it all the words you want to index. When you want to search in it you can provide any automaton you want, in MeiliSearch [a custom levenshtein automaton](https://github.com/tantivy-search/levenshtein-automata/) is used.
-
-#### Document indexes
-
-The `fst` will only return the words that match with the search automaton but the goal of the search engine is to retrieve all matches in all the documents when a query is made. You want it to return some sort of position in an attribute in a document, an information about where the given word matched.
-
-To make it possible we retrieve all of the `DocIndex` corresponding to all the matching words in the fst, we use the [`WordsIndex`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/words_index.rs#L11-L21) Tree to get the `DocIndexes` corresponding the words.
-
-### The schema
-
-The schema is a data structure that represents which documents attributes should be stored and which should be indexed. It is stored under a the [`MainIndex`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/main_index.rs#L12) Tree and given to MeiliSearch only at the creation of an index.
-
-Each document attribute is associated to a unique 16 bit number named [`SchemaAttr`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/schema.rs#L186).
-
-In the future, this schema type could be given along with updates, the database could be able to handled a new schema and reindex the database according to the new one.
-
-### Document attributes
-
-When the engine handle a query the result that the requester want is a document, not only the [`Matches`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/lib.rs#L62-L88) associated to it, fields of the original document must be returned too.
-
-So MeiliSearch again uses the power of the underlying key-value store and save the documents attributes marked as _STORE_ in the schema. The dedicated Tree for this information is the [`DocumentsIndex`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/documents_index.rs#L11).
-
-When a document field is saved in the key-value store its value is binary encoded using [message pack](https://github.com/3Hren/msgpack-rust), so a document must be serializable using serde.
-
-
-
-## How is a request processed?
-
-Now that we have our inverted index we are able to return results based on a query. In the MeiliSearch universe a query is a simple string containing words.
-
-### Query lexemes
-
-The first step to be able to call the underlying structures is to split the query in words, for that we use a [custom tokenizer](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-tokenizer/src/lib.rs#L82-L84). Note that a tokenizer is specialized for a human language, this is the hard part.
-
-### Automatons and query index
-
-So to query the fst we need an automaton, in MeiliSearch we use a [levenshtein automaton](https://en.wikipedia.org/wiki/Levenshtein_automaton), this automaton is constructed using a string and a maximum distance. According to the [Algolia's blog post](https://blog.algolia.com/inside-the-algolia-engine-part-3-query-processing/#algolia%e2%80%99s-way-of-searching-for-alternatives) we [created the DFAs](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/automaton.rs#L59-L78) with different settings.
-
-Thanks to the power of the fst library [it is possible to union multiple automatons](https://docs.rs/fst/0.3.2/fst/map/struct.OpBuilder.html#method.union) on the same fst set. The `Stream` is able to return all the matching words. We use these words to find the whole list of `DocIndexes` associated.
-
-With all these informations it is possible [to reconstruct a list of all the `DocIndexes` associated](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/query_builder.rs#L103-L130) with the words queried.
-
-### Sort by criteria
-
-Now that we are able to get a big list of [DocIndexes](https://github.com/Kerollmops/MeiliSearch/blob/550dc1e99224e386516877450320f694947332d4/src/lib.rs#L21-L36) it is not enough to sort them by criteria, we need more informations like the levenshtein distance or the fact that a query word match exactly the word stored in the fst. So [we stuff it a little bit](https://github.com/Kerollmops/MeiliSearch/blob/550dc1e99224e386516877450320f694947332d4/src/rank/query_builder.rs#L86-L93), and aggregate all these [Matches](https://github.com/Kerollmops/MeiliSearch/blob/550dc1e99224e386516877450320f694947332d4/src/lib.rs#L47-L74) for each document. This way it will be easy to sort a simple vector of document using a bunch of functions.
-
-With this big list of documents and associated matches [we are able to sort only the part of the slice that we want](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/query_builder.rs#L160-L188) using bucket sorting. [Each criterion](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/criterion/mod.rs#L95-L101) is evaluated on each subslice without copy, thanks to [GroupByMut](https://docs.rs/slice-group-by/0.2.4/slice_group_by/) which, I hope [will soon be merged](https://github.com/rust-lang/rfcs/pull/2477).
-
-Note that it is possible to customize the criteria used by using the `QueryBuilder::with_criteria` constructor, this way you can implement some custom ranking based on the document attributes using the appropriate structure and the [`document` method](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/index.rs#L86).
-
-At this point, MeiliSearch work is over 🎉

download-latest.sh

@@ -0,0 +1,129 @@
+#!/bin/sh
+
+# COLORS
+RED='\033[31m'
+GREEN='\033[32m'
+DEFAULT='\033[0m'
+
+# GLOBALS
+GREP_SEMVER_REGEXP='\"v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\"' # i.e. "v[number].[number].[number]"
+BINARY_NAME='meilisearch'
+
+# semverParseInto and semverLT from https://github.com/cloudflare/semver_bash/blob/master/semver.sh
+
+# usage: semverParseInto version major minor patch special
+# version: the string version
+# major, minor, patch, special: will be assigned by the function
+semverParseInto() {
+    local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)'
+    #MAJOR
+    eval $2=`echo $1 | sed -e "s#$RE#\1#"`
+    #MINOR
+    eval $3=`echo $1 | sed -e "s#$RE#\2#"`
+    #MINOR
+    eval $4=`echo $1 | sed -e "s#$RE#\3#"`
+    #SPECIAL
+    eval $5=`echo $1 | sed -e "s#$RE#\4#"`
+}
+
+# usage: semverLT version1 version2
+semverLT() {
+    local MAJOR_A=0
+    local MINOR_A=0
+    local PATCH_A=0
+    local SPECIAL_A=0
+
+    local MAJOR_B=0
+    local MINOR_B=0
+    local PATCH_B=0
+    local SPECIAL_B=0
+
+    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
+    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
+
+    if [ $MAJOR_A -lt $MAJOR_B ]; then
+        return 0
+    fi
+    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -lt $MINOR_B ]; then
+        return 0
+    fi
+    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then
+        return 0
+    fi
+    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+        return 1
+    fi
+    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  != "_" ] ; then
+        return 1
+    fi
+    if [ "_$SPECIAL_A"  != "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+        return 0
+    fi
+    if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then
+        return 0
+    fi
+
+    return 1
+}
+
+success_usage() {
+    printf "$GREEN%s\n$DEFAULT" "MeiliSearch binary successfully downloaded as '$BINARY_NAME' file."
+    echo ''
+    echo 'Run it:'
+    echo '    $ ./meilisearch'
+    echo 'Usage:'
+    echo '    $ ./meilisearch --help'
+}
+
+failure_usage() {
+    printf "$RED%s\n$DEFAULT" 'ERROR: MeiliSearch binary is not available for your OS distribution yet.'
+    echo ''
+    echo 'However, you can easily compile the binary from the source files.'
+    echo 'Follow the steps on the docs: https://docs.meilisearch.com/advanced_guides/binary.html#how-to-compile-meilisearch'
+}
+
+# OS DETECTION
+echo 'Detecting OS distribution...'
+os_name=$(uname -s)
+if [ "$os_name" != "Darwin" ]; then
+    os_name=$(cat /etc/os-release | grep '^ID=' | tr -d '"' | cut -d '=' -f 2)
+fi
+echo "OS distribution detected: $os_name"
+case "$os_name" in
+'Darwin')
+    os='macos'
+    ;;
+'ubuntu' | 'debian')
+    os='linux'
+    ;;
+*)
+    failure_usage
+    exit 1
+esac
+
+# GET LATEST VERSION
+tags=$(curl -s 'https://api.github.com/repos/meilisearch/MeiliSearch/tags' \
+    | grep "$GREP_SEMVER_REGEXP" \
+    | grep 'name' \
+    | tr -d '"' | tr -d ',' | cut -d 'v' -f 2)
+
+latest=""
+for tag in $tags; do
+    if [ "$latest" = "" ]; then
+        latest="$tag"
+    else
+        semverLT $tag $latest
+        if [ $? -eq 1 ]; then
+            latest="$tag"
+        fi
+    fi
+done
+
+# DOWNLOAD THE LATEST
+echo "Downloading MeiliSearch binary v$latest for $os..."
+release_file="meilisearch-$os-amd64"
+link="https://github.com/meilisearch/MeiliSearch/releases/download/v$latest/$release_file"
+curl -OL "$link"
+mv "$release_file" "$BINARY_NAME"
+chmod 744 "$BINARY_NAME"
+success_usage

meilisearch-core/Cargo.toml

@@ -1,40 +1,53 @@
 [package]
 name = "meilisearch-core"
-version = "0.8.1"
+version = "0.10.0"
+license = "MIT"
 authors = ["Kerollmops <clement@meilisearch.com>"]
 edition = "2018"
 
 [dependencies]
-arc-swap = "0.4.3"
-bincode = "1.1.4"
-byteorder = "1.3.2"
-chrono = { version = "0.4.9", features = ["serde"] }
-crossbeam-channel = "0.4.0"
-deunicode = "1.0.0"
-env_logger = "0.7.0"
+arc-swap = "0.4.5"
+bincode = "1.2.1"
+byteorder = "1.3.4"
+chrono = { version = "0.4.11", features = ["serde"] }
+compact_arena = "0.4.0"
+crossbeam-channel = "0.4.2"
+deunicode = "1.1.0"
+env_logger = "0.7.1"
 fst = { version = "0.3.5", default-features = false }
-hashbrown = { version = "0.6.0", features = ["serde"] }
-heed = "0.6.0"
+hashbrown = { version = "0.7.1", features = ["serde"] }
+heed = "0.7.0"
+indexmap = { version = "1.3.2", features = ["serde-1"] }
+intervaltree = "0.2.5"
+itertools = "0.9.0"
 levenshtein_automata = { version = "0.1.1", features = ["fst_automaton"] }
 log = "0.4.8"
-meilisearch-schema = { path = "../meilisearch-schema", version = "0.8.1" }
-meilisearch-tokenizer = { path = "../meilisearch-tokenizer", version = "0.8.1" }
-meilisearch-types = { path = "../meilisearch-types", version = "0.8.1" }
-once_cell = "1.2.0"
+meilisearch-schema = { path = "../meilisearch-schema", version = "0.10.0" }
+meilisearch-tokenizer = { path = "../meilisearch-tokenizer", version = "0.10.0" }
+meilisearch-types = { path = "../meilisearch-types", version = "0.10.0" }
+once_cell = "1.3.1"
 ordered-float = { version = "1.0.2", features = ["serde"] }
-sdset = "0.3.3"
-serde = { version = "1.0.101", features = ["derive"] }
-serde_json = "1.0.41"
-siphasher = "0.3.1"
+pest = { git = "https://github.com/MarinPostma/pest.git", tag = "meilisearch-patch1" }
+pest_derive = "2.0"
+regex = "1.3.6"
+sdset = "0.4.0"
+serde = { version = "1.0.105", features = ["derive"] }
+serde_json = "1.0.50"
+siphasher = "0.3.2"
 slice-group-by = "0.2.6"
-zerocopy = "0.2.8"
+unicase = "2.6.0"
+zerocopy = "0.3.0"
 
 [dev-dependencies]
-assert_matches = "1.3"
-csv = "1.0.7"
-indexmap = { version = "1.2.0", features = ["serde-1"] }
-rustyline = { version = "5.0.0", default-features = false }
-structopt = "0.3.2"
+assert_matches = "1.3.0"
+criterion = "0.3.1"
+csv = "1.1.3"
+jemallocator = "0.3.2"
+rustyline = { version = "6.0.0", default-features = false }
+structopt = "0.3.12"
 tempfile = "3.1.0"
-termcolor = "1.0.4"
-toml = "0.5.3"
+termcolor = "1.1.0"
+
+[[bench]]
+name = "search_benchmark"
+harness = false

meilisearch-core/benches/search_benchmark.rs

@@ -0,0 +1,104 @@
+#[cfg(test)]
+#[macro_use]
+extern crate assert_matches;
+
+use std::sync::mpsc;
+use std::path::Path;
+use std::fs::File;
+use std::io::BufReader;
+use std::iter;
+
+use meilisearch_core::Database;
+use meilisearch_core::{ProcessedUpdateResult, UpdateStatus};
+use meilisearch_core::settings::{Settings, SettingsUpdate};
+use meilisearch_schema::Schema;
+use serde_json::Value;
+
+use criterion::{criterion_group, criterion_main, Criterion, BenchmarkId};
+
+fn prepare_database(path: &Path) -> Database {
+    let database = Database::open_or_create(path).unwrap();
+    let db = &database;
+
+    let (sender, receiver) = mpsc::sync_channel(100);
+    let update_fn = move |_name: &str, update: ProcessedUpdateResult| {
+        sender.send(update.update_id).unwrap()
+    };
+    let index = database.create_index("bench").unwrap();
+
+    database.set_update_callback(Box::new(update_fn));
+    
+    let mut writer = db.main_write_txn().unwrap();
+    index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
+    writer.commit().unwrap();
+
+    let settings_update: SettingsUpdate = {
+        let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../datasets/movies/settings.json");
+        let file = File::open(path).unwrap();
+        let reader = BufReader::new(file);
+        let settings: Settings = serde_json::from_reader(reader).unwrap();
+        settings.into_update().unwrap()
+    };
+
+    let mut update_writer = db.update_write_txn().unwrap();
+    let _update_id = index.settings_update(&mut update_writer, settings_update).unwrap();
+    update_writer.commit().unwrap();
+
+    let mut additions = index.documents_addition();
+
+    let json: Value = {
+        let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../datasets/movies/movies.json");
+        let movies_file = File::open(path).expect("find movies");
+        serde_json::from_reader(movies_file).unwrap()
+    };
+
+    let documents = json.as_array().unwrap();
+
+    for document in documents {
+        additions.update_document(document);
+    }
+
+    let mut update_writer = db.update_write_txn().unwrap();
+    let update_id = additions.finalize(&mut update_writer).unwrap();
+    update_writer.commit().unwrap();
+
+    // block until the transaction is processed
+    let _ = receiver.into_iter().find(|id| *id == update_id);
+
+    let update_reader = db.update_read_txn().unwrap();
+    let result = index.update_status(&update_reader, update_id).unwrap();
+    assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
+
+    database
+}
+
+pub fn criterion_benchmark(c: &mut Criterion) {
+    let dir = tempfile::tempdir().unwrap();
+    let database = prepare_database(dir.path());
+
+    let reader = database.main_read_txn().unwrap();
+    let index = database.open_index("bench").unwrap();
+
+    let mut count = 0;
+    let query = "I love paris ";
+
+    let iter = iter::from_fn(|| {
+        count += 1;
+        query.get(0..count)
+    });
+
+    let mut group = c.benchmark_group("searching in movies (19654 docs)");
+    group.sample_size(10);
+
+    for query in iter {
+        let bench_name = BenchmarkId::from_parameter(format!("{:?}", query));
+        group.bench_with_input(bench_name, &query, |b, query| b.iter(|| {
+            let builder = index.query_builder();
+            builder.query(&reader, query, 0..20).unwrap();
+        }));
+    }
+    group.finish();
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

meilisearch-core/examples/from_file.rs

@@ -1,7 +1,7 @@
-use std::collections::btree_map::{BTreeMap, Entry};
 use std::collections::HashSet;
+use std::collections::btree_map::{BTreeMap, Entry};
 use std::error::Error;
-use std::io::Write;
+use std::io::{Read, Write};
 use std::iter::FromIterator;
 use std::path::{Path, PathBuf};
 use std::time::{Duration, Instant};
@@ -13,7 +13,12 @@ use structopt::StructOpt;
 use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
 
 use meilisearch_core::{Database, Highlight, ProcessedUpdateResult};
-use meilisearch_schema::SchemaAttr;
+use meilisearch_core::settings::Settings;
+use meilisearch_schema::FieldId;
+
+// #[cfg(target_os = "linux")]
+#[global_allocator]
+static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
 
 #[derive(Debug, StructOpt)]
 struct IndexCommand {
@@ -24,13 +29,13 @@ struct IndexCommand {
     #[structopt(long, default_value = "default")]
     index_uid: String,
 
-    /// The csv file to index.
+    /// The csv file path to index, you can also use `-` to specify the standard input.
     #[structopt(parse(from_os_str))]
     csv_data_path: PathBuf,
 
-    /// The path to the schema.
+    /// The path to the settings.
     #[structopt(long, parse(from_os_str))]
-    schema: PathBuf,
+    settings: PathBuf,
 
     #[structopt(long)]
     update_group_size: Option<usize>,
@@ -115,27 +120,23 @@ fn index_command(command: IndexCommand, database: Database) -> Result<(), Box<dy
 
     let db = &database;
 
-    let schema = {
-        let string = fs::read_to_string(&command.schema)?;
-        toml::from_str(&string).unwrap()
+    let settings = {
+        let string = fs::read_to_string(&command.settings)?;
+        let settings: Settings = serde_json::from_str(&string).unwrap();
+        settings.into_update().unwrap()
     };
 
-    let reader = db.main_read_txn().unwrap();
     let mut update_writer = db.update_write_txn().unwrap();
-    match index.main.schema(&reader)? {
-        Some(current_schema) => {
-            if current_schema != schema {
-                return Err(meilisearch_core::Error::SchemaDiffer.into());
-            }
-            update_writer.abort();
-        }
-        None => {
-            index.schema_update(&mut update_writer, schema)?;
-            update_writer.commit().unwrap();
-        }
-    }
+    index.settings_update(&mut update_writer, settings)?;
+    update_writer.commit().unwrap();
+
+    let mut rdr = if command.csv_data_path.as_os_str() == "-" {
+        csv::Reader::from_reader(Box::new(io::stdin()) as Box<dyn Read>)
+    } else {
+        let file = std::fs::File::open(command.csv_data_path)?;
+        csv::Reader::from_reader(Box::new(file) as Box<dyn Read>)
+    };
 
-    let mut rdr = csv::Reader::from_path(command.csv_data_path)?;
     let mut raw_record = csv::StringRecord::new();
     let headers = rdr.headers()?.clone();
 
@@ -358,7 +359,7 @@ fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<
                     };
 
                     let attr = schema
-                        .attribute(&filter)
+                        .id(filter)
                         .expect("Could not find filtered attribute");
 
                     builder.with_filter(move |document_id| {
@@ -370,7 +371,7 @@ fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<
                     });
                 }
 
-                let documents = builder.query(ref_reader, &query, 0..command.number_results)?;
+                let (documents, _nb_hits) = builder.query(ref_reader, &query, 0..command.number_results)?;
 
                 let mut retrieve_duration = Duration::default();
 
@@ -389,11 +390,11 @@ fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<
                             for (name, text) in document.0 {
                                 print!("{}: ", name);
 
-                                let attr = schema.attribute(&name).unwrap();
+                                let attr = schema.id(&name).unwrap();
                                 let highlights = doc
                                     .highlights
                                     .iter()
-                                    .filter(|m| SchemaAttr::new(m.attribute) == attr)
+                                    .filter(|m| FieldId::new(m.attribute) == attr)
                                     .cloned();
                                 let (text, highlights) =
                                     crop_text(&text, highlights, command.char_context);
@@ -408,8 +409,8 @@ fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<
 
                     let mut matching_attributes = HashSet::new();
                     for highlight in doc.highlights {
-                        let attr = SchemaAttr::new(highlight.attribute);
-                        let name = schema.attribute_name(attr);
+                        let attr = FieldId::new(highlight.attribute);
+                        let name = schema.name(attr);
                         matching_attributes.insert(name);
                     }
 

meilisearch-core/src/automaton/dfa.rs

@@ -46,3 +46,8 @@ pub fn build_prefix_dfa(query: &str) -> DFA {
 pub fn build_dfa(query: &str) -> DFA {
     build_dfa_with_setting(query, PrefixSetting::NoPrefix)
 }
+
+pub fn build_exact_dfa(query: &str) -> DFA {
+    let builder = LEVDIST0.get_or_init(|| LevBuilder::new(0, true));
+    builder.build_dfa(query)
+}

meilisearch-core/src/automaton/mod.rs

@@ -1,125 +1,8 @@
 mod dfa;
-mod query_enhancer;
 
-use std::cmp::Reverse;
-use std::{cmp, vec};
+use meilisearch_tokenizer::is_cjk;
 
-use fst::{IntoStreamer, Streamer};
-use levenshtein_automata::DFA;
-use meilisearch_tokenizer::{is_cjk, split_query_string};
-
-use crate::database::MainT;
-use crate::error::MResult;
-use crate::store;
-
-use self::dfa::{build_dfa, build_prefix_dfa};
-pub use self::query_enhancer::QueryEnhancer;
-use self::query_enhancer::QueryEnhancerBuilder;
-
-const NGRAMS: usize = 3;
-
-pub struct AutomatonProducer {
-    automatons: Vec<AutomatonGroup>,
-}
-
-impl AutomatonProducer {
-    pub fn new(
-        reader: &heed::RoTxn<MainT>,
-        query: &str,
-        main_store: store::Main,
-        postings_list_store: store::PostingsLists,
-        synonyms_store: store::Synonyms,
-    ) -> MResult<(AutomatonProducer, QueryEnhancer)> {
-        let (automatons, query_enhancer) = generate_automatons(
-            reader,
-            query,
-            main_store,
-            postings_list_store,
-            synonyms_store,
-        )?;
-
-        Ok((AutomatonProducer { automatons }, query_enhancer))
-    }
-
-    pub fn into_iter(self) -> vec::IntoIter<AutomatonGroup> {
-        self.automatons.into_iter()
-    }
-}
-
-#[derive(Debug)]
-pub struct AutomatonGroup {
-    pub is_phrase_query: bool,
-    pub automatons: Vec<Automaton>,
-}
-
-impl AutomatonGroup {
-    fn normal(automatons: Vec<Automaton>) -> AutomatonGroup {
-        AutomatonGroup {
-            is_phrase_query: false,
-            automatons,
-        }
-    }
-
-    fn phrase_query(automatons: Vec<Automaton>) -> AutomatonGroup {
-        AutomatonGroup {
-            is_phrase_query: true,
-            automatons,
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct Automaton {
-    pub index: usize,
-    pub ngram: usize,
-    pub query_len: usize,
-    pub is_exact: bool,
-    pub is_prefix: bool,
-    pub query: String,
-}
-
-impl Automaton {
-    pub fn dfa(&self) -> DFA {
-        if self.is_prefix {
-            build_prefix_dfa(&self.query)
-        } else {
-            build_dfa(&self.query)
-        }
-    }
-
-    fn exact(index: usize, ngram: usize, query: &str) -> Automaton {
-        Automaton {
-            index,
-            ngram,
-            query_len: query.len(),
-            is_exact: true,
-            is_prefix: false,
-            query: query.to_string(),
-        }
-    }
-
-    fn prefix_exact(index: usize, ngram: usize, query: &str) -> Automaton {
-        Automaton {
-            index,
-            ngram,
-            query_len: query.len(),
-            is_exact: true,
-            is_prefix: true,
-            query: query.to_string(),
-        }
-    }
-
-    fn non_exact(index: usize, ngram: usize, query: &str) -> Automaton {
-        Automaton {
-            index,
-            ngram,
-            query_len: query.len(),
-            is_exact: false,
-            is_prefix: false,
-            query: query.to_string(),
-        }
-    }
-}
+pub use self::dfa::{build_dfa, build_prefix_dfa, build_exact_dfa};
 
 pub fn normalize_str(string: &str) -> String {
     let mut string = string.to_lowercase();
@@ -130,167 +13,3 @@ pub fn normalize_str(string: &str) -> String {
 
     string
 }
-
-fn split_best_frequency<'a>(
-    reader: &heed::RoTxn<MainT>,
-    word: &'a str,
-    postings_lists_store: store::PostingsLists,
-) -> MResult<Option<(&'a str, &'a str)>> {
-    let chars = word.char_indices().skip(1);
-    let mut best = None;
-
-    for (i, _) in chars {
-        let (left, right) = word.split_at(i);
-
-        let left_freq = postings_lists_store
-            .postings_list(reader, left.as_ref())?
-            .map_or(0, |i| i.len());
-
-        let right_freq = postings_lists_store
-            .postings_list(reader, right.as_ref())?
-            .map_or(0, |i| i.len());
-
-        let min_freq = cmp::min(left_freq, right_freq);
-        if min_freq != 0 && best.map_or(true, |(old, _, _)| min_freq > old) {
-            best = Some((min_freq, left, right));
-        }
-    }
-
-    Ok(best.map(|(_, l, r)| (l, r)))
-}
-
-fn generate_automatons(
-    reader: &heed::RoTxn<MainT>,
-    query: &str,
-    main_store: store::Main,
-    postings_lists_store: store::PostingsLists,
-    synonym_store: store::Synonyms,
-) -> MResult<(Vec<AutomatonGroup>, QueryEnhancer)> {
-    let has_end_whitespace = query.chars().last().map_or(false, char::is_whitespace);
-    let query_words: Vec<_> = split_query_string(query).map(str::to_lowercase).collect();
-    let synonyms = match main_store.synonyms_fst(reader)? {
-        Some(synonym) => synonym,
-        None => fst::Set::default(),
-    };
-
-    let mut automaton_index = 0;
-    let mut automatons = Vec::new();
-    let mut enhancer_builder = QueryEnhancerBuilder::new(&query_words);
-
-    // We must not declare the original words to the query enhancer
-    // *but* we need to push them in the automatons list first
-    let mut original_automatons = Vec::new();
-    let mut original_words = query_words.iter().peekable();
-    while let Some(word) = original_words.next() {
-        let has_following_word = original_words.peek().is_some();
-        let not_prefix_dfa = has_following_word || has_end_whitespace || word.chars().all(is_cjk);
-
-        let automaton = if not_prefix_dfa {
-            Automaton::exact(automaton_index, 1, word)
-        } else {
-            Automaton::prefix_exact(automaton_index, 1, word)
-        };
-        automaton_index += 1;
-        original_automatons.push(automaton);
-    }
-
-    automatons.push(AutomatonGroup::normal(original_automatons));
-
-    for n in 1..=NGRAMS {
-        let mut ngrams = query_words.windows(n).enumerate().peekable();
-        while let Some((query_index, ngram_slice)) = ngrams.next() {
-            let query_range = query_index..query_index + n;
-            let ngram_nb_words = ngram_slice.len();
-            let ngram = ngram_slice.join(" ");
-
-            let has_following_word = ngrams.peek().is_some();
-            let not_prefix_dfa =
-                has_following_word || has_end_whitespace || ngram.chars().all(is_cjk);
-
-            // automaton of synonyms of the ngrams
-            let normalized = normalize_str(&ngram);
-            let lev = if not_prefix_dfa {
-                build_dfa(&normalized)
-            } else {
-                build_prefix_dfa(&normalized)
-            };
-
-            let mut stream = synonyms.search(&lev).into_stream();
-            while let Some(base) = stream.next() {
-                // only trigger alternatives when the last word has been typed
-                // i.e. "new " do not but "new yo" triggers alternatives to "new york"
-                let base = std::str::from_utf8(base).unwrap();
-                let base_nb_words = split_query_string(base).count();
-                if ngram_nb_words != base_nb_words {
-                    continue;
-                }
-
-                if let Some(synonyms) = synonym_store.synonyms(reader, base.as_bytes())? {
-                    let mut stream = synonyms.into_stream();
-                    while let Some(synonyms) = stream.next() {
-                        let synonyms = std::str::from_utf8(synonyms).unwrap();
-                        let synonyms_words: Vec<_> = split_query_string(synonyms).collect();
-                        let nb_synonym_words = synonyms_words.len();
-
-                        let real_query_index = automaton_index;
-                        enhancer_builder.declare(
-                            query_range.clone(),
-                            real_query_index,
-                            &synonyms_words,
-                        );
-
-                        for synonym in synonyms_words {
-                            let automaton = if nb_synonym_words == 1 {
-                                Automaton::exact(automaton_index, n, synonym)
-                            } else {
-                                Automaton::non_exact(automaton_index, n, synonym)
-                            };
-                            automaton_index += 1;
-                            automatons.push(AutomatonGroup::normal(vec![automaton]));
-                        }
-                    }
-                }
-            }
-
-            if n == 1 {
-                if let Some((left, right)) =
-                    split_best_frequency(reader, &normalized, postings_lists_store)?
-                {
-                    let a = Automaton::exact(automaton_index, 1, left);
-                    enhancer_builder.declare(query_range.clone(), automaton_index, &[left]);
-                    automaton_index += 1;
-
-                    let b = Automaton::exact(automaton_index, 1, right);
-                    enhancer_builder.declare(query_range.clone(), automaton_index, &[left]);
-                    automaton_index += 1;
-
-                    automatons.push(AutomatonGroup::phrase_query(vec![a, b]));
-                }
-            } else {
-                // automaton of concatenation of query words
-                let concat = ngram_slice.concat();
-                let normalized = normalize_str(&concat);
-
-                let real_query_index = automaton_index;
-                enhancer_builder.declare(query_range.clone(), real_query_index, &[&normalized]);
-
-                let automaton = Automaton::exact(automaton_index, n, &normalized);
-                automaton_index += 1;
-                automatons.push(AutomatonGroup::normal(vec![automaton]));
-            }
-        }
-    }
-
-    // order automatons, the most important first,
-    // we keep the original automatons at the front.
-    automatons[1..].sort_by_key(|group| {
-        let a = group.automatons.first().unwrap();
-        (
-            Reverse(a.is_exact),
-            a.ngram,
-            Reverse(group.automatons.len()),
-        )
-    });
-
-    Ok((automatons, enhancer_builder.build()))
-}

meilisearch-core/src/automaton/query_enhancer.rs

@@ -1,423 +0,0 @@
-use std::cmp::Ordering::{Equal, Greater, Less};
-use std::ops::Range;
-
-/// Return `true` if the specified range can accept the given replacements words.
-/// Returns `false` if the replacements words are already present in the original query
-/// or if there is fewer replacement words than the range to replace.
-//
-//
-// ## Ignored because already present in original
-//
-//     new york city subway
-//     -------- ^^^^
-//   /          \
-//  [new york city]
-//
-//
-// ## Ignored because smaller than the original
-//
-//   new york city subway
-//   -------------
-//   \          /
-//    [new york]
-//
-//
-// ## Accepted because bigger than the original
-//
-//        NYC subway
-//        ---
-//       /   \
-//      /     \
-//     /       \
-//    /         \
-//   /           \
-//  [new york city]
-//
-fn rewrite_range_with<S, T>(query: &[S], range: Range<usize>, words: &[T]) -> bool
-where
-    S: AsRef<str>,
-    T: AsRef<str>,
-{
-    if words.len() <= range.len() {
-        // there is fewer or equal replacement words
-        // than there is already in the replaced range
-        return false;
-    }
-
-    // retrieve the part to rewrite but with the length
-    // of the replacement part
-    let original = query.iter().skip(range.start).take(words.len());
-
-    // check if the original query doesn't already contain
-    // the replacement words
-    !original
-        .map(AsRef::as_ref)
-        .eq(words.iter().map(AsRef::as_ref))
-}
-
-type Origin = usize;
-type RealLength = usize;
-
-struct FakeIntervalTree {
-    intervals: Vec<(Range<usize>, (Origin, RealLength))>,
-}
-
-impl FakeIntervalTree {
-    fn new(mut intervals: Vec<(Range<usize>, (Origin, RealLength))>) -> FakeIntervalTree {
-        intervals.sort_unstable_by_key(|(r, _)| (r.start, r.end));
-        FakeIntervalTree { intervals }
-    }
-
-    fn query(&self, point: usize) -> Option<(Range<usize>, (Origin, RealLength))> {
-        let element = self.intervals.binary_search_by(|(r, _)| {
-            if point >= r.start {
-                if point < r.end {
-                    Equal
-                } else {
-                    Less
-                }
-            } else {
-                Greater
-            }
-        });
-
-        let n = match element {
-            Ok(n) => n,
-            Err(n) => n,
-        };
-
-        match self.intervals.get(n) {
-            Some((range, value)) if range.contains(&point) => Some((range.clone(), *value)),
-            _otherwise => None,
-        }
-    }
-}
-
-pub struct QueryEnhancerBuilder<'a, S> {
-    query: &'a [S],
-    origins: Vec<usize>,
-    real_to_origin: Vec<(Range<usize>, (Origin, RealLength))>,
-}
-
-impl<S: AsRef<str>> QueryEnhancerBuilder<'_, S> {
-    pub fn new(query: &[S]) -> QueryEnhancerBuilder<S> {
-        // we initialize origins query indices based on their positions
-        let origins: Vec<_> = (0..=query.len()).collect();
-        let real_to_origin = origins.iter().map(|&o| (o..o + 1, (o, 1))).collect();
-
-        QueryEnhancerBuilder {
-            query,
-            origins,
-            real_to_origin,
-        }
-    }
-
-    /// Update the final real to origin query indices mapping.
-    ///
-    /// `range` is the original words range that this `replacement` words replace
-    /// and `real` is the first real query index of these replacement words.
-    pub fn declare<T>(&mut self, range: Range<usize>, real: usize, replacement: &[T])
-    where
-        T: AsRef<str>,
-    {
-        // check if the range of original words
-        // can be rewritten with the replacement words
-        if rewrite_range_with(self.query, range.clone(), replacement) {
-            // this range can be replaced so we need to
-            // modify the origins accordingly
-            let offset = replacement.len() - range.len();
-
-            let previous_padding = self.origins[range.end - 1];
-            let current_offset = (self.origins[range.end] - 1) - previous_padding;
-            let diff = offset.saturating_sub(current_offset);
-            self.origins[range.end] += diff;
-
-            for r in &mut self.origins[range.end + 1..] {
-                *r += diff;
-            }
-        }
-
-        // we need to store the real number and origins relations
-        // this way it will be possible to know by how many
-        // we need to pad real query indices
-        let real_range = real..real + replacement.len().max(range.len());
-        let real_length = replacement.len();
-        self.real_to_origin
-            .push((real_range, (range.start, real_length)));
-    }
-
-    pub fn build(self) -> QueryEnhancer {
-        QueryEnhancer {
-            origins: self.origins,
-            real_to_origin: FakeIntervalTree::new(self.real_to_origin),
-        }
-    }
-}
-
-pub struct QueryEnhancer {
-    origins: Vec<usize>,
-    real_to_origin: FakeIntervalTree,
-}
-
-impl QueryEnhancer {
-    /// Returns the query indices to use to replace this real query index.
-    pub fn replacement(&self, real: u32) -> Range<u32> {
-        let real = real as usize;
-
-        // query the fake interval tree with the real query index
-        let (range, (origin, real_length)) = self
-            .real_to_origin
-            .query(real)
-            .expect("real has never been declared");
-
-        // if `real` is the end bound of the range
-        if (range.start + real_length - 1) == real {
-            let mut count = range.len();
-            let mut new_origin = origin;
-            for (i, slice) in self.origins[new_origin..].windows(2).enumerate() {
-                let len = slice[1] - slice[0];
-                count = count.saturating_sub(len);
-                if count == 0 {
-                    new_origin = origin + i;
-                    break;
-                }
-            }
-
-            let n = real - range.start;
-            let start = self.origins[origin];
-            let end = self.origins[new_origin + 1];
-            let remaining = (end - start) - n;
-
-            Range {
-                start: (start + n) as u32,
-                end: (start + n + remaining) as u32,
-            }
-        } else {
-            // just return the origin along with
-            // the real position of the word
-            let n = real as usize - range.start;
-            let origin = self.origins[origin];
-
-            Range {
-                start: (origin + n) as u32,
-                end: (origin + n + 1) as u32,
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn original_unmodified() {
-        let query = ["new", "york", "city", "subway"];
-        //             0       1       2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // new york = new york city
-        builder.declare(0..2, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // new
-        assert_eq!(enhancer.replacement(1), 1..2); // york
-        assert_eq!(enhancer.replacement(2), 2..3); // city
-        assert_eq!(enhancer.replacement(3), 3..4); // subway
-        assert_eq!(enhancer.replacement(4), 0..1); // new
-        assert_eq!(enhancer.replacement(5), 1..2); // york
-        assert_eq!(enhancer.replacement(6), 2..3); // city
-    }
-
-    #[test]
-    fn simple_growing() {
-        let query = ["new", "york", "subway"];
-        //             0       1        2
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // new york = new york city
-        builder.declare(0..2, 3, &["new", "york", "city"]);
-        //                    ^      3       4       5
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // new
-        assert_eq!(enhancer.replacement(1), 1..3); // york
-        assert_eq!(enhancer.replacement(2), 3..4); // subway
-        assert_eq!(enhancer.replacement(3), 0..1); // new
-        assert_eq!(enhancer.replacement(4), 1..2); // york
-        assert_eq!(enhancer.replacement(5), 2..3); // city
-    }
-
-    #[test]
-    fn same_place_growings() {
-        let query = ["NY", "subway"];
-        //             0       1
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NY = new york
-        builder.declare(0..1, 2, &["new", "york"]);
-        //                    ^      2       3
-
-        // NY = new york city
-        builder.declare(0..1, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // NY = NYC
-        builder.declare(0..1, 7, &["NYC"]);
-        //                    ^      7
-
-        // NY = new york city
-        builder.declare(0..1, 8, &["new", "york", "city"]);
-        //                    ^      8       9      10
-
-        // subway = underground train
-        builder.declare(1..2, 11, &["underground", "train"]);
-        //                    ^          11          12
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..3); // NY
-        assert_eq!(enhancer.replacement(1), 3..5); // subway
-        assert_eq!(enhancer.replacement(2), 0..1); // new
-        assert_eq!(enhancer.replacement(3), 1..3); // york
-        assert_eq!(enhancer.replacement(4), 0..1); // new
-        assert_eq!(enhancer.replacement(5), 1..2); // york
-        assert_eq!(enhancer.replacement(6), 2..3); // city
-        assert_eq!(enhancer.replacement(7), 0..3); // NYC
-        assert_eq!(enhancer.replacement(8), 0..1); // new
-        assert_eq!(enhancer.replacement(9), 1..2); // york
-        assert_eq!(enhancer.replacement(10), 2..3); // city
-        assert_eq!(enhancer.replacement(11), 3..4); // underground
-        assert_eq!(enhancer.replacement(12), 4..5); // train
-    }
-
-    #[test]
-    fn bigger_growing() {
-        let query = ["NYC", "subway"];
-        //             0        1
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(0..1, 2, &["new", "york", "city"]);
-        //                    ^      2       3       4
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..3); // NYC
-        assert_eq!(enhancer.replacement(1), 3..4); // subway
-        assert_eq!(enhancer.replacement(2), 0..1); // new
-        assert_eq!(enhancer.replacement(3), 1..2); // york
-        assert_eq!(enhancer.replacement(4), 2..3); // city
-    }
-
-    #[test]
-    fn middle_query_growing() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // great
-        assert_eq!(enhancer.replacement(1), 1..2); // awesome
-        assert_eq!(enhancer.replacement(2), 2..5); // NYC
-        assert_eq!(enhancer.replacement(3), 5..6); // subway
-        assert_eq!(enhancer.replacement(4), 2..3); // new
-        assert_eq!(enhancer.replacement(5), 3..4); // york
-        assert_eq!(enhancer.replacement(6), 4..5); // city
-    }
-
-    #[test]
-    fn end_query_growing() {
-        let query = ["NYC", "subway"];
-        //             0        1
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(1..2, 2, &["underground", "train"]);
-        //                    ^         2            3
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // NYC
-        assert_eq!(enhancer.replacement(1), 1..3); // subway
-        assert_eq!(enhancer.replacement(2), 1..2); // underground
-        assert_eq!(enhancer.replacement(3), 2..3); // train
-    }
-
-    #[test]
-    fn multiple_growings() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // subway = underground train
-        builder.declare(3..4, 7, &["underground", "train"]);
-        //                    ^          7           8
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // great
-        assert_eq!(enhancer.replacement(1), 1..2); // awesome
-        assert_eq!(enhancer.replacement(2), 2..5); // NYC
-        assert_eq!(enhancer.replacement(3), 5..7); // subway
-        assert_eq!(enhancer.replacement(4), 2..3); // new
-        assert_eq!(enhancer.replacement(5), 3..4); // york
-        assert_eq!(enhancer.replacement(6), 4..5); // city
-        assert_eq!(enhancer.replacement(7), 5..6); // underground
-        assert_eq!(enhancer.replacement(8), 6..7); // train
-    }
-
-    #[test]
-    fn multiple_probable_growings() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // subway = underground train
-        builder.declare(3..4, 7, &["underground", "train"]);
-        //                    ^          7           8
-
-        // great awesome = good
-        builder.declare(0..2, 9, &["good"]);
-        //                    ^       9
-
-        // awesome NYC = NY
-        builder.declare(1..3, 10, &["NY"]);
-        //                    ^^     10
-
-        // NYC subway = metro
-        builder.declare(2..4, 11, &["metro"]);
-        //                    ^^      11
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // great
-        assert_eq!(enhancer.replacement(1), 1..2); // awesome
-        assert_eq!(enhancer.replacement(2), 2..5); // NYC
-        assert_eq!(enhancer.replacement(3), 5..7); // subway
-        assert_eq!(enhancer.replacement(4), 2..3); // new
-        assert_eq!(enhancer.replacement(5), 3..4); // york
-        assert_eq!(enhancer.replacement(6), 4..5); // city
-        assert_eq!(enhancer.replacement(7), 5..6); // underground
-        assert_eq!(enhancer.replacement(8), 6..7); // train
-        assert_eq!(enhancer.replacement(9), 0..2); // good
-        assert_eq!(enhancer.replacement(10), 1..5); // NY
-        assert_eq!(enhancer.replacement(11), 2..5); // metro
-    }
-}

meilisearch-core/src/bucket_sort.rs

@@ -0,0 +1,560 @@
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::mem;
+use std::ops::Deref;
+use std::ops::Range;
+use std::rc::Rc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::time::Instant;
+use std::fmt;
+
+use compact_arena::{SmallArena, Idx32, mk_arena};
+use log::debug;
+use meilisearch_types::DocIndex;
+use sdset::{Set, SetBuf, exponential_search};
+use slice_group_by::{GroupBy, GroupByMut};
+
+use crate::error::Error;
+use crate::criterion::{Criteria, Context, ContextMut};
+use crate::distinct_map::{BufferedDistinctMap, DistinctMap};
+use crate::raw_document::RawDocument;
+use crate::{database::MainT, reordered_attrs::ReorderedAttrs};
+use crate::{store, Document, DocumentId, MResult};
+use crate::query_tree::{create_query_tree, traverse_query_tree};
+use crate::query_tree::{Operation, QueryResult, QueryKind, QueryId, PostingsKey};
+use crate::query_tree::Context as QTContext;
+
+pub fn bucket_sort<'c, FI>(
+    reader: &heed::RoTxn<MainT>,
+    query: &str,
+    range: Range<usize>,
+    filter: Option<FI>,
+    criteria: Criteria<'c>,
+    searchable_attrs: Option<ReorderedAttrs>,
+    main_store: store::Main,
+    postings_lists_store: store::PostingsLists,
+    documents_fields_counts_store: store::DocumentsFieldsCounts,
+    synonyms_store: store::Synonyms,
+    prefix_documents_cache_store: store::PrefixDocumentsCache,
+    prefix_postings_lists_cache_store: store::PrefixPostingsListsCache,
+) -> MResult<(Vec<Document>, usize)>
+where
+    FI: Fn(DocumentId) -> bool,
+{
+    // We delegate the filter work to the distinct query builder,
+    // specifying a distinct rule that has no effect.
+    if filter.is_some() {
+        let distinct = |_| None;
+        let distinct_size = 1;
+        return bucket_sort_with_distinct(
+            reader,
+            query,
+            range,
+            filter,
+            distinct,
+            distinct_size,
+            criteria,
+            searchable_attrs,
+            main_store,
+            postings_lists_store,
+            documents_fields_counts_store,
+            synonyms_store,
+            prefix_documents_cache_store,
+            prefix_postings_lists_cache_store,
+        );
+    }
+
+    let words_set = match unsafe { main_store.static_words_fst(reader)? } {
+        Some(words) => words,
+        None => return Ok((Vec::new(), 0)),
+    };
+
+    let stop_words = main_store.stop_words_fst(reader)?.unwrap_or_default();
+
+    let context = QTContext {
+        words_set,
+        stop_words,
+        synonyms: synonyms_store,
+        postings_lists: postings_lists_store,
+        prefix_postings_lists: prefix_postings_lists_cache_store,
+    };
+
+    let (operation, mapping) = create_query_tree(reader, &context, query)?;
+    debug!("operation:\n{:?}", operation);
+    debug!("mapping:\n{:?}", mapping);
+
+    fn recurs_operation<'o>(map: &mut HashMap<QueryId, &'o QueryKind>, operation: &'o Operation) {
+        match operation {
+            Operation::And(ops) => ops.iter().for_each(|op| recurs_operation(map, op)),
+            Operation::Or(ops) => ops.iter().for_each(|op| recurs_operation(map, op)),
+            Operation::Query(query) => { map.insert(query.id, &query.kind); },
+        }
+    }
+
+    let mut queries_kinds = HashMap::new();
+    recurs_operation(&mut queries_kinds, &operation);
+
+    let QueryResult { docids, queries } = traverse_query_tree(reader, &context, &operation)?;
+    debug!("found {} documents", docids.len());
+    debug!("number of postings {:?}", queries.len());
+
+    let before = Instant::now();
+    mk_arena!(arena);
+    let mut bare_matches = cleanup_bare_matches(&mut arena, &docids, queries);
+    debug!("matches cleaned in {:.02?}", before.elapsed());
+
+    let before_bucket_sort = Instant::now();
+
+    let before_raw_documents_building = Instant::now();
+    let mut raw_documents = Vec::new();
+    for bare_matches in bare_matches.linear_group_by_key_mut(|sm| sm.document_id) {
+        let raw_document = RawDocument::new(bare_matches, &mut arena, searchable_attrs.as_ref());
+        raw_documents.push(raw_document);
+    }
+    debug!("creating {} candidates documents took {:.02?}",
+        raw_documents.len(),
+        before_raw_documents_building.elapsed(),
+    );
+
+    let before_criterion_loop = Instant::now();
+    let proximity_count = AtomicUsize::new(0);
+
+    let mut groups = vec![raw_documents.as_mut_slice()];
+
+    'criteria: for criterion in criteria.as_ref() {
+        let tmp_groups = mem::replace(&mut groups, Vec::new());
+        let mut documents_seen = 0;
+
+        for mut group in tmp_groups {
+            let before_criterion_preparation = Instant::now();
+
+            let ctx = ContextMut {
+                reader,
+                postings_lists: &mut arena,
+                query_mapping: &mapping,
+                documents_fields_counts_store,
+            };
+
+            criterion.prepare(ctx, &mut group)?;
+            debug!("{:?} preparation took {:.02?}", criterion.name(), before_criterion_preparation.elapsed());
+
+            let ctx = Context {
+                postings_lists: &arena,
+                query_mapping: &mapping,
+            };
+
+            let before_criterion_sort = Instant::now();
+            group.sort_unstable_by(|a, b| criterion.evaluate(&ctx, a, b));
+            debug!("{:?} evaluation took {:.02?}", criterion.name(), before_criterion_sort.elapsed());
+
+            for group in group.binary_group_by_mut(|a, b| criterion.eq(&ctx, a, b)) {
+                debug!("{:?} produced a group of size {}", criterion.name(), group.len());
+
+                documents_seen += group.len();
+                groups.push(group);
+
+                // we have sort enough documents if the last document sorted is after
+                // the end of the requested range, we can continue to the next criterion
+                if documents_seen >= range.end {
+                    continue 'criteria;
+                }
+            }
+        }
+    }
+
+    debug!("criterion loop took {:.02?}", before_criterion_loop.elapsed());
+    debug!("proximity evaluation called {} times", proximity_count.load(Ordering::Relaxed));
+
+    let schema = main_store.schema(reader)?.ok_or(Error::SchemaMissing)?;
+    let iter = raw_documents.into_iter().skip(range.start).take(range.len());
+    let iter = iter.map(|rd| Document::from_raw(rd, &queries_kinds, &arena, searchable_attrs.as_ref(), &schema));
+    let documents = iter.collect();
+
+    debug!("bucket sort took {:.02?}", before_bucket_sort.elapsed());
+
+    Ok((documents, docids.len()))
+}
+
+pub fn bucket_sort_with_distinct<'c, FI, FD>(
+    reader: &heed::RoTxn<MainT>,
+    query: &str,
+    range: Range<usize>,
+    filter: Option<FI>,
+    distinct: FD,
+    distinct_size: usize,
+    criteria: Criteria<'c>,
+    searchable_attrs: Option<ReorderedAttrs>,
+    main_store: store::Main,
+    postings_lists_store: store::PostingsLists,
+    documents_fields_counts_store: store::DocumentsFieldsCounts,
+    synonyms_store: store::Synonyms,
+    _prefix_documents_cache_store: store::PrefixDocumentsCache,
+    prefix_postings_lists_cache_store: store::PrefixPostingsListsCache,
+) -> MResult<(Vec<Document>, usize)>
+where
+    FI: Fn(DocumentId) -> bool,
+    FD: Fn(DocumentId) -> Option<u64>,
+{
+    let words_set = match unsafe { main_store.static_words_fst(reader)? } {
+        Some(words) => words,
+        None => return Ok((Vec::new(), 0)),
+    };
+
+    let stop_words = main_store.stop_words_fst(reader)?.unwrap_or_default();
+
+    let context = QTContext {
+        words_set,
+        stop_words,
+        synonyms: synonyms_store,
+        postings_lists: postings_lists_store,
+        prefix_postings_lists: prefix_postings_lists_cache_store,
+    };
+
+    let (operation, mapping) = create_query_tree(reader, &context, query)?;
+    debug!("operation:\n{:?}", operation);
+    debug!("mapping:\n{:?}", mapping);
+
+    fn recurs_operation<'o>(map: &mut HashMap<QueryId, &'o QueryKind>, operation: &'o Operation) {
+        match operation {
+            Operation::And(ops) => ops.iter().for_each(|op| recurs_operation(map, op)),
+            Operation::Or(ops) => ops.iter().for_each(|op| recurs_operation(map, op)),
+            Operation::Query(query) => { map.insert(query.id, &query.kind); },
+        }
+    }
+
+    let mut queries_kinds = HashMap::new();
+    recurs_operation(&mut queries_kinds, &operation);
+
+    let QueryResult { docids, queries } = traverse_query_tree(reader, &context, &operation)?;
+    debug!("found {} documents", docids.len());
+    debug!("number of postings {:?}", queries.len());
+
+    let before = Instant::now();
+    mk_arena!(arena);
+    let mut bare_matches = cleanup_bare_matches(&mut arena, &docids, queries);
+    debug!("matches cleaned in {:.02?}", before.elapsed());
+
+    let before_raw_documents_building = Instant::now();
+    let mut raw_documents = Vec::new();
+    for bare_matches in bare_matches.linear_group_by_key_mut(|sm| sm.document_id) {
+        let raw_document = RawDocument::new(bare_matches, &mut arena, searchable_attrs.as_ref());
+        raw_documents.push(raw_document);
+    }
+    debug!("creating {} candidates documents took {:.02?}",
+        raw_documents.len(),
+        before_raw_documents_building.elapsed(),
+    );
+
+    let mut groups = vec![raw_documents.as_mut_slice()];
+    let mut key_cache = HashMap::new();
+
+    let mut filter_map = HashMap::new();
+    // these two variables informs on the current distinct map and
+    // on the raw offset of the start of the group where the
+    // range.start bound is located according to the distinct function
+    let mut distinct_map = DistinctMap::new(distinct_size);
+    let mut distinct_raw_offset = 0;
+
+    'criteria: for criterion in criteria.as_ref() {
+        let tmp_groups = mem::replace(&mut groups, Vec::new());
+        let mut buf_distinct = BufferedDistinctMap::new(&mut distinct_map);
+        let mut documents_seen = 0;
+
+        for mut group in tmp_groups {
+            // if this group does not overlap with the requested range,
+            // push it without sorting and splitting it
+            if documents_seen + group.len() < distinct_raw_offset {
+                documents_seen += group.len();
+                groups.push(group);
+                continue;
+            }
+
+            let ctx = ContextMut {
+                reader,
+                postings_lists: &mut arena,
+                query_mapping: &mapping,
+                documents_fields_counts_store,
+            };
+
+            let before_criterion_preparation = Instant::now();
+            criterion.prepare(ctx, &mut group)?;
+            debug!("{:?} preparation took {:.02?}", criterion.name(), before_criterion_preparation.elapsed());
+
+            let ctx = Context {
+                postings_lists: &arena,
+                query_mapping: &mapping,
+            };
+
+            let before_criterion_sort = Instant::now();
+            group.sort_unstable_by(|a, b| criterion.evaluate(&ctx, a, b));
+            debug!("{:?} evaluation took {:.02?}", criterion.name(), before_criterion_sort.elapsed());
+
+            for group in group.binary_group_by_mut(|a, b| criterion.eq(&ctx, a, b)) {
+                // we must compute the real distinguished len of this sub-group
+                for document in group.iter() {
+                    let filter_accepted = match &filter {
+                        Some(filter) => {
+                            let entry = filter_map.entry(document.id);
+                            *entry.or_insert_with(|| (filter)(document.id))
+                        }
+                        None => true,
+                    };
+
+                    if filter_accepted {
+                        let entry = key_cache.entry(document.id);
+                        let key = entry.or_insert_with(|| (distinct)(document.id).map(Rc::new));
+
+                        match key.clone() {
+                            Some(key) => buf_distinct.register(key),
+                            None => buf_distinct.register_without_key(),
+                        };
+                    }
+
+                    // the requested range end is reached: stop computing distinct
+                    if buf_distinct.len() >= range.end {
+                        break;
+                    }
+                }
+
+                documents_seen += group.len();
+                groups.push(group);
+
+                // if this sub-group does not overlap with the requested range
+                // we must update the distinct map and its start index
+                if buf_distinct.len() < range.start {
+                    buf_distinct.transfert_to_internal();
+                    distinct_raw_offset = documents_seen;
+                }
+
+                // we have sort enough documents if the last document sorted is after
+                // the end of the requested range, we can continue to the next criterion
+                if buf_distinct.len() >= range.end {
+                    continue 'criteria;
+                }
+            }
+        }
+    }
+
+    // once we classified the documents related to the current
+    // automatons we save that as the next valid result
+    let mut seen = BufferedDistinctMap::new(&mut distinct_map);
+    let schema = main_store.schema(reader)?.ok_or(Error::SchemaMissing)?;
+
+    let mut documents = Vec::with_capacity(range.len());
+    for raw_document in raw_documents.into_iter().skip(distinct_raw_offset) {
+        let filter_accepted = match &filter {
+            Some(_) => filter_map.remove(&raw_document.id).unwrap(),
+            None => true,
+        };
+
+        if filter_accepted {
+            let key = key_cache.remove(&raw_document.id).unwrap();
+            let distinct_accepted = match key {
+                Some(key) => seen.register(key),
+                None => seen.register_without_key(),
+            };
+
+            if distinct_accepted && seen.len() > range.start {
+                documents.push(Document::from_raw(raw_document, &queries_kinds, &arena, searchable_attrs.as_ref(), &schema));
+                if documents.len() == range.len() {
+                    break;
+                }
+            }
+        }
+    }
+
+    Ok((documents, docids.len()))
+}
+
+fn cleanup_bare_matches<'tag, 'txn>(
+    arena: &mut SmallArena<'tag, PostingsListView<'txn>>,
+    docids: &Set<DocumentId>,
+    queries: HashMap<PostingsKey, Cow<'txn, Set<DocIndex>>>,
+) -> Vec<BareMatch<'tag>>
+{
+    let docidslen = docids.len() as f32;
+    let mut bare_matches = Vec::new();
+
+    for (PostingsKey { query, input, distance, is_exact }, matches) in queries {
+        let postings_list_view = PostingsListView::original(Rc::from(input), Rc::new(matches));
+        let pllen = postings_list_view.len() as f32;
+
+        if docidslen / pllen >= 0.8 {
+            let mut offset = 0;
+            for matches in postings_list_view.linear_group_by_key(|m| m.document_id) {
+                let document_id = matches[0].document_id;
+                if docids.contains(&document_id) {
+                    let range = postings_list_view.range(offset, matches.len());
+                    let posting_list_index = arena.add(range);
+
+                    let bare_match = BareMatch {
+                        document_id,
+                        query_index: query.id,
+                        distance,
+                        is_exact,
+                        postings_list: posting_list_index,
+                    };
+
+                    bare_matches.push(bare_match);
+                }
+
+                offset += matches.len();
+            }
+
+        } else {
+            let mut offset = 0;
+            for id in docids.as_slice() {
+                let di = DocIndex { document_id: *id, ..DocIndex::default() };
+                let pos = exponential_search(&postings_list_view[offset..], &di).unwrap_or_else(|x| x);
+
+                offset += pos;
+
+                let group = postings_list_view[offset..]
+                    .linear_group_by_key(|m| m.document_id)
+                    .next()
+                    .filter(|matches| matches[0].document_id == *id);
+
+                if let Some(matches) = group {
+                    let range = postings_list_view.range(offset, matches.len());
+                    let posting_list_index = arena.add(range);
+
+                    let bare_match = BareMatch {
+                        document_id: *id,
+                        query_index: query.id,
+                        distance,
+                        is_exact,
+                        postings_list: posting_list_index,
+                    };
+
+                    bare_matches.push(bare_match);
+                }
+            }
+        }
+    }
+
+    let before_raw_documents_presort = Instant::now();
+    bare_matches.sort_unstable_by_key(|sm| sm.document_id);
+    debug!("sort by documents ids took {:.02?}", before_raw_documents_presort.elapsed());
+
+    bare_matches
+}
+
+pub struct BareMatch<'tag> {
+    pub document_id: DocumentId,
+    pub query_index: usize,
+    pub distance: u8,
+    pub is_exact: bool,
+    pub postings_list: Idx32<'tag>,
+}
+
+impl fmt::Debug for BareMatch<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("BareMatch")
+            .field("document_id", &self.document_id)
+            .field("query_index", &self.query_index)
+            .field("distance", &self.distance)
+            .field("is_exact", &self.is_exact)
+            .finish()
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct SimpleMatch {
+    pub query_index: usize,
+    pub distance: u8,
+    pub attribute: u16,
+    pub word_index: u16,
+    pub is_exact: bool,
+}
+
+#[derive(Clone)]
+pub enum PostingsListView<'txn> {
+    Original {
+        input: Rc<[u8]>,
+        postings_list: Rc<Cow<'txn, Set<DocIndex>>>,
+        offset: usize,
+        len: usize,
+    },
+    Rewritten {
+        input: Rc<[u8]>,
+        postings_list: SetBuf<DocIndex>,
+    },
+}
+
+impl fmt::Debug for PostingsListView<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("PostingsListView")
+            .field("input", &std::str::from_utf8(&self.input()).unwrap())
+            .field("postings_list", &self.as_ref())
+            .finish()
+    }
+}
+
+impl<'txn> PostingsListView<'txn> {
+    pub fn original(input: Rc<[u8]>, postings_list: Rc<Cow<'txn, Set<DocIndex>>>) -> PostingsListView<'txn> {
+        let len = postings_list.len();
+        PostingsListView::Original { input, postings_list, offset: 0, len }
+    }
+
+    pub fn rewritten(input: Rc<[u8]>, postings_list: SetBuf<DocIndex>) -> PostingsListView<'txn> {
+        PostingsListView::Rewritten { input, postings_list }
+    }
+
+    pub fn rewrite_with(&mut self, postings_list: SetBuf<DocIndex>) {
+        let input = match self {
+            PostingsListView::Original { input, .. } => input.clone(),
+            PostingsListView::Rewritten { input, .. } => input.clone(),
+        };
+        *self = PostingsListView::rewritten(input, postings_list);
+    }
+
+    pub fn len(&self) -> usize {
+        match self {
+            PostingsListView::Original { len, .. } => *len,
+            PostingsListView::Rewritten { postings_list, .. } => postings_list.len(),
+        }
+    }
+
+    pub fn input(&self) -> &[u8] {
+        match self {
+            PostingsListView::Original { ref input, .. } => input,
+            PostingsListView::Rewritten { ref input, .. } => input,
+        }
+    }
+
+    pub fn range(&self, range_offset: usize, range_len: usize) -> PostingsListView<'txn> {
+        match self {
+            PostingsListView::Original { input, postings_list, offset, len } => {
+                assert!(range_offset + range_len <= *len);
+                PostingsListView::Original {
+                    input: input.clone(),
+                    postings_list: postings_list.clone(),
+                    offset: offset + range_offset,
+                    len: range_len,
+                }
+            },
+            PostingsListView::Rewritten { .. } => {
+                panic!("Cannot create a range on a rewritten postings list view");
+            }
+        }
+    }
+}
+
+impl AsRef<Set<DocIndex>> for PostingsListView<'_> {
+    fn as_ref(&self) -> &Set<DocIndex> {
+        self
+    }
+}
+
+impl Deref for PostingsListView<'_> {
+    type Target = Set<DocIndex>;
+
+    fn deref(&self) -> &Set<DocIndex> {
+        match *self {
+            PostingsListView::Original { ref postings_list, offset, len, .. } => {
+                Set::new_unchecked(&postings_list[offset..offset + len])
+            },
+            PostingsListView::Rewritten { ref postings_list, .. } => postings_list,
+        }
+    }
+}

meilisearch-core/src/criterion/attribute.rs

@@ -0,0 +1,37 @@
+use std::cmp::Ordering;
+use slice_group_by::GroupBy;
+use crate::{RawDocument, MResult};
+use crate::bucket_sort::SimpleMatch;
+use super::{Criterion, Context, ContextMut, prepare_bare_matches};
+
+pub struct Attribute;
+
+impl Criterion for Attribute {
+    fn name(&self) -> &str { "attribute" }
+
+    fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>(
+        &self,
+        ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>,
+        documents: &mut [RawDocument<'r, 'tag>],
+    ) -> MResult<()>
+    {
+        prepare_bare_matches(documents, ctx.postings_lists, ctx.query_mapping);
+        Ok(())
+    }
+
+    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
+        #[inline]
+        fn sum_of_attribute(matches: &[SimpleMatch]) -> usize {
+            let mut sum_of_attribute = 0;
+            for group in matches.linear_group_by_key(|bm| bm.query_index) {
+                sum_of_attribute += group[0].attribute as usize;
+            }
+            sum_of_attribute
+        }
+
+        let lhs = sum_of_attribute(&lhs.processed_matches);
+        let rhs = sum_of_attribute(&rhs.processed_matches);
+
+        lhs.cmp(&rhs)
+    }
+}

meilisearch-core/src/criterion/document_id.rs

@@ -1,16 +1,16 @@
-use crate::criterion::Criterion;
-use crate::RawDocument;
 use std::cmp::Ordering;
+use crate::RawDocument;
+use super::{Criterion, Context};
 
-#[derive(Debug, Clone, Copy)]
 pub struct DocumentId;
 
 impl Criterion for DocumentId {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        lhs.id.cmp(&rhs.id)
-    }
+    fn name(&self) -> &str { "stable document id" }
 
-    fn name(&self) -> &str {
-        "DocumentId"
+    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
+        let lhs = &lhs.id;
+        let rhs = &rhs.id;
+
+        lhs.cmp(rhs)
     }
 }

meilisearch-core/src/criterion/exact.rs

@@ -1,132 +0,0 @@
-use std::cmp::Ordering;
-
-use meilisearch_schema::SchemaAttr;
-use sdset::Set;
-use slice_group_by::GroupBy;
-
-use crate::criterion::Criterion;
-use crate::RawDocument;
-
-#[inline]
-fn number_exact_matches(
-    query_index: &[u32],
-    attribute: &[u16],
-    is_exact: &[bool],
-    fields_counts: &Set<(SchemaAttr, u64)>,
-) -> usize {
-    let mut count = 0;
-    let mut index = 0;
-
-    for group in query_index.linear_group() {
-        let len = group.len();
-
-        let mut found_exact = false;
-        for (pos, is_exact) in is_exact[index..index + len].iter().enumerate() {
-            if *is_exact {
-                found_exact = true;
-                let attr = &attribute[index + pos];
-                if let Ok(pos) = fields_counts.binary_search_by_key(attr, |(a, _)| a.0) {
-                    let (_, count) = fields_counts[pos];
-                    if count == 1 {
-                        return usize::max_value();
-                    }
-                }
-            }
-        }
-
-        count += found_exact as usize;
-        index += len;
-    }
-
-    count
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct Exact;
-
-impl Criterion for Exact {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = {
-            let query_index = lhs.query_index();
-            let is_exact = lhs.is_exact();
-            let attribute = lhs.attribute();
-            let fields_counts = &lhs.fields_counts;
-
-            number_exact_matches(query_index, attribute, is_exact, fields_counts)
-        };
-
-        let rhs = {
-            let query_index = rhs.query_index();
-            let is_exact = rhs.is_exact();
-            let attribute = rhs.attribute();
-            let fields_counts = &rhs.fields_counts;
-
-            number_exact_matches(query_index, attribute, is_exact, fields_counts)
-        };
-
-        lhs.cmp(&rhs).reverse()
-    }
-
-    fn name(&self) -> &str {
-        "Exact"
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    // typing: "soulier"
-    //
-    // doc0: "Soulier bleu"
-    // doc1: "souliereres rouge"
-    #[test]
-    fn easy_case() {
-        let doc0 = {
-            let query_index = &[0];
-            let attribute = &[0];
-            let is_exact = &[true];
-            let fields_counts = Set::new(&[(SchemaAttr(0), 2)]).unwrap();
-
-            number_exact_matches(query_index, attribute, is_exact, fields_counts)
-        };
-
-        let doc1 = {
-            let query_index = &[0];
-            let attribute = &[0];
-            let is_exact = &[false];
-            let fields_counts = Set::new(&[(SchemaAttr(0), 2)]).unwrap();
-
-            number_exact_matches(query_index, attribute, is_exact, fields_counts)
-        };
-
-        assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
-    }
-
-    // typing: "soulier"
-    //
-    // doc0: { 0. "soulier" }
-    // doc1: { 0. "soulier bleu et blanc" }
-    #[test]
-    fn basic() {
-        let doc0 = {
-            let query_index = &[0];
-            let attribute = &[0];
-            let is_exact = &[true];
-            let fields_counts = Set::new(&[(SchemaAttr(0), 1)]).unwrap();
-
-            number_exact_matches(query_index, attribute, is_exact, fields_counts)
-        };
-
-        let doc1 = {
-            let query_index = &[0];
-            let attribute = &[0];
-            let is_exact = &[true];
-            let fields_counts = Set::new(&[(SchemaAttr(0), 4)]).unwrap();
-
-            number_exact_matches(query_index, attribute, is_exact, fields_counts)
-        };
-
-        assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
-    }
-}

meilisearch-core/src/criterion/exactness.rs

@@ -0,0 +1,78 @@
+use std::cmp::{Ordering, Reverse};
+use std::collections::hash_map::{HashMap, Entry};
+use meilisearch_schema::IndexedPos;
+use slice_group_by::GroupBy;
+use crate::{RawDocument, MResult};
+use crate::bucket_sort::BareMatch;
+use super::{Criterion, Context, ContextMut};
+
+pub struct Exactness;
+
+impl Criterion for Exactness {
+    fn name(&self) -> &str { "exactness" }
+
+    fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>(
+        &self,
+        ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>,
+        documents: &mut [RawDocument<'r, 'tag>],
+    ) -> MResult<()>
+    {
+        let store = ctx.documents_fields_counts_store;
+        let reader = ctx.reader;
+
+        'documents: for doc in documents {
+            doc.bare_matches.sort_unstable_by_key(|bm| (bm.query_index, Reverse(bm.is_exact)));
+
+            // mark the document if we find a "one word field" that matches
+            let mut fields_counts = HashMap::new();
+            for group in doc.bare_matches.linear_group_by_key(|bm| bm.query_index) {
+                for group in group.linear_group_by_key(|bm| bm.is_exact) {
+                    if !group[0].is_exact { break }
+
+                    for bm in group {
+                        for di in ctx.postings_lists[bm.postings_list].as_ref() {
+
+                            let attr = IndexedPos(di.attribute);
+                            let count = match fields_counts.entry(attr) {
+                                Entry::Occupied(entry) => *entry.get(),
+                                Entry::Vacant(entry) => {
+                                    let count = store.document_field_count(reader, doc.id, attr)?;
+                                    *entry.insert(count)
+                                },
+                            };
+
+                            if count == Some(1) {
+                                doc.contains_one_word_field = true;
+                                continue 'documents
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
+        #[inline]
+        fn sum_exact_query_words(matches: &[BareMatch]) -> usize {
+            let mut sum_exact_query_words = 0;
+
+            for group in matches.linear_group_by_key(|bm| bm.query_index) {
+                sum_exact_query_words += group[0].is_exact as usize;
+            }
+
+            sum_exact_query_words
+        }
+
+        // does it contains a "one word field"
+        lhs.contains_one_word_field.cmp(&rhs.contains_one_word_field).reverse()
+        // if not, with document contains the more exact words
+        .then_with(|| {
+            let lhs = sum_exact_query_words(&lhs.bare_matches);
+            let rhs = sum_exact_query_words(&rhs.bare_matches);
+            lhs.cmp(&rhs).reverse()
+        })
+    }
+}

meilisearch-core/src/criterion/mod.rs

@@ -1,59 +1,75 @@
+use std::cmp::{self, Ordering};
+use std::collections::HashMap;
+use std::ops::Range;
+
+use compact_arena::SmallArena;
+use sdset::SetBuf;
+use slice_group_by::GroupBy;
+
+use crate::bucket_sort::{SimpleMatch, PostingsListView};
+use crate::database::MainT;
+use crate::query_tree::QueryId;
+use crate::{store, RawDocument, MResult};
+
+mod typo;
+mod words;
+mod proximity;
+mod attribute;
+mod words_position;
+mod exactness;
 mod document_id;
-mod exact;
-mod number_of_words;
 mod sort_by_attr;
-mod sum_of_typos;
-mod sum_of_words_attribute;
-mod sum_of_words_position;
-mod words_proximity;
 
-use crate::RawDocument;
-use std::cmp::Ordering;
-
-pub use self::{
-    document_id::DocumentId, exact::Exact, number_of_words::NumberOfWords,
-    sort_by_attr::SortByAttr, sum_of_typos::SumOfTypos,
-    sum_of_words_attribute::SumOfWordsAttribute, sum_of_words_position::SumOfWordsPosition,
-    words_proximity::WordsProximity,
-};
-
-pub trait Criterion: Send + Sync {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering;
+pub use self::typo::Typo;
+pub use self::words::Words;
+pub use self::proximity::Proximity;
+pub use self::attribute::Attribute;
+pub use self::words_position::WordsPosition;
+pub use self::exactness::Exactness;
+pub use self::document_id::DocumentId;
+pub use self::sort_by_attr::SortByAttr;
 
+pub trait Criterion {
     fn name(&self) -> &str;
 
+    fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>(
+        &self,
+        _ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>,
+        _documents: &mut [RawDocument<'r, 'tag>],
+    ) -> MResult<()>
+    {
+        Ok(())
+    }
+
+    fn evaluate<'p, 'tag, 'txn, 'q, 'r>(
+        &self,
+        ctx: &Context<'p, 'tag, 'txn, 'q>,
+        lhs: &RawDocument<'r, 'tag>,
+        rhs: &RawDocument<'r, 'tag>,
+    ) -> Ordering;
+
     #[inline]
-    fn eq(&self, lhs: &RawDocument, rhs: &RawDocument) -> bool {
-        self.evaluate(lhs, rhs) == Ordering::Equal
+    fn eq<'p, 'tag, 'txn, 'q, 'r>(
+        &self,
+        ctx: &Context<'p, 'tag, 'txn, 'q>,
+        lhs: &RawDocument<'r, 'tag>,
+        rhs: &RawDocument<'r, 'tag>,
+    ) -> bool
+    {
+        self.evaluate(ctx, lhs, rhs) == Ordering::Equal
     }
 }
 
-impl<'a, T: Criterion + ?Sized + Send + Sync> Criterion for &'a T {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        (**self).evaluate(lhs, rhs)
-    }
-
-    fn name(&self) -> &str {
-        (**self).name()
-    }
-
-    fn eq(&self, lhs: &RawDocument, rhs: &RawDocument) -> bool {
-        (**self).eq(lhs, rhs)
-    }
+pub struct ContextMut<'h, 'p, 'tag, 'txn, 'q> {
+    pub reader: &'h heed::RoTxn<MainT>,
+    pub postings_lists: &'p mut SmallArena<'tag, PostingsListView<'txn>>,
+    pub query_mapping: &'q HashMap<QueryId, Range<usize>>,
+    pub documents_fields_counts_store: store::DocumentsFieldsCounts,
 }
 
-impl<T: Criterion + ?Sized> Criterion for Box<T> {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        (**self).evaluate(lhs, rhs)
-    }
-
-    fn name(&self) -> &str {
-        (**self).name()
-    }
-
-    fn eq(&self, lhs: &RawDocument, rhs: &RawDocument) -> bool {
-        (**self).eq(lhs, rhs)
-    }
+pub struct Context<'p, 'tag, 'txn, 'q> {
+    pub postings_lists: &'p SmallArena<'tag, PostingsListView<'txn>>,
+    pub query_mapping: &'q HashMap<QueryId, Range<usize>>,
 }
 
 #[derive(Default)]
@@ -103,12 +119,12 @@ pub struct Criteria<'a> {
 impl<'a> Default for Criteria<'a> {
     fn default() -> Self {
         CriteriaBuilder::with_capacity(7)
-            .add(SumOfTypos)
-            .add(NumberOfWords)
-            .add(WordsProximity)
-            .add(SumOfWordsAttribute)
-            .add(SumOfWordsPosition)
-            .add(Exact)
+            .add(Typo)
+            .add(Words)
+            .add(Proximity)
+            .add(Attribute)
+            .add(WordsPosition)
+            .add(Exactness)
             .add(DocumentId)
             .build()
     }
@@ -119,3 +135,157 @@ impl<'a> AsRef<[Box<dyn Criterion + 'a>]> for Criteria<'a> {
         &self.inner
     }
 }
+
+fn prepare_query_distances<'a, 'tag, 'txn>(
+    documents: &mut [RawDocument<'a, 'tag>],
+    query_mapping: &HashMap<QueryId, Range<usize>>,
+    postings_lists: &SmallArena<'tag, PostingsListView<'txn>>,
+) {
+    for document in documents {
+        if !document.processed_distances.is_empty() { continue }
+
+        let mut processed = Vec::new();
+        for m in document.bare_matches.iter() {
+            if postings_lists[m.postings_list].is_empty() { continue }
+
+            let range = query_mapping[&(m.query_index as usize)].clone();
+            let new_len = cmp::max(range.end as usize, processed.len());
+            processed.resize(new_len, None);
+
+            for index in range {
+                let index = index as usize;
+                processed[index] = match processed[index] {
+                    Some(distance) if distance > m.distance => Some(m.distance),
+                    Some(distance) => Some(distance),
+                    None => Some(m.distance),
+                };
+            }
+        }
+
+        document.processed_distances = processed;
+    }
+}
+
+fn prepare_bare_matches<'a, 'tag, 'txn>(
+    documents: &mut [RawDocument<'a, 'tag>],
+    postings_lists: &mut SmallArena<'tag, PostingsListView<'txn>>,
+    query_mapping: &HashMap<QueryId, Range<usize>>,
+) {
+    for document in documents {
+        if !document.processed_matches.is_empty() { continue }
+
+        let mut processed = Vec::new();
+        for m in document.bare_matches.iter() {
+            let postings_list = &postings_lists[m.postings_list];
+            processed.reserve(postings_list.len());
+            for di in postings_list.as_ref() {
+                let simple_match = SimpleMatch {
+                    query_index: m.query_index,
+                    distance: m.distance,
+                    attribute: di.attribute,
+                    word_index: di.word_index,
+                    is_exact: m.is_exact,
+                };
+                processed.push(simple_match);
+            }
+        }
+
+        let processed = multiword_rewrite_matches(&mut processed, query_mapping);
+        document.processed_matches = processed.into_vec();
+    }
+}
+
+fn multiword_rewrite_matches(
+    matches: &mut [SimpleMatch],
+    query_mapping: &HashMap<QueryId, Range<usize>>,
+) -> SetBuf<SimpleMatch>
+{
+    matches.sort_unstable_by_key(|m| (m.attribute, m.word_index));
+
+    let mut padded_matches = Vec::with_capacity(matches.len());
+
+    // let before_padding = Instant::now();
+    // for each attribute of each document
+    for same_document_attribute in matches.linear_group_by_key(|m| m.attribute) {
+        // padding will only be applied
+        // to word indices in the same attribute
+        let mut padding = 0;
+        let mut iter = same_document_attribute.linear_group_by_key(|m| m.word_index);
+
+        // for each match at the same position
+        // in this document attribute
+        while let Some(same_word_index) = iter.next() {
+            // find the biggest padding
+            let mut biggest = 0;
+            for match_ in same_word_index {
+                let mut replacement = query_mapping[&(match_.query_index as usize)].clone();
+                let replacement_len = replacement.len();
+                let nexts = iter.remainder().linear_group_by_key(|m| m.word_index);
+
+                if let Some(query_index) = replacement.next() {
+                    let word_index = match_.word_index + padding as u16;
+                    let match_ = SimpleMatch { query_index, word_index, ..*match_ };
+                    padded_matches.push(match_);
+                }
+
+                let mut found = false;
+
+                // look ahead and if there already is a match
+                // corresponding to this padding word, abort the padding
+                'padding: for (x, next_group) in nexts.enumerate() {
+                    for (i, query_index) in replacement.clone().enumerate().skip(x) {
+                        let word_index = match_.word_index + padding as u16 + (i + 1) as u16;
+                        let padmatch = SimpleMatch { query_index, word_index, ..*match_ };
+
+                        for nmatch_ in next_group {
+                            let mut rep = query_mapping[&(nmatch_.query_index as usize)].clone();
+                            let query_index = rep.next().unwrap();
+                            if query_index == padmatch.query_index {
+                                if !found {
+                                    // if we find a corresponding padding for the
+                                    // first time we must push preceding paddings
+                                    for (i, query_index) in replacement.clone().enumerate().take(i) {
+                                        let word_index = match_.word_index + padding as u16 + (i + 1) as u16;
+                                        let match_ = SimpleMatch { query_index, word_index, ..*match_ };
+                                        padded_matches.push(match_);
+                                        biggest = biggest.max(i + 1);
+                                    }
+                                }
+
+                                padded_matches.push(padmatch);
+                                found = true;
+                                continue 'padding;
+                            }
+                        }
+                    }
+
+                    // if we do not find a corresponding padding in the
+                    // next groups so stop here and pad what was found
+                    break;
+                }
+
+                if !found {
+                    // if no padding was found in the following matches
+                    // we must insert the entire padding
+                    for (i, query_index) in replacement.enumerate() {
+                        let word_index = match_.word_index + padding as u16 + (i + 1) as u16;
+                        let match_ = SimpleMatch { query_index, word_index, ..*match_ };
+                        padded_matches.push(match_);
+                    }
+
+                    biggest = biggest.max(replacement_len - 1);
+                }
+            }
+
+            padding += biggest;
+        }
+    }
+
+    // debug!("padding matches took {:.02?}", before_padding.elapsed());
+
+    // With this check we can see that the loop above takes something
+    // like 43% of the search time even when no rewrite is needed.
+    // assert_eq!(before_matches, padded_matches);
+
+    SetBuf::from_dirty(padded_matches)
+}

meilisearch-core/src/criterion/number_of_words.rs

@@ -1,31 +0,0 @@
-use crate::criterion::Criterion;
-use crate::RawDocument;
-use slice_group_by::GroupBy;
-use std::cmp::Ordering;
-
-#[inline]
-fn number_of_query_words(query_index: &[u32]) -> usize {
-    query_index.linear_group().count()
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct NumberOfWords;
-
-impl Criterion for NumberOfWords {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = {
-            let query_index = lhs.query_index();
-            number_of_query_words(query_index)
-        };
-        let rhs = {
-            let query_index = rhs.query_index();
-            number_of_query_words(query_index)
-        };
-
-        lhs.cmp(&rhs).reverse()
-    }
-
-    fn name(&self) -> &str {
-        "NumberOfWords"
-    }
-}

meilisearch-core/src/criterion/proximity.rs

@@ -0,0 +1,68 @@
+use std::cmp::{self, Ordering};
+use slice_group_by::GroupBy;
+use crate::bucket_sort::{SimpleMatch};
+use crate::{RawDocument, MResult};
+use super::{Criterion, Context, ContextMut, prepare_bare_matches};
+
+const MAX_DISTANCE: u16 = 8;
+
+pub struct Proximity;
+
+impl Criterion for Proximity {
+    fn name(&self) -> &str { "proximity" }
+
+    fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>(
+        &self,
+        ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>,
+        documents: &mut [RawDocument<'r, 'tag>],
+    ) -> MResult<()>
+    {
+        prepare_bare_matches(documents, ctx.postings_lists, ctx.query_mapping);
+        Ok(())
+    }
+
+    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
+        fn index_proximity(lhs: u16, rhs: u16) -> u16 {
+            if lhs < rhs {
+                cmp::min(rhs - lhs, MAX_DISTANCE)
+            } else {
+                cmp::min(lhs - rhs, MAX_DISTANCE) + 1
+            }
+        }
+
+        fn attribute_proximity(lhs: SimpleMatch, rhs: SimpleMatch) -> u16 {
+            if lhs.attribute != rhs.attribute { MAX_DISTANCE }
+            else { index_proximity(lhs.word_index, rhs.word_index) }
+        }
+
+        fn min_proximity(lhs: &[SimpleMatch], rhs: &[SimpleMatch]) -> u16 {
+            let mut min_prox = u16::max_value();
+            for a in lhs {
+                for b in rhs {
+                    let prox = attribute_proximity(*a, *b);
+                    min_prox = cmp::min(min_prox, prox);
+                }
+            }
+            min_prox
+        }
+
+        fn matches_proximity(matches: &[SimpleMatch],) -> u16 {
+            let mut proximity = 0;
+            let mut iter = matches.linear_group_by_key(|m| m.query_index);
+
+            // iterate over groups by windows of size 2
+            let mut last = iter.next();
+            while let (Some(lhs), Some(rhs)) = (last, iter.next()) {
+                proximity += min_proximity(lhs, rhs);
+                last = Some(rhs);
+            }
+
+            proximity
+        }
+
+        let lhs = matches_proximity(&lhs.processed_matches);
+        let rhs = matches_proximity(&rhs.processed_matches);
+
+        lhs.cmp(&rhs)
+    }
+}

meilisearch-core/src/criterion/sort_by_attr.rs

@@ -1,10 +1,9 @@
 use std::cmp::Ordering;
 use std::error::Error;
 use std::fmt;
-
-use crate::criterion::Criterion;
+use meilisearch_schema::{Schema, FieldId};
 use crate::{RankedMap, RawDocument};
-use meilisearch_schema::{Schema, SchemaAttr};
+use super::{Criterion, Context};
 
 /// An helper struct that permit to sort documents by
 /// some of their stored attributes.
@@ -28,12 +27,12 @@ use meilisearch_schema::{Schema, SchemaAttr};
 /// let custom_ranking = SortByAttr::lower_is_better(&ranked_map, &schema, "published_at")?;
 ///
 /// let builder = CriteriaBuilder::with_capacity(8)
-///        .add(SumOfTypos)
-///        .add(NumberOfWords)
-///        .add(WordsProximity)
-///        .add(SumOfWordsAttribute)
-///        .add(SumOfWordsPosition)
-///        .add(Exact)
+///        .add(Typo)
+///        .add(Words)
+///        .add(Proximity)
+///        .add(Attribute)
+///        .add(WordsPosition)
+///        .add(Exactness)
 ///        .add(custom_ranking)
 ///        .add(DocumentId);
 ///
@@ -42,7 +41,7 @@ use meilisearch_schema::{Schema, SchemaAttr};
 /// ```
 pub struct SortByAttr<'a> {
     ranked_map: &'a RankedMap,
-    attr: SchemaAttr,
+    field_id: FieldId,
     reversed: bool,
 }
 
@@ -69,27 +68,31 @@ impl<'a> SortByAttr<'a> {
         attr_name: &str,
         reversed: bool,
     ) -> Result<SortByAttr<'a>, SortByAttrError> {
-        let attr = match schema.attribute(attr_name) {
-            Some(attr) => attr,
+        let field_id = match schema.id(attr_name) {
+            Some(field_id) => field_id,
             None => return Err(SortByAttrError::AttributeNotFound),
         };
 
-        if !schema.props(attr).is_ranked() {
+        if !schema.is_ranked(field_id) {
             return Err(SortByAttrError::AttributeNotRegisteredForRanking);
         }
 
         Ok(SortByAttr {
             ranked_map,
-            attr,
+            field_id,
             reversed,
         })
     }
 }
 
-impl<'a> Criterion for SortByAttr<'a> {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = self.ranked_map.get(lhs.id, self.attr);
-        let rhs = self.ranked_map.get(rhs.id, self.attr);
+impl Criterion for SortByAttr<'_> {
+    fn name(&self) -> &str {
+        "sort by attribute"
+    }
+
+    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
+        let lhs = self.ranked_map.get(lhs.id, self.field_id);
+        let rhs = self.ranked_map.get(rhs.id, self.field_id);
 
         match (lhs, rhs) {
             (Some(lhs), Some(rhs)) => {
@@ -105,10 +108,6 @@ impl<'a> Criterion for SortByAttr<'a> {
             (None, None) => Ordering::Equal,
         }
     }
-
-    fn name(&self) -> &str {
-        "SortByAttr"
-    }
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]

meilisearch-core/src/criterion/sum_of_typos.rs

@@ -1,116 +0,0 @@
-use std::cmp::Ordering;
-
-use slice_group_by::GroupBy;
-
-use crate::criterion::Criterion;
-use crate::RawDocument;
-
-// This function is a wrong logarithmic 10 function.
-// It is safe to panic on input number higher than 3,
-// the number of typos is never bigger than that.
-#[inline]
-fn custom_log10(n: u8) -> f32 {
-    match n {
-        0 => 0.0,     // log(1)
-        1 => 0.30102, // log(2)
-        2 => 0.47712, // log(3)
-        3 => 0.60205, // log(4)
-        _ => panic!("invalid number"),
-    }
-}
-
-#[inline]
-fn sum_matches_typos(query_index: &[u32], distance: &[u8]) -> usize {
-    let mut number_words: usize = 0;
-    let mut sum_typos = 0.0;
-    let mut index = 0;
-
-    for group in query_index.linear_group() {
-        sum_typos += custom_log10(distance[index]);
-        number_words += 1;
-        index += group.len();
-    }
-
-    (number_words as f32 / (sum_typos + 1.0) * 1000.0) as usize
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct SumOfTypos;
-
-impl Criterion for SumOfTypos {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = {
-            let query_index = lhs.query_index();
-            let distance = lhs.distance();
-            sum_matches_typos(query_index, distance)
-        };
-
-        let rhs = {
-            let query_index = rhs.query_index();
-            let distance = rhs.distance();
-            sum_matches_typos(query_index, distance)
-        };
-
-        lhs.cmp(&rhs).reverse()
-    }
-
-    fn name(&self) -> &str {
-        "SumOfTypos"
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    // typing: "Geox CEO"
-    //
-    // doc0: "Geox SpA: CEO and Executive"
-    // doc1: "Mt. Gox CEO Resigns From Bitcoin Foundation"
-    #[test]
-    fn one_typo_reference() {
-        let query_index0 = &[0, 1];
-        let distance0 = &[0, 0];
-
-        let query_index1 = &[0, 1];
-        let distance1 = &[1, 0];
-
-        let doc0 = sum_matches_typos(query_index0, distance0);
-        let doc1 = sum_matches_typos(query_index1, distance1);
-        assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
-    }
-
-    // typing: "bouton manchette"
-    //
-    // doc0: "bouton manchette"
-    // doc1: "bouton"
-    #[test]
-    fn no_typo() {
-        let query_index0 = &[0, 1];
-        let distance0 = &[0, 0];
-
-        let query_index1 = &[0];
-        let distance1 = &[0];
-
-        let doc0 = sum_matches_typos(query_index0, distance0);
-        let doc1 = sum_matches_typos(query_index1, distance1);
-        assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
-    }
-
-    // typing: "bouton manchztte"
-    //
-    // doc0: "bouton manchette"
-    // doc1: "bouton"
-    #[test]
-    fn one_typo() {
-        let query_index0 = &[0, 1];
-        let distance0 = &[0, 1];
-
-        let query_index1 = &[0];
-        let distance1 = &[0];
-
-        let doc0 = sum_matches_typos(query_index0, distance0);
-        let doc1 = sum_matches_typos(query_index1, distance1);
-        assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
-    }
-}

meilisearch-core/src/criterion/sum_of_words_attribute.rs

@@ -1,64 +0,0 @@
-use crate::criterion::Criterion;
-use crate::RawDocument;
-use slice_group_by::GroupBy;
-use std::cmp::Ordering;
-
-#[inline]
-fn sum_matches_attributes(query_index: &[u32], attribute: &[u16]) -> usize {
-    let mut sum_attributes = 0;
-    let mut index = 0;
-
-    for group in query_index.linear_group() {
-        sum_attributes += attribute[index] as usize;
-        index += group.len();
-    }
-
-    sum_attributes
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct SumOfWordsAttribute;
-
-impl Criterion for SumOfWordsAttribute {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = {
-            let query_index = lhs.query_index();
-            let attribute = lhs.attribute();
-            sum_matches_attributes(query_index, attribute)
-        };
-
-        let rhs = {
-            let query_index = rhs.query_index();
-            let attribute = rhs.attribute();
-            sum_matches_attributes(query_index, attribute)
-        };
-
-        lhs.cmp(&rhs)
-    }
-
-    fn name(&self) -> &str {
-        "SumOfWordsAttribute"
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    // typing: "soulier"
-    //
-    // doc0: { 0. "Soulier bleu", 1. "bla bla bla" }
-    // doc1: { 0. "Botte rouge", 1. "Soulier en cuir" }
-    #[test]
-    fn title_vs_description() {
-        let query_index0 = &[0];
-        let attribute0 = &[0];
-
-        let query_index1 = &[0];
-        let attribute1 = &[1];
-
-        let doc0 = sum_matches_attributes(query_index0, attribute0);
-        let doc1 = sum_matches_attributes(query_index1, attribute1);
-        assert_eq!(doc0.cmp(&doc1), Ordering::Less);
-    }
-}

meilisearch-core/src/criterion/sum_of_words_position.rs

@@ -1,64 +0,0 @@
-use crate::criterion::Criterion;
-use crate::RawDocument;
-use slice_group_by::GroupBy;
-use std::cmp::Ordering;
-
-#[inline]
-fn sum_matches_attribute_index(query_index: &[u32], word_index: &[u16]) -> usize {
-    let mut sum_word_index = 0;
-    let mut index = 0;
-
-    for group in query_index.linear_group() {
-        sum_word_index += word_index[index] as usize;
-        index += group.len();
-    }
-
-    sum_word_index
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct SumOfWordsPosition;
-
-impl Criterion for SumOfWordsPosition {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = {
-            let query_index = lhs.query_index();
-            let word_index = lhs.word_index();
-            sum_matches_attribute_index(query_index, word_index)
-        };
-
-        let rhs = {
-            let query_index = rhs.query_index();
-            let word_index = rhs.word_index();
-            sum_matches_attribute_index(query_index, word_index)
-        };
-
-        lhs.cmp(&rhs)
-    }
-
-    fn name(&self) -> &str {
-        "SumOfWordsPosition"
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    // typing: "soulier"
-    //
-    // doc0: "Soulier bleu"
-    // doc1: "Botte rouge et soulier noir"
-    #[test]
-    fn easy_case() {
-        let query_index0 = &[0];
-        let word_index0 = &[0];
-
-        let query_index1 = &[0];
-        let word_index1 = &[3];
-
-        let doc0 = sum_matches_attribute_index(query_index0, word_index0);
-        let doc1 = sum_matches_attribute_index(query_index1, word_index1);
-        assert_eq!(doc0.cmp(&doc1), Ordering::Less);
-    }
-}

meilisearch-core/src/criterion/typo.rs

@@ -0,0 +1,55 @@
+use std::cmp::Ordering;
+use crate::{RawDocument, MResult};
+use super::{Criterion, Context, ContextMut, prepare_query_distances};
+
+pub struct Typo;
+
+impl Criterion for Typo {
+    fn name(&self) -> &str { "typo" }
+
+    fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>(
+        &self,
+        ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>,
+        documents: &mut [RawDocument<'r, 'tag>],
+    ) -> MResult<()>
+    {
+        prepare_query_distances(documents, ctx.query_mapping, ctx.postings_lists);
+        Ok(())
+    }
+
+    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
+        // This function is a wrong logarithmic 10 function.
+        // It is safe to panic on input number higher than 3,
+        // the number of typos is never bigger than that.
+        #[inline]
+        fn custom_log10(n: u8) -> f32 {
+            match n {
+                0 => 0.0,     // log(1)
+                1 => 0.30102, // log(2)
+                2 => 0.47712, // log(3)
+                3 => 0.60205, // log(4)
+                _ => panic!("invalid number"),
+            }
+        }
+
+        #[inline]
+        fn compute_typos(distances: &[Option<u8>]) -> usize {
+            let mut number_words: usize = 0;
+            let mut sum_typos = 0.0;
+
+            for distance in distances {
+                if let Some(distance) = distance {
+                    sum_typos += custom_log10(*distance);
+                    number_words += 1;
+                }
+            }
+
+            (number_words as f32 / (sum_typos + 1.0) * 1000.0) as usize
+        }
+
+        let lhs = compute_typos(&lhs.processed_distances);
+        let rhs = compute_typos(&rhs.processed_distances);
+
+        lhs.cmp(&rhs).reverse()
+    }
+}

meilisearch-core/src/criterion/words.rs

@@ -0,0 +1,31 @@
+use std::cmp::Ordering;
+use crate::{RawDocument, MResult};
+use super::{Criterion, Context, ContextMut, prepare_query_distances};
+
+pub struct Words;
+
+impl Criterion for Words {
+    fn name(&self) -> &str { "words" }
+
+    fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>(
+        &self,
+        ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>,
+        documents: &mut [RawDocument<'r, 'tag>],
+    ) -> MResult<()>
+    {
+        prepare_query_distances(documents, ctx.query_mapping, ctx.postings_lists);
+        Ok(())
+    }
+
+    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
+        #[inline]
+        fn number_of_query_words(distances: &[Option<u8>]) -> usize {
+            distances.iter().cloned().filter(Option::is_some).count()
+        }
+
+        let lhs = number_of_query_words(&lhs.processed_distances);
+        let rhs = number_of_query_words(&rhs.processed_distances);
+
+        lhs.cmp(&rhs).reverse()
+    }
+}

meilisearch-core/src/criterion/words_position.rs

@@ -0,0 +1,37 @@
+use std::cmp::Ordering;
+use slice_group_by::GroupBy;
+use crate::bucket_sort::SimpleMatch;
+use crate::{RawDocument, MResult};
+use super::{Criterion, Context, ContextMut, prepare_bare_matches};
+
+pub struct WordsPosition;
+
+impl Criterion for WordsPosition {
+    fn name(&self) -> &str { "words position" }
+
+    fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>(
+        &self,
+        ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>,
+        documents: &mut [RawDocument<'r, 'tag>],
+    ) -> MResult<()>
+    {
+        prepare_bare_matches(documents, ctx.postings_lists, ctx.query_mapping);
+        Ok(())
+    }
+
+    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
+        #[inline]
+        fn sum_words_position(matches: &[SimpleMatch]) -> usize {
+            let mut sum_words_position = 0;
+            for group in matches.linear_group_by_key(|bm| bm.query_index) {
+                sum_words_position += group[0].word_index as usize;
+            }
+            sum_words_position
+        }
+
+        let lhs = sum_words_position(&lhs.processed_matches);
+        let rhs = sum_words_position(&rhs.processed_matches);
+
+        lhs.cmp(&rhs)
+    }
+}

meilisearch-core/src/criterion/words_proximity.rs

@@ -1,164 +0,0 @@
-use crate::criterion::Criterion;
-use crate::RawDocument;
-use slice_group_by::GroupBy;
-use std::cmp::{self, Ordering};
-
-const MAX_DISTANCE: u16 = 8;
-
-#[inline]
-fn clone_tuple<T: Clone, U: Clone>((a, b): (&T, &U)) -> (T, U) {
-    (a.clone(), b.clone())
-}
-
-fn index_proximity(lhs: u16, rhs: u16) -> u16 {
-    if lhs < rhs {
-        cmp::min(rhs - lhs, MAX_DISTANCE)
-    } else {
-        cmp::min(lhs - rhs, MAX_DISTANCE) + 1
-    }
-}
-
-fn attribute_proximity((lattr, lwi): (u16, u16), (rattr, rwi): (u16, u16)) -> u16 {
-    if lattr != rattr {
-        return MAX_DISTANCE;
-    }
-    index_proximity(lwi, rwi)
-}
-
-fn min_proximity((lattr, lwi): (&[u16], &[u16]), (rattr, rwi): (&[u16], &[u16])) -> u16 {
-    let mut min_prox = u16::max_value();
-
-    for a in lattr.iter().zip(lwi) {
-        for b in rattr.iter().zip(rwi) {
-            let a = clone_tuple(a);
-            let b = clone_tuple(b);
-            min_prox = cmp::min(min_prox, attribute_proximity(a, b));
-        }
-    }
-
-    min_prox
-}
-
-fn matches_proximity(
-    query_index: &[u32],
-    distance: &[u8],
-    attribute: &[u16],
-    word_index: &[u16],
-) -> u16 {
-    let mut query_index_groups = query_index.linear_group();
-    let mut proximity = 0;
-    let mut index = 0;
-
-    let get_attr_wi = |index: usize, group_len: usize| {
-        // retrieve the first distance group (with the lowest values)
-        let len = distance[index..index + group_len]
-            .linear_group()
-            .next()
-            .unwrap()
-            .len();
-
-        let rattr = &attribute[index..index + len];
-        let rwi = &word_index[index..index + len];
-
-        (rattr, rwi)
-    };
-
-    let mut last = query_index_groups.next().map(|group| {
-        let attr_wi = get_attr_wi(index, group.len());
-        index += group.len();
-        attr_wi
-    });
-
-    // iter by windows of size 2
-    while let (Some(lhs), Some(rhs)) = (last, query_index_groups.next()) {
-        let attr_wi = get_attr_wi(index, rhs.len());
-        proximity += min_proximity(lhs, attr_wi);
-        last = Some(attr_wi);
-        index += rhs.len();
-    }
-
-    proximity
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct WordsProximity;
-
-impl Criterion for WordsProximity {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = {
-            let query_index = lhs.query_index();
-            let distance = lhs.distance();
-            let attribute = lhs.attribute();
-            let word_index = lhs.word_index();
-            matches_proximity(query_index, distance, attribute, word_index)
-        };
-
-        let rhs = {
-            let query_index = rhs.query_index();
-            let distance = rhs.distance();
-            let attribute = rhs.attribute();
-            let word_index = rhs.word_index();
-            matches_proximity(query_index, distance, attribute, word_index)
-        };
-
-        lhs.cmp(&rhs)
-    }
-
-    fn name(&self) -> &str {
-        "WordsProximity"
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn three_different_attributes() {
-        // "soup" "of the" "the day"
-        //
-        // { id: 0, attr: 0, attr_index: 0 }
-        // { id: 1, attr: 1, attr_index: 0 }
-        // { id: 2, attr: 1, attr_index: 1 }
-        // { id: 2, attr: 2, attr_index: 0 }
-        // { id: 3, attr: 3, attr_index: 1 }
-
-        let query_index = &[0, 1, 2, 2, 3];
-        let distance = &[0, 0, 0, 0, 0];
-        let attribute = &[0, 1, 1, 2, 3];
-        let word_index = &[0, 0, 1, 0, 1];
-
-        //   soup -> of = 8
-        // + of -> the  = 1
-        // + the -> day = 8 (not 1)
-        assert_eq!(
-            matches_proximity(query_index, distance, attribute, word_index),
-            17
-        );
-    }
-
-    #[test]
-    fn two_different_attributes() {
-        // "soup day" "soup of the day"
-        //
-        // { id: 0, attr: 0, attr_index: 0 }
-        // { id: 0, attr: 1, attr_index: 0 }
-        // { id: 1, attr: 1, attr_index: 1 }
-        // { id: 2, attr: 1, attr_index: 2 }
-        // { id: 3, attr: 0, attr_index: 1 }
-        // { id: 3, attr: 1, attr_index: 3 }
-
-        let query_index = &[0, 0, 1, 2, 3, 3];
-        let distance = &[0, 0, 0, 0, 0, 0];
-        let attribute = &[0, 1, 1, 1, 0, 1];
-        let word_index = &[0, 0, 1, 2, 1, 3];
-
-        //   soup -> of = 1
-        // + of -> the  = 1
-        // + the -> day = 1
-        assert_eq!(
-            matches_proximity(query_index, distance, attribute, word_index),
-            3
-        );
-    }
-}

meilisearch-core/src/database.rs

@@ -8,6 +8,7 @@ use crossbeam_channel::{Receiver, Sender};
 use heed::types::{Str, Unit};
 use heed::{CompactionOption, Result as ZResult};
 use log::debug;
+use meilisearch_schema::Schema;
 
 use crate::{store, update, Index, MResult};
 
@@ -55,7 +56,23 @@ fn update_awaiter(
     index: Index,
 ) -> MResult<()> {
     let mut receiver = receiver.into_iter();
-    while let Some(UpdateEvent::NewUpdate) = receiver.next() {
+    while let Some(event) = receiver.next() {
+
+        // if we receive a *MustClear* event, clear the index and break the loop
+        if let UpdateEvent::MustClear = event {
+            let mut writer = env.typed_write_txn::<MainT>()?;
+            let mut update_writer = update_env.typed_write_txn::<UpdateT>()?;
+
+            store::clear(&mut writer, &mut update_writer, &index)?;
+
+            writer.commit()?;
+            update_writer.commit()?;
+
+            debug!("store {} cleared", index_uid);
+
+            break
+        }
+
         loop {
             // We instantiate a *write* transaction to *block* the thread
             // until the *other*, notifiying, thread commits
@@ -115,14 +132,6 @@ fn update_awaiter(
 
     debug!("update loop system stopped");
 
-    let mut writer = env.typed_write_txn::<MainT>()?;
-    let mut update_writer = update_env.typed_write_txn::<UpdateT>()?;
-    store::clear(&mut writer, &mut update_writer, &index)?;
-    writer.commit()?;
-    update_writer.commit()?;
-
-    debug!("store {} cleared", index_uid);
-
     Ok(())
 }
 
@@ -133,13 +142,13 @@ impl Database {
 
         fs::create_dir_all(&main_path)?;
         let env = heed::EnvOpenOptions::new()
-            .map_size(10 * 1024 * 1024 * 1024) // 10GB
+            .map_size(100 * 1024 * 1024 * 1024) // 100GB
             .max_dbs(3000)
             .open(main_path)?;
 
         fs::create_dir_all(&update_path)?;
         let update_env = heed::EnvOpenOptions::new()
-            .map_size(10 * 1024 * 1024 * 1024) // 10GB
+            .map_size(100 * 1024 * 1024 * 1024) // 100GB
             .max_dbs(3000)
             .open(update_path)?;
 
@@ -160,7 +169,7 @@ impl Database {
         // open the previously aggregated indexes
         let mut indexes = HashMap::new();
         for index_uid in must_open {
-            let (sender, receiver) = crossbeam_channel::bounded(100);
+            let (sender, receiver) = crossbeam_channel::unbounded();
             let index = match store::open(&env, &update_env, &index_uid, sender.clone())? {
                 Some(index) => index,
                 None => {
@@ -225,12 +234,17 @@ impl Database {
         match indexes_lock.entry(name.to_owned()) {
             Entry::Occupied(_) => Err(crate::Error::IndexAlreadyExists),
             Entry::Vacant(entry) => {
-                let (sender, receiver) = crossbeam_channel::bounded(100);
+                let (sender, receiver) = crossbeam_channel::unbounded();
                 let index = store::create(&self.env, &self.update_env, name, sender)?;
 
-                let mut writer = self.env.write_txn()?;
+                let mut writer = self.env.typed_write_txn::<MainT>()?;
                 self.indexes_store.put(&mut writer, name, &())?;
 
+                index.main.put_name(&mut writer, name)?;
+                index.main.put_created_at(&mut writer)?;
+                index.main.put_updated_at(&mut writer)?;
+                index.main.put_schema(&mut writer, &Schema::new())?;
+
                 let env_clone = self.env.clone();
                 let update_env_clone = self.update_env.clone();
                 let index_clone = index.clone();
@@ -307,8 +321,26 @@ impl Database {
         self.update_env.typed_write_txn::<UpdateT>()
     }
 
-    pub fn copy_and_compact_to_path<P: AsRef<Path>>(&self, path: P) -> ZResult<File> {
-        self.env.copy_to_path(path, CompactionOption::Enabled)
+    pub fn copy_and_compact_to_path<P: AsRef<Path>>(&self, path: P) -> ZResult<(File, File)> {
+        let path = path.as_ref();
+
+        let env_path = path.join("main");
+        let env_update_path = path.join("update");
+
+        fs::create_dir(&env_path)?;
+        fs::create_dir(&env_update_path)?;
+
+        let env_path = env_path.join("data.mdb");
+        let env_file = self.env.copy_to_path(&env_path, CompactionOption::Enabled)?;
+
+        let env_update_path = env_update_path.join("data.mdb");
+        match self.update_env.copy_to_path(env_update_path, CompactionOption::Enabled) {
+            Ok(update_env_file) => Ok((env_file, update_env_file)),
+            Err(e) => {
+                fs::remove_file(env_path)?;
+                Err(e)
+            },
+        }
     }
 
     pub fn indexes_uids(&self) -> Vec<String> {
@@ -327,6 +359,7 @@ mod tests {
 
     use crate::criterion::{self, CriteriaBuilder};
     use crate::update::{ProcessedUpdateResult, UpdateStatus};
+    use crate::settings::Settings;
     use crate::{Document, DocumentId};
     use serde::de::IgnoredAny;
     use std::sync::mpsc;
@@ -346,23 +379,25 @@ mod tests {
 
         database.set_update_callback(Box::new(update_fn));
 
-        let schema = {
+        let mut writer = db.main_write_txn().unwrap();
+        index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
+        writer.commit().unwrap();
+
+        // block until the transaction is processed
+
+        let settings = {
             let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
+                {
+                    "searchableAttributes": ["name", "description"],
+                    "displayedAttributes": ["name", "description"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.into_update().unwrap()
         };
 
         let mut update_writer = db.update_write_txn().unwrap();
-        let _update_id = index.schema_update(&mut update_writer, schema).unwrap();
+        let _update_id = index.settings_update(&mut update_writer, settings).unwrap();
         update_writer.commit().unwrap();
 
         let mut additions = index.documents_addition();
@@ -409,23 +444,23 @@ mod tests {
 
         database.set_update_callback(Box::new(update_fn));
 
-        let schema = {
+        let mut writer = db.main_write_txn().unwrap();
+        index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
+        writer.commit().unwrap();
+
+        let settings = {
             let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
+                {
+                    "searchableAttributes": ["name", "description"],
+                    "displayedAttributes": ["name", "description"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.into_update().unwrap()
         };
 
         let mut update_writer = db.update_write_txn().unwrap();
-        let _update_id = index.schema_update(&mut update_writer, schema).unwrap();
+        let _update_id = index.settings_update(&mut update_writer, settings).unwrap();
         update_writer.commit().unwrap();
 
         let mut additions = index.documents_addition();
@@ -453,7 +488,7 @@ mod tests {
 
         let update_reader = db.update_read_txn().unwrap();
         let result = index.update_status(&update_reader, update_id).unwrap();
-        assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_some());
+        assert_matches!(result, Some(UpdateStatus::Failed { content }) if content.error.is_some());
     }
 
     #[test]
@@ -471,19 +506,23 @@ mod tests {
 
         database.set_update_callback(Box::new(update_fn));
 
-        let schema = {
-            let data = r#"
-                identifier = "id"
+        let mut writer = db.main_write_txn().unwrap();
+        index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
+        writer.commit().unwrap();
 
-                [attributes."name"]
-                displayed = true
-                indexed = true
+        let settings = {
+            let data = r#"
+                {
+                    "searchableAttributes": ["name"],
+                    "displayedAttributes": ["name"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.into_update().unwrap()
         };
 
         let mut update_writer = db.update_write_txn().unwrap();
-        let _update_id = index.schema_update(&mut update_writer, schema).unwrap();
+        let _update_id = index.settings_update(&mut update_writer, settings).unwrap();
         update_writer.commit().unwrap();
 
         let mut additions = index.documents_addition();
@@ -522,23 +561,23 @@ mod tests {
 
         database.set_update_callback(Box::new(update_fn));
 
-        let schema = {
+        let mut writer = db.main_write_txn().unwrap();
+        index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
+        writer.commit().unwrap();
+
+        let settings = {
             let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
+                {
+                    "searchableAttributes": ["name", "description"],
+                    "displayedAttributes": ["name", "description"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.into_update().unwrap()
         };
 
         let mut update_writer = db.update_write_txn().unwrap();
-        let _update_id = index.schema_update(&mut update_writer, schema).unwrap();
+        let _update_id = index.settings_update(&mut update_writer, settings).unwrap();
         update_writer.commit().unwrap();
 
         let mut additions = index.documents_addition();
@@ -562,31 +601,19 @@ mod tests {
         let _update_id = additions.finalize(&mut update_writer).unwrap();
         update_writer.commit().unwrap();
 
-        let schema = {
+        let settings = {
             let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
-
-                [attributes."age"]
-                displayed = true
-                indexed = true
-
-                [attributes."sex"]
-                displayed = true
-                indexed = true
+                {
+                    "searchableAttributes": ["name", "description", "age", "sex"],
+                    "displayedAttributes": ["name", "description", "age", "sex"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.into_update().unwrap()
         };
 
         let mut writer = db.update_write_txn().unwrap();
-        let update_id = index.schema_update(&mut writer, schema).unwrap();
+        let update_id = index.settings_update(&mut writer, settings).unwrap();
         writer.commit().unwrap();
 
         // block until the transaction is processed
@@ -634,50 +661,33 @@ mod tests {
 
         // even try to search for a document
         let reader = db.main_read_txn().unwrap();
-        let results = index.query_builder().query(&reader, "21 ", 0..20).unwrap();
+        let (results, _nb_hits) = index.query_builder().query(&reader, "21 ", 0..20).unwrap();
         assert_matches!(results.len(), 1);
 
         reader.abort();
 
         // try to introduce attributes in the middle of the schema
-        let schema = {
+        let settings = {
             let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
-
-                [attributes."city"]
-                displayed = true
-                indexed = true
-
-                [attributes."age"]
-                displayed = true
-                indexed = true
-
-                [attributes."sex"]
-                displayed = true
-                indexed = true
+                {
+                    "searchableAttributes": ["name", "description", "city", "age", "sex"],
+                    "displayedAttributes": ["name", "description", "city", "age", "sex"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.into_update().unwrap()
         };
 
         let mut writer = db.update_write_txn().unwrap();
-        let update_id = index.schema_update(&mut writer, schema).unwrap();
+        let update_id = index.settings_update(&mut writer, settings).unwrap();
         writer.commit().unwrap();
 
         // block until the transaction is processed
         let _ = receiver.iter().find(|id| *id == update_id);
-
         // check if it has been accepted
         let update_reader = db.update_read_txn().unwrap();
         let result = index.update_status(&update_reader, update_id).unwrap();
-        assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_some());
+        assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
     }
 
     #[test]
@@ -695,23 +705,23 @@ mod tests {
 
         database.set_update_callback(Box::new(update_fn));
 
-        let schema = {
+        let mut writer = db.main_write_txn().unwrap();
+        index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
+        writer.commit().unwrap();
+
+        let settings = {
             let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
+                {
+                    "searchableAttributes": ["name", "description"],
+                    "displayedAttributes": ["name", "description"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.into_update().unwrap()
         };
 
         let mut writer = db.update_write_txn().unwrap();
-        let _update_id = index.schema_update(&mut writer, schema).unwrap();
+        let _update_id = index.settings_update(&mut writer, settings).unwrap();
         writer.commit().unwrap();
 
         let mut additions = index.documents_addition();
@@ -750,12 +760,12 @@ mod tests {
         assert!(document.is_none());
 
         let document: Option<IgnoredAny> = index
-            .document(&reader, None, DocumentId(7900334843754999545))
+            .document(&reader, None, DocumentId(7_900_334_843_754_999_545))
             .unwrap();
         assert!(document.is_some());
 
         let document: Option<IgnoredAny> = index
-            .document(&reader, None, DocumentId(8367468610878465872))
+            .document(&reader, None, DocumentId(8_367_468_610_878_465_872))
             .unwrap();
         assert!(document.is_some());
     }
@@ -775,26 +785,23 @@ mod tests {
 
         database.set_update_callback(Box::new(update_fn));
 
-        let schema = {
+        let mut writer = db.main_write_txn().unwrap();
+        index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
+        writer.commit().unwrap();
+
+        let settings = {
             let data = r#"
-                identifier = "id"
-
-                [attributes."id"]
-                displayed = true
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
+                {
+                    "searchableAttributes": ["name", "description"],
+                    "displayedAttributes": ["name", "description", "id"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.into_update().unwrap()
         };
 
         let mut writer = db.update_write_txn().unwrap();
-        let _update_id = index.schema_update(&mut writer, schema).unwrap();
+        let _update_id = index.settings_update(&mut writer, settings).unwrap();
         writer.commit().unwrap();
 
         let mut additions = index.documents_addition();
@@ -833,12 +840,12 @@ mod tests {
         assert!(document.is_none());
 
         let document: Option<IgnoredAny> = index
-            .document(&reader, None, DocumentId(7900334843754999545))
+            .document(&reader, None, DocumentId(7_900_334_843_754_999_545))
             .unwrap();
         assert!(document.is_some());
 
         let document: Option<IgnoredAny> = index
-            .document(&reader, None, DocumentId(8367468610878465872))
+            .document(&reader, None, DocumentId(8_367_468_610_878_465_872))
             .unwrap();
         assert!(document.is_some());
 
@@ -875,7 +882,7 @@ mod tests {
 
         let reader = db.main_read_txn().unwrap();
         let document: Option<serde_json::Value> = index
-            .document(&reader, None, DocumentId(7900334843754999545))
+            .document(&reader, None, DocumentId(7_900_334_843_754_999_545))
             .unwrap();
 
         let new_doc1 = serde_json::json!({
@@ -886,7 +893,7 @@ mod tests {
         assert_eq!(document, Some(new_doc1));
 
         let document: Option<serde_json::Value> = index
-            .document(&reader, None, DocumentId(8367468610878465872))
+            .document(&reader, None, DocumentId(8_367_468_610_878_465_872))
             .unwrap();
 
         let new_doc2 = serde_json::json!({
@@ -917,24 +924,23 @@ mod tests {
 
         database.set_update_callback(Box::new(update_fn));
 
-        let schema = {
+        let mut writer = db.main_write_txn().unwrap();
+        index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
+        writer.commit().unwrap();
+
+        let settings = {
             let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
+                {
+                    "searchableAttributes": ["name", "description"],
+                    "displayedAttributes": ["name", "description"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.into_update().unwrap()
         };
 
-        // add a schema to the index
         let mut writer = db.update_write_txn().unwrap();
-        let _update_id = index.schema_update(&mut writer, schema).unwrap();
+        let _update_id = index.settings_update(&mut writer, settings).unwrap();
         writer.commit().unwrap();
 
         // add documents to the index
@@ -985,23 +991,32 @@ mod tests {
 
         database.set_update_callback(Box::new(update_fn));
 
-        let schema = {
+        let mut writer = db.main_write_txn().unwrap();
+        index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
+        writer.commit().unwrap();
+
+        let settings = {
             let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."release_date"]
-                displayed = true
-                ranked = true
+                {
+                    "rankingRules": [
+                        "typo",
+                        "words",
+                        "proximity",
+                        "attribute",
+                        "wordsPosition",
+                        "exactness",
+                        "desc(release_date)"
+                    ],
+                    "searchableAttributes": ["name", "release_date"],
+                    "displayedAttributes": ["name", "release_date"]
+                }
             "#;
-            toml::from_str(data).unwrap()
+            let settings: Settings = serde_json::from_str(data).unwrap();
+            settings.into_update().unwrap()
         };
 
         let mut writer = db.update_write_txn().unwrap();
-        let _update_id = index.schema_update(&mut writer, schema).unwrap();
+        let _update_id = index.settings_update(&mut writer, settings).unwrap();
         writer.commit().unwrap();
 
         let mut additions = index.documents_addition();
@@ -1044,20 +1059,20 @@ mod tests {
 
         let builder = index.query_builder_with_criteria(criteria);
 
-        let results = builder.query(&reader, "Kevin", 0..20).unwrap();
+        let (results, _nb_hits) = builder.query(&reader, "Kevin", 0..20).unwrap();
         let mut iter = results.into_iter();
 
         assert_matches!(
             iter.next(),
             Some(Document {
-                id: DocumentId(7900334843754999545),
+                id: DocumentId(7_900_334_843_754_999_545),
                 ..
             })
         );
         assert_matches!(
             iter.next(),
             Some(Document {
-                id: DocumentId(8367468610878465872),
+                id: DocumentId(8_367_468_610_878_465_872),
                 ..
             })
         );

meilisearch-core/src/error.rs

@@ -1,17 +1,26 @@
 use crate::serde::{DeserializerError, SerializerError};
 use serde_json::Error as SerdeJsonError;
+use pest::error::Error as PestError;
+use crate::filters::Rule;
 use std::{error, fmt, io};
 
+pub use bincode::Error as BincodeError;
+pub use fst::Error as FstError;
+pub use heed::Error as HeedError;
+pub use pest::error as pest_error;
+
 pub type MResult<T> = Result<T, Error>;
 
 #[derive(Debug)]
 pub enum Error {
     Io(io::Error),
     IndexAlreadyExists,
-    SchemaDiffer,
+    MissingPrimaryKey,
     SchemaMissing,
     WordIndexMissing,
     MissingDocumentId,
+    MaxFieldsLimitExceeded,
+    Schema(meilisearch_schema::Error),
     Zlmdb(heed::Error),
     Fst(fst::Error),
     SerdeJson(SerdeJsonError),
@@ -19,6 +28,7 @@ pub enum Error {
     Serializer(SerializerError),
     Deserializer(DeserializerError),
     UnsupportedOperation(UnsupportedOperation),
+    FilterParseError(PestError<Rule>)
 }
 
 impl From<io::Error> for Error {
@@ -27,14 +37,42 @@ impl From<io::Error> for Error {
     }
 }
 
-impl From<heed::Error> for Error {
-    fn from(error: heed::Error) -> Error {
+impl From<PestError<Rule>> for Error {
+    fn from(error: PestError<Rule>) -> Error {
+        Error::FilterParseError(error.renamed_rules(|r| {
+            let s  = match r {
+                Rule::or => "OR",
+                Rule::and => "AND",
+                Rule::not => "NOT",
+                Rule::string => "string",
+                Rule::word => "word",
+                Rule::greater => "field > value",
+                Rule::less => "field < value",
+                Rule::eq => "field = value",
+                Rule::leq => "field <= value",
+                Rule::geq => "field >= value",
+                Rule::key => "key",
+                _ => "other",
+            };
+            s.to_string()
+        }))
+    }
+} 
+
+impl From<meilisearch_schema::Error> for Error {
+    fn from(error: meilisearch_schema::Error) -> Error {
+        Error::Schema(error)
+    }
+}
+
+impl From<HeedError> for Error {
+    fn from(error: HeedError) -> Error {
         Error::Zlmdb(error)
     }
 }
 
-impl From<fst::Error> for Error {
-    fn from(error: fst::Error) -> Error {
+impl From<FstError> for Error {
+    fn from(error: FstError) -> Error {
         Error::Fst(error)
     }
 }
@@ -45,8 +83,8 @@ impl From<SerdeJsonError> for Error {
     }
 }
 
-impl From<bincode::Error> for Error {
-    fn from(error: bincode::Error) -> Error {
+impl From<BincodeError> for Error {
+    fn from(error: BincodeError) -> Error {
         Error::Bincode(error)
     }
 }
@@ -75,10 +113,12 @@ impl fmt::Display for Error {
         match self {
             Io(e) => write!(f, "{}", e),
             IndexAlreadyExists => write!(f, "index already exists"),
-            SchemaDiffer => write!(f, "schemas differ"),
+            MissingPrimaryKey => write!(f, "schema cannot be built without a primary key"),
             SchemaMissing => write!(f, "this index does not have a schema"),
             WordIndexMissing => write!(f, "this index does not have a word index"),
             MissingDocumentId => write!(f, "document id is missing"),
+            MaxFieldsLimitExceeded => write!(f, "maximum number of fields in a document exceeded"),
+            Schema(e) => write!(f, "schema error; {}", e),
             Zlmdb(e) => write!(f, "heed error; {}", e),
             Fst(e) => write!(f, "fst error; {}", e),
             SerdeJson(e) => write!(f, "serde json error; {}", e),
@@ -86,6 +126,7 @@ impl fmt::Display for Error {
             Serializer(e) => write!(f, "serializer error; {}", e),
             Deserializer(e) => write!(f, "deserializer error; {}", e),
             UnsupportedOperation(op) => write!(f, "unsupported operation; {}", op),
+            FilterParseError(e) => write!(f, "error parsing filter; {}", e),
         }
     }
 }
@@ -95,7 +136,7 @@ impl error::Error for Error {}
 #[derive(Debug)]
 pub enum UnsupportedOperation {
     SchemaAlreadyExists,
-    CannotUpdateSchemaIdentifier,
+    CannotUpdateSchemaPrimaryKey,
     CannotReorderSchemaAttribute,
     CanOnlyIntroduceNewSchemaAttributesAtEnd,
     CannotRemoveSchemaAttribute,
@@ -106,7 +147,7 @@ impl fmt::Display for UnsupportedOperation {
         use self::UnsupportedOperation::*;
         match self {
             SchemaAlreadyExists => write!(f, "Cannot update index which already have a schema"),
-            CannotUpdateSchemaIdentifier => write!(f, "Cannot update the identifier of a schema"),
+            CannotUpdateSchemaPrimaryKey => write!(f, "Cannot update the primary key of a schema"),
             CannotReorderSchemaAttribute => write!(f, "Cannot reorder the attributes of a schema"),
             CanOnlyIntroduceNewSchemaAttributesAtEnd => {
                 write!(f, "Can only introduce new attributes at end of a schema")

meilisearch-core/src/filters/condition.rs

@@ -0,0 +1,267 @@
+use std::str::FromStr;
+use std::cmp::Ordering;
+
+use crate::error::Error;
+use crate::{store::Index, DocumentId, MainT};
+use heed::RoTxn;
+use meilisearch_schema::{FieldId, Schema};
+use pest::error::{Error as PestError, ErrorVariant};
+use pest::iterators::Pair;
+use serde_json::{Value, Number};
+use super::parser::Rule;
+
+#[derive(Debug)]
+enum ConditionType {
+    Greater,
+    Less,
+    Equal,
+    LessEqual,
+    GreaterEqual,
+    NotEqual,
+}
+
+/// We need to infer type when the filter is constructed
+/// and match every possible types it can be parsed into.
+#[derive(Debug)]
+struct ConditionValue<'a> {
+    string: &'a str,
+    boolean: Option<bool>,
+    number: Option<Number>
+}
+
+impl<'a> ConditionValue<'a> {
+    pub fn new(value: &Pair<'a, Rule>) -> Self {
+        let value = match value.as_rule() {
+            Rule::string | Rule::word => {
+                let string =  value.as_str();
+                let boolean = match value.as_str() {
+                    "true" => Some(true),
+                    "false" => Some(false),
+                    _ => None,
+                };
+                let number = Number::from_str(value.as_str()).ok();
+                ConditionValue { string, boolean, number }
+            },
+            _ => unreachable!(),
+        };
+        value
+    }
+
+    pub fn as_str(&self) -> &str {
+        self.string.as_ref()
+    }
+
+    pub fn as_number(&self) -> Option<&Number> {
+        self.number.as_ref()
+    }
+
+    pub fn as_bool(&self) -> Option<bool> {
+        self.boolean
+    }
+}
+
+#[derive(Debug)]
+pub struct Condition<'a> {
+    field: FieldId,
+    condition: ConditionType,
+    value: ConditionValue<'a>
+}
+
+fn get_field_value<'a>(schema: &Schema, pair: Pair<'a, Rule>) -> Result<(FieldId, ConditionValue<'a>), Error> {
+    let mut items = pair.into_inner();
+    // lexing ensures that we at least have a key
+    let key = items.next().unwrap();
+    let field = schema
+        .id(key.as_str())
+        .ok_or::<PestError<Rule>>(PestError::new_from_span(
+                ErrorVariant::CustomError {
+                    message: format!(
+                                 "attribute `{}` not found, available attributes are: {}",
+                                 key.as_str(),
+                                 schema.names().collect::<Vec<_>>().join(", ")
+                             ),
+                },
+                key.as_span()))?;
+    let value = ConditionValue::new(&items.next().unwrap());
+    Ok((field, value))
+}
+
+// undefined behavior with big numbers
+fn compare_numbers(lhs: &Number, rhs: &Number) -> Option<Ordering> {
+    match (lhs.as_i64(), lhs.as_u64(), lhs.as_f64(),
+        rhs.as_i64(), rhs.as_u64(), rhs.as_f64()) {
+    //    i64   u64  f64  i64  u64  f64
+        (Some(lhs), _, _, Some(rhs), _, _) => lhs.partial_cmp(&rhs),
+        (_, Some(lhs), _, _, Some(rhs), _) => lhs.partial_cmp(&rhs),
+        (_, _, Some(lhs), _, _, Some(rhs)) => lhs.partial_cmp(&rhs),
+        (_, _, _, _, _, _) => None,
+    }
+}
+
+impl<'a> Condition<'a> {
+    pub fn less(
+        item: Pair<'a, Rule>,
+        schema: &'a Schema,
+    ) -> Result<Self, Error> {
+        let (field, value) = get_field_value(schema, item)?;
+        let condition = ConditionType::Less;
+        Ok(Self { field, condition, value })
+    }
+
+    pub fn greater(
+        item: Pair<'a, Rule>,
+        schema: &'a Schema,
+    ) -> Result<Self, Error> {
+        let (field, value) = get_field_value(schema, item)?;
+        let condition = ConditionType::Greater;
+        Ok(Self { field, condition, value })
+    }
+
+    pub fn neq(
+        item: Pair<'a, Rule>,
+        schema: &'a Schema,
+    ) -> Result<Self, Error> {
+        let (field, value) = get_field_value(schema, item)?;
+        let condition = ConditionType::NotEqual;
+        Ok(Self { field, condition, value })
+    }
+
+    pub fn geq(
+        item: Pair<'a, Rule>,
+        schema: &'a Schema,
+    ) -> Result<Self, Error> {
+        let (field, value) = get_field_value(schema, item)?;
+        let condition = ConditionType::GreaterEqual;
+        Ok(Self { field, condition, value })
+    }
+
+    pub fn leq(
+        item: Pair<'a, Rule>,
+        schema: &'a Schema,
+    ) -> Result<Self, Error> {
+        let (field, value) = get_field_value(schema, item)?;
+        let condition = ConditionType::LessEqual;
+        Ok(Self { field, condition, value })
+    }
+
+    pub fn eq(
+        item: Pair<'a, Rule>,
+        schema: &'a Schema,
+    ) -> Result<Self, Error> {
+        let (field, value) = get_field_value(schema, item)?;
+        let condition = ConditionType::Equal;
+        Ok(Self { field, condition, value })
+    }
+
+    pub fn test(
+        &self,
+        reader: &RoTxn<MainT>,
+        index: &Index,
+        document_id: DocumentId,
+    ) -> Result<bool, Error> {
+        match index.document_attribute::<Value>(reader, document_id, self.field)? {
+            Some(Value::String(s)) => {
+                let value = self.value.as_str();
+                match self.condition {
+                    ConditionType::Equal => Ok(unicase::eq(value, &s)),
+                    ConditionType::NotEqual => Ok(!unicase::eq(value, &s)),
+                    _ => Ok(false)
+                }
+            },
+            Some(Value::Number(n)) => { 
+                if let Some(value) = self.value.as_number() {
+                    if let Some(ord) = compare_numbers(&n, value) {
+                        let res =  match self.condition {
+                            ConditionType::Equal => ord == Ordering::Equal,
+                            ConditionType::NotEqual => ord != Ordering::Equal,
+                            ConditionType::GreaterEqual => ord != Ordering::Less,
+                            ConditionType::LessEqual => ord != Ordering::Greater,
+                            ConditionType::Greater => ord == Ordering::Greater,
+                            ConditionType::Less => ord == Ordering::Less,
+                        };
+                        return Ok(res)
+                    } 
+                } 
+                Ok(false)
+            },
+            Some(Value::Bool(b)) => {
+                if let Some(value) = self.value.as_bool() {
+                    return match self.condition {
+                        ConditionType::Equal => Ok(b == value),
+                        ConditionType::NotEqual => Ok(b != value),
+                        _ => Ok(false)
+                    }
+                }
+                Ok(false)
+            },
+            _ => Ok(false),
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use serde_json::Number;
+    use std::cmp::Ordering;
+
+    #[test]
+    fn test_number_comp() {
+        // test both u64
+        let n1 = Number::from(1u64);
+        let n2 = Number::from(2u64);
+        assert_eq!(Some(Ordering::Less), compare_numbers(&n1, &n2));
+        assert_eq!(Some(Ordering::Greater), compare_numbers(&n2, &n1));
+        let n1 = Number::from(1u64);
+        let n2 = Number::from(1u64);
+        assert_eq!(Some(Ordering::Equal), compare_numbers(&n1, &n2));
+
+        // test both i64
+        let n1 = Number::from(1i64);
+        let n2 = Number::from(2i64);
+        assert_eq!(Some(Ordering::Less), compare_numbers(&n1, &n2));
+        assert_eq!(Some(Ordering::Greater), compare_numbers(&n2, &n1));
+        let n1 = Number::from(1i64);
+        let n2 = Number::from(1i64);
+        assert_eq!(Some(Ordering::Equal), compare_numbers(&n1, &n2));
+
+        // test both f64
+        let n1 = Number::from_f64(1f64).unwrap();
+        let n2 = Number::from_f64(2f64).unwrap();
+        assert_eq!(Some(Ordering::Less), compare_numbers(&n1, &n2));
+        assert_eq!(Some(Ordering::Greater), compare_numbers(&n2, &n1));
+        let n1 = Number::from_f64(1f64).unwrap();
+        let n2 = Number::from_f64(1f64).unwrap();
+        assert_eq!(Some(Ordering::Equal), compare_numbers(&n1, &n2));
+
+        // test one u64 and one f64
+        let n1 = Number::from_f64(1f64).unwrap();
+        let n2 = Number::from(2u64);
+        assert_eq!(Some(Ordering::Less), compare_numbers(&n1, &n2));
+        assert_eq!(Some(Ordering::Greater), compare_numbers(&n2, &n1));
+
+        // equality
+        let n1 = Number::from_f64(1f64).unwrap();
+        let n2 = Number::from(1u64);
+        assert_eq!(Some(Ordering::Equal), compare_numbers(&n1, &n2));
+        assert_eq!(Some(Ordering::Equal), compare_numbers(&n2, &n1));
+
+        // float is neg
+        let n1 = Number::from_f64(-1f64).unwrap();
+        let n2 = Number::from(1u64);
+        assert_eq!(Some(Ordering::Less), compare_numbers(&n1, &n2));
+        assert_eq!(Some(Ordering::Greater), compare_numbers(&n2, &n1));
+
+        // float is too big
+        let n1 = Number::from_f64(std::f64::MAX).unwrap();
+        let n2 = Number::from(1u64);
+        assert_eq!(Some(Ordering::Greater), compare_numbers(&n1, &n2));
+        assert_eq!(Some(Ordering::Less), compare_numbers(&n2, &n1));
+
+        // misc
+        let n1 = Number::from_f64(std::f64::MAX).unwrap();
+        let n2 = Number::from(std::u64::MAX);
+        assert_eq!(Some(Ordering::Greater), compare_numbers(&n1, &n2));
+        assert_eq!(Some( Ordering::Less ), compare_numbers(&n2, &n1));
+    }
+}

meilisearch-core/src/filters/mod.rs

@@ -0,0 +1,125 @@
+mod parser;
+mod condition;
+
+pub(crate) use parser::Rule;
+
+use std::ops::Not;
+
+use condition::Condition;
+use crate::error::Error;
+use crate::{DocumentId, MainT, store::Index};
+use heed::RoTxn;
+use meilisearch_schema::Schema;
+use parser::{PREC_CLIMBER, FilterParser};
+use pest::iterators::{Pair, Pairs};
+use pest::Parser;
+
+type FilterResult<'a> = Result<Filter<'a>, Error>;
+
+#[derive(Debug)]
+pub enum Filter<'a> {
+    Condition(Condition<'a>),
+    Or(Box<Self>, Box<Self>),
+    And(Box<Self>, Box<Self>),
+    Not(Box<Self>),
+}
+
+impl<'a> Filter<'a> {
+    pub fn parse(expr: &'a str, schema: &'a Schema) -> FilterResult<'a> {
+        let mut lexed = FilterParser::parse(Rule::prgm, expr.as_ref())?;
+        Self::build(lexed.next().unwrap().into_inner(), schema)
+    }
+
+    pub fn test(
+        &self,
+        reader: &RoTxn<MainT>,
+        index: &Index,
+        document_id: DocumentId,
+    ) -> Result<bool, Error> {
+        use Filter::*;
+        match self {
+            Condition(c) => c.test(reader, index, document_id),
+            Or(lhs, rhs) => Ok(
+                lhs.test(reader, index, document_id)? || rhs.test(reader, index, document_id)?
+            ),
+            And(lhs, rhs) => Ok(
+                lhs.test(reader, index, document_id)? && rhs.test(reader, index, document_id)?
+            ),
+            Not(op) => op.test(reader, index, document_id).map(bool::not),
+        }
+    }
+
+    fn build(expression: Pairs<'a, Rule>, schema: &'a Schema) -> FilterResult<'a> {
+        PREC_CLIMBER.climb(
+            expression,
+            |pair: Pair<Rule>| match pair.as_rule() {
+                Rule::eq => Ok(Filter::Condition(Condition::eq(pair, schema)?)),
+                Rule::greater => Ok(Filter::Condition(Condition::greater(pair, schema)?)),
+                Rule::less => Ok(Filter::Condition(Condition::less(pair, schema)?)),
+                Rule::neq => Ok(Filter::Condition(Condition::neq(pair, schema)?)),
+                Rule::geq => Ok(Filter::Condition(Condition::geq(pair, schema)?)),
+                Rule::leq => Ok(Filter::Condition(Condition::leq(pair, schema)?)),
+                Rule::prgm => Self::build(pair.into_inner(), schema),
+                Rule::not => Ok(Filter::Not(Box::new(Self::build(
+                    pair.into_inner(),
+                    schema,
+                )?))),
+                _ => unreachable!(),
+            },
+            |lhs: FilterResult, op: Pair<Rule>, rhs: FilterResult| match op.as_rule() {
+                Rule::or => Ok(Filter::Or(Box::new(lhs?), Box::new(rhs?))),
+                Rule::and => Ok(Filter::And(Box::new(lhs?), Box::new(rhs?))),
+                _ => unreachable!(),
+            },
+        )
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn invalid_syntax() {
+        assert!(FilterParser::parse(Rule::prgm, "field : id").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "field=hello hello").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "field=hello OR OR").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "OR field:hello").is_err());
+        assert!(FilterParser::parse(Rule::prgm, r#"field="hello world"#).is_err());
+        assert!(FilterParser::parse(Rule::prgm, r#"field='hello world"#).is_err());
+        assert!(FilterParser::parse(Rule::prgm, "NOT field=").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "N").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "(field=1").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "(field=1))").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "field=1ORfield=2").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "field=1 ( OR field=2)").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "hello world=1").is_err());
+        assert!(FilterParser::parse(Rule::prgm, "").is_err());
+        assert!(FilterParser::parse(Rule::prgm, r#"((((((hello=world)))))"#).is_err());
+    }
+
+    #[test]
+    fn valid_syntax() {
+        assert!(FilterParser::parse(Rule::prgm, "field = id").is_ok());
+        assert!(FilterParser::parse(Rule::prgm, "field=id").is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field >= 10"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field <= 10"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field="hello world""#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field='hello world'"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field > 10"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field < 10"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field < 10 AND NOT field=5"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field=true OR NOT field=5"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"NOT field=true OR NOT field=5"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field='hello world' OR ( NOT field=true OR NOT field=5 )"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field='hello \'worl\'d' OR ( NOT field=true OR NOT field=5 )"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"field="hello \"worl\"d" OR ( NOT field=true OR NOT field=5 )"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"((((((hello=world))))))"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#""foo bar" > 10"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#""foo bar" = 10"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"'foo bar' = 10"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"'foo bar' <= 10"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"'foo bar' != 10"#).is_ok());
+        assert!(FilterParser::parse(Rule::prgm, r#"bar != 10"#).is_ok());
+    }
+}

meilisearch-core/src/filters/parser/grammar.pest

@@ -0,0 +1,28 @@
+key = _{quoted | word}
+value = _{quoted | word}
+quoted = _{ (PUSH("'") | PUSH("\"")) ~ string ~ POP  }
+string = {char*}
+word = ${(LETTER | NUMBER | "_" | "-")+}
+
+char =  _{ !(PEEK | "\\") ~ ANY
+    | "\\" ~ (PEEK | "\\" | "/" | "b" | "f" | "n" | "r" | "t")
+    | "\\" ~ ("u" ~ ASCII_HEX_DIGIT{4})}
+
+condition = _{eq | greater | less | geq | leq | neq}
+geq = {key ~ ">=" ~ value}
+leq = {key ~ "<=" ~ value}
+neq = {key ~ "!=" ~ value}
+eq = {key ~ "=" ~ value}
+greater = {key ~ ">" ~ value}
+less = {key ~ "<" ~ value}
+
+prgm = {SOI ~ expr ~ EOI}
+expr = _{ ( term ~ (operation ~ term)* ) }
+term = _{ ("(" ~ expr ~ ")") | condition | not }
+operation = _{ and | or }
+	and = {"AND"}
+	or = {"OR"}
+
+not = {"NOT" ~ term}
+
+WHITESPACE = _{ " " }

meilisearch-core/src/filters/parser/mod.rs

@@ -0,0 +1,12 @@
+use once_cell::sync::Lazy;
+use pest::prec_climber::{Operator, Assoc, PrecClimber};
+
+pub static PREC_CLIMBER: Lazy<PrecClimber<Rule>> = Lazy::new(|| {
+    use Assoc::*;
+    use Rule::*;
+    pest::prec_climber::PrecClimber::new(vec![Operator::new(or, Left), Operator::new(and, Left)])
+});
+
+#[derive(Parser)]
+#[grammar = "filters/parser/grammar.pest"]
+pub struct FilterParser;

meilisearch-core/src/lib.rs

@@ -1,87 +1,185 @@
 #[cfg(test)]
 #[macro_use]
 extern crate assert_matches;
+#[macro_use]
+extern crate pest_derive;
 
 mod automaton;
-pub mod criterion;
+mod bucket_sort;
 mod database;
 mod distinct_map;
 mod error;
+mod filters;
 mod levenshtein;
 mod number;
 mod query_builder;
+mod query_tree;
+mod query_words_mapper;
 mod ranked_map;
 mod raw_document;
-pub mod raw_indexer;
 mod reordered_attrs;
+mod update;
+pub mod settings;
+pub mod criterion;
+pub mod raw_indexer;
 pub mod serde;
 pub mod store;
-mod update;
 
 pub use self::database::{BoxUpdateFn, Database, MainT, UpdateT};
-pub use self::error::{Error, MResult};
+pub use self::error::{Error, HeedError, FstError, MResult, pest_error};
+pub use self::filters::Filter;
 pub use self::number::{Number, ParseNumberError};
 pub use self::ranked_map::RankedMap;
 pub use self::raw_document::RawDocument;
 pub use self::store::Index;
 pub use self::update::{EnqueuedUpdateResult, ProcessedUpdateResult, UpdateStatus, UpdateType};
 pub use meilisearch_types::{DocIndex, DocumentId, Highlight};
+pub use meilisearch_schema::Schema;
+pub use query_words_mapper::QueryWordsMapper;
 
-#[doc(hidden)]
-#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct TmpMatch {
-    pub query_index: u32,
-    pub distance: u8,
-    pub attribute: u16,
-    pub word_index: u16,
-    pub is_exact: bool,
-}
+use std::convert::TryFrom;
+use std::collections::HashMap;
+use compact_arena::SmallArena;
+use log::{error, trace};
 
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+use crate::bucket_sort::PostingsListView;
+use crate::levenshtein::prefix_damerau_levenshtein;
+use crate::query_tree::{QueryId, QueryKind};
+use crate::reordered_attrs::ReorderedAttrs;
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
 pub struct Document {
     pub id: DocumentId,
     pub highlights: Vec<Highlight>,
 
     #[cfg(test)]
-    pub matches: Vec<TmpMatch>,
+    pub matches: Vec<crate::bucket_sort::SimpleMatch>,
+}
+
+fn highlights_from_raw_document<'a, 'tag, 'txn>(
+    raw_document: &RawDocument<'a, 'tag>,
+    queries_kinds: &HashMap<QueryId, &QueryKind>,
+    arena: &SmallArena<'tag, PostingsListView<'txn>>,
+    searchable_attrs: Option<&ReorderedAttrs>,
+    schema: &Schema,
+) -> Vec<Highlight>
+{
+    let mut highlights = Vec::new();
+
+    for bm in raw_document.bare_matches.iter() {
+        let postings_list = &arena[bm.postings_list];
+        let input = postings_list.input();
+        let kind = &queries_kinds.get(&bm.query_index);
+
+        for di in postings_list.iter() {
+            let covered_area = match kind {
+                Some(QueryKind::NonTolerant(query)) | Some(QueryKind::Tolerant(query)) => {
+                    let len = if query.len() > input.len() {
+                        input.len()
+                    } else {
+                        prefix_damerau_levenshtein(query.as_bytes(), input).1
+                    };
+                    u16::try_from(len).unwrap_or(u16::max_value())
+                },
+                _ => di.char_length,
+            };
+
+            let attribute = searchable_attrs
+                .and_then(|sa| sa.reverse(di.attribute))
+                .unwrap_or(di.attribute);
+
+            let attribute = match schema.indexed_pos_to_field_id(attribute) {
+                Some(field_id) => field_id.0,
+                None => {
+                    error!("Cannot convert indexed_pos {} to field_id", attribute);
+                    trace!("Schema is compromized; {:?}", schema);
+                    continue
+                }
+            };
+
+            let highlight = Highlight {
+                attribute,
+                char_index: di.char_index,
+                char_length: covered_area,
+            };
+
+            highlights.push(highlight);
+        }
+    }
+
+    highlights
 }
 
 impl Document {
     #[cfg(not(test))]
-    fn from_raw(raw: RawDocument) -> Document {
-        Document {
-            id: raw.id,
-            highlights: raw.highlights,
-        }
+    pub fn from_highlights(id: DocumentId, highlights: &[Highlight]) -> Document {
+        Document { id, highlights: highlights.to_owned() }
     }
 
     #[cfg(test)]
-    fn from_raw(raw: RawDocument) -> Document {
-        let len = raw.query_index().len();
-        let mut matches = Vec::with_capacity(len);
+    pub fn from_highlights(id: DocumentId, highlights: &[Highlight]) -> Document {
+        Document { id, highlights: highlights.to_owned(), matches: Vec::new() }
+    }
 
-        let query_index = raw.query_index();
-        let distance = raw.distance();
-        let attribute = raw.attribute();
-        let word_index = raw.word_index();
-        let is_exact = raw.is_exact();
+    #[cfg(not(test))]
+    pub fn from_raw<'a, 'tag, 'txn>(
+        raw_document: RawDocument<'a, 'tag>,
+        queries_kinds: &HashMap<QueryId, &QueryKind>,
+        arena: &SmallArena<'tag, PostingsListView<'txn>>,
+        searchable_attrs: Option<&ReorderedAttrs>,
+        schema: &Schema,
+    ) -> Document
+    {
+        let highlights = highlights_from_raw_document(
+            &raw_document,
+            queries_kinds,
+            arena,
+            searchable_attrs,
+            schema,
+        );
 
-        for i in 0..len {
-            let match_ = TmpMatch {
-                query_index: query_index[i],
-                distance: distance[i],
-                attribute: attribute[i],
-                word_index: word_index[i],
-                is_exact: is_exact[i],
+        Document { id: raw_document.id, highlights }
+    }
+
+    #[cfg(test)]
+    pub fn from_raw<'a, 'tag, 'txn>(
+        raw_document: RawDocument<'a, 'tag>,
+        queries_kinds: &HashMap<QueryId, &QueryKind>,
+        arena: &SmallArena<'tag, PostingsListView<'txn>>,
+        searchable_attrs: Option<&ReorderedAttrs>,
+        schema: &Schema,
+    ) -> Document
+    {
+        use crate::bucket_sort::SimpleMatch;
+
+        let highlights = highlights_from_raw_document(
+            &raw_document,
+            queries_kinds,
+            arena,
+            searchable_attrs,
+            schema,
+        );
+
+        let mut matches = Vec::new();
+        for sm in raw_document.processed_matches {
+            let attribute = searchable_attrs
+                .and_then(|sa| sa.reverse(sm.attribute))
+                .unwrap_or(sm.attribute);
+
+            let attribute = match schema.indexed_pos_to_field_id(attribute) {
+                Some(field_id) => field_id.0,
+                None => {
+                    error!("Cannot convert indexed_pos {} to field_id", attribute);
+                    trace!("Schema is compromized; {:?}", schema);
+                    continue
+                }
             };
-            matches.push(match_);
-        }
 
-        Document {
-            id: raw.id,
-            matches,
-            highlights: raw.highlights,
+            matches.push(SimpleMatch { attribute, ..sm });
         }
+        matches.sort_unstable();
+
+        Document { id: raw_document.id, highlights, matches }
     }
 }
 

meilisearch-core/src/number.rs

@@ -11,6 +11,13 @@ pub enum Number {
     Unsigned(u64),
     Signed(i64),
     Float(OrderedFloat<f64>),
+    Null,
+}
+
+impl Default for Number {
+    fn default() -> Self {
+        Self::Null
+    }
 }
 
 impl FromStr for Number {
@@ -56,7 +63,7 @@ impl PartialOrd for Number {
 
 impl Ord for Number {
     fn cmp(&self, other: &Self) -> Ordering {
-        use Number::{Float, Signed, Unsigned};
+        use Number::{Float, Signed, Unsigned, Null};
 
         match (*self, *other) {
             (Unsigned(a), Unsigned(b)) => a.cmp(&b),
@@ -80,6 +87,9 @@ impl Ord for Number {
             (Float(a), Unsigned(b)) => a.cmp(&OrderedFloat(b as f64)),
             (Float(a), Signed(b)) => a.cmp(&OrderedFloat(b as f64)),
             (Float(a), Float(b)) => a.cmp(&b),
+            (Null, Null) => Ordering::Equal,
+            (_, Null) => Ordering::Less,
+            (Null, _) => Ordering::Greater,
         }
     }
 }

meilisearch-core/src/query_enhancer.rs

@@ -1,398 +0,0 @@
-use std::ops::Range;
-use std::cmp::Ordering::{Less, Greater, Equal};
-
-/// Return `true` if the specified range can accept the given replacements words.
-/// Returns `false` if the replacements words are already present in the original query
-/// or if there is fewer replacement words than the range to replace.
-//
-//
-// ## Ignored because already present in original
-//
-//     new york city subway
-//     -------- ^^^^
-//   /          \
-//  [new york city]
-//
-//
-// ## Ignored because smaller than the original
-//
-//   new york city subway
-//   -------------
-//   \          /
-//    [new york]
-//
-//
-// ## Accepted because bigger than the original
-//
-//        NYC subway
-//        ---
-//       /   \
-//      /     \
-//     /       \
-//    /         \
-//   /           \
-//  [new york city]
-//
-fn rewrite_range_with<S, T>(query: &[S], range: Range<usize>, words: &[T]) -> bool
-where S: AsRef<str>,
-      T: AsRef<str>,
-{
-    if words.len() <= range.len() {
-        // there is fewer or equal replacement words
-        // than there is already in the replaced range
-        return false
-    }
-
-    // retrieve the part to rewrite but with the length
-    // of the replacement part
-    let original = query.iter().skip(range.start).take(words.len());
-
-    // check if the original query doesn't already contain
-    // the replacement words
-    !original.map(AsRef::as_ref).eq(words.iter().map(AsRef::as_ref))
-}
-
-type Origin = usize;
-type RealLength = usize;
-
-struct FakeIntervalTree {
-    intervals: Vec<(Range<usize>, (Origin, RealLength))>,
-}
-
-impl FakeIntervalTree {
-    fn new(mut intervals: Vec<(Range<usize>, (Origin, RealLength))>) -> FakeIntervalTree {
-        intervals.sort_unstable_by_key(|(r, _)| (r.start, r.end));
-        FakeIntervalTree { intervals }
-    }
-
-    fn query(&self, point: usize) -> Option<(Range<usize>, (Origin, RealLength))> {
-        let element = self.intervals.binary_search_by(|(r, _)| {
-            if point >= r.start {
-                if point < r.end { Equal } else { Less }
-            } else { Greater }
-        });
-
-        let n = match element { Ok(n) => n, Err(n) => n };
-
-        match self.intervals.get(n) {
-            Some((range, value)) if range.contains(&point) => Some((range.clone(), *value)),
-            _otherwise => None,
-        }
-    }
-}
-
-pub struct QueryEnhancerBuilder<'a, S> {
-    query: &'a [S],
-    origins: Vec<usize>,
-    real_to_origin: Vec<(Range<usize>, (Origin, RealLength))>,
-}
-
-impl<S: AsRef<str>> QueryEnhancerBuilder<'_, S> {
-    pub fn new(query: &[S]) -> QueryEnhancerBuilder<S> {
-        // we initialize origins query indices based on their positions
-        let origins: Vec<_> = (0..query.len() + 1).collect();
-        let real_to_origin = origins.iter().map(|&o| (o..o+1, (o, 1))).collect();
-
-        QueryEnhancerBuilder { query, origins, real_to_origin }
-    }
-
-    /// Update the final real to origin query indices mapping.
-    ///
-    /// `range` is the original words range that this `replacement` words replace
-    /// and `real` is the first real query index of these replacement words.
-    pub fn declare<T>(&mut self, range: Range<usize>, real: usize, replacement: &[T])
-    where T: AsRef<str>,
-    {
-        // check if the range of original words
-        // can be rewritten with the replacement words
-        if rewrite_range_with(self.query, range.clone(), replacement) {
-
-            // this range can be replaced so we need to
-            // modify the origins accordingly
-            let offset = replacement.len() - range.len();
-
-            let previous_padding = self.origins[range.end - 1];
-            let current_offset = (self.origins[range.end] - 1) - previous_padding;
-            let diff = offset.saturating_sub(current_offset);
-            self.origins[range.end] += diff;
-
-            for r in &mut self.origins[range.end + 1..] {
-                *r += diff;
-            }
-        }
-
-        // we need to store the real number and origins relations
-        // this way it will be possible to know by how many
-        // we need to pad real query indices
-        let real_range = real..real + replacement.len().max(range.len());
-        let real_length = replacement.len();
-        self.real_to_origin.push((real_range, (range.start, real_length)));
-    }
-
-    pub fn build(self) -> QueryEnhancer {
-        QueryEnhancer {
-            origins: self.origins,
-            real_to_origin: FakeIntervalTree::new(self.real_to_origin),
-        }
-    }
-}
-
-pub struct QueryEnhancer {
-    origins: Vec<usize>,
-    real_to_origin: FakeIntervalTree,
-}
-
-impl QueryEnhancer {
-    /// Returns the query indices to use to replace this real query index.
-    pub fn replacement(&self, real: u32) -> Range<u32> {
-        let real = real as usize;
-
-        // query the fake interval tree with the real query index
-        let (range, (origin, real_length)) =
-            self.real_to_origin
-                .query(real)
-                .expect("real has never been declared");
-
-        // if `real` is the end bound of the range
-        if (range.start + real_length - 1) == real {
-            let mut count = range.len();
-            let mut new_origin = origin;
-            for (i, slice) in self.origins[new_origin..].windows(2).enumerate() {
-                let len = slice[1] - slice[0];
-                count = count.saturating_sub(len);
-                if count == 0 { new_origin = origin + i; break }
-            }
-
-            let n = real - range.start;
-            let start = self.origins[origin];
-            let end = self.origins[new_origin + 1];
-            let remaining = (end - start) - n;
-
-            Range { start: (start + n) as u32, end: (start + n + remaining) as u32 }
-
-        } else {
-            // just return the origin along with
-            // the real position of the word
-            let n = real as usize - range.start;
-            let origin = self.origins[origin];
-
-            Range { start: (origin + n) as u32, end: (origin + n + 1) as u32 }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn original_unmodified() {
-        let query = ["new", "york", "city", "subway"];
-        //             0       1       2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // new york = new york city
-        builder.declare(0..2, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // new
-        assert_eq!(enhancer.replacement(1), 1..2); // york
-        assert_eq!(enhancer.replacement(2), 2..3); // city
-        assert_eq!(enhancer.replacement(3), 3..4); // subway
-        assert_eq!(enhancer.replacement(4), 0..1); // new
-        assert_eq!(enhancer.replacement(5), 1..2); // york
-        assert_eq!(enhancer.replacement(6), 2..3); // city
-    }
-
-    #[test]
-    fn simple_growing() {
-        let query = ["new", "york", "subway"];
-        //             0       1        2
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // new york = new york city
-        builder.declare(0..2, 3, &["new", "york", "city"]);
-        //                    ^      3       4       5
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // new
-        assert_eq!(enhancer.replacement(1), 1..3); // york
-        assert_eq!(enhancer.replacement(2), 3..4); // subway
-        assert_eq!(enhancer.replacement(3), 0..1); // new
-        assert_eq!(enhancer.replacement(4), 1..2); // york
-        assert_eq!(enhancer.replacement(5), 2..3); // city
-    }
-
-    #[test]
-    fn same_place_growings() {
-        let query = ["NY", "subway"];
-        //             0       1
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NY = new york
-        builder.declare(0..1, 2, &["new", "york"]);
-        //                    ^      2       3
-
-        // NY = new york city
-        builder.declare(0..1, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // NY = NYC
-        builder.declare(0..1, 7, &["NYC"]);
-        //                    ^      7
-
-        // NY = new york city
-        builder.declare(0..1, 8, &["new", "york", "city"]);
-        //                    ^      8       9      10
-
-        // subway = underground train
-        builder.declare(1..2, 11, &["underground", "train"]);
-        //                    ^          11          12
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..3); // NY
-        assert_eq!(enhancer.replacement(1), 3..5); // subway
-        assert_eq!(enhancer.replacement(2), 0..1); // new
-        assert_eq!(enhancer.replacement(3), 1..3); // york
-        assert_eq!(enhancer.replacement(4), 0..1); // new
-        assert_eq!(enhancer.replacement(5), 1..2); // york
-        assert_eq!(enhancer.replacement(6), 2..3); // city
-        assert_eq!(enhancer.replacement(7), 0..3); // NYC
-        assert_eq!(enhancer.replacement(8), 0..1); // new
-        assert_eq!(enhancer.replacement(9), 1..2); // york
-        assert_eq!(enhancer.replacement(10), 2..3); // city
-        assert_eq!(enhancer.replacement(11), 3..4); // underground
-        assert_eq!(enhancer.replacement(12), 4..5); // train
-    }
-
-    #[test]
-    fn bigger_growing() {
-        let query = ["NYC", "subway"];
-        //             0        1
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(0..1, 2, &["new", "york", "city"]);
-        //                    ^      2       3       4
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..3); // NYC
-        assert_eq!(enhancer.replacement(1), 3..4); // subway
-        assert_eq!(enhancer.replacement(2), 0..1); // new
-        assert_eq!(enhancer.replacement(3), 1..2); // york
-        assert_eq!(enhancer.replacement(4), 2..3); // city
-    }
-
-    #[test]
-    fn middle_query_growing() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // great
-        assert_eq!(enhancer.replacement(1), 1..2); // awesome
-        assert_eq!(enhancer.replacement(2), 2..5); // NYC
-        assert_eq!(enhancer.replacement(3), 5..6); // subway
-        assert_eq!(enhancer.replacement(4), 2..3); // new
-        assert_eq!(enhancer.replacement(5), 3..4); // york
-        assert_eq!(enhancer.replacement(6), 4..5); // city
-    }
-
-    #[test]
-    fn end_query_growing() {
-        let query = ["NYC", "subway"];
-        //             0        1
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(1..2, 2, &["underground", "train"]);
-        //                    ^         2            3
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // NYC
-        assert_eq!(enhancer.replacement(1), 1..3); // subway
-        assert_eq!(enhancer.replacement(2), 1..2); // underground
-        assert_eq!(enhancer.replacement(3), 2..3); // train
-    }
-
-    #[test]
-    fn multiple_growings() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // subway = underground train
-        builder.declare(3..4, 7, &["underground", "train"]);
-        //                    ^          7           8
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // great
-        assert_eq!(enhancer.replacement(1), 1..2); // awesome
-        assert_eq!(enhancer.replacement(2), 2..5); // NYC
-        assert_eq!(enhancer.replacement(3), 5..7); // subway
-        assert_eq!(enhancer.replacement(4), 2..3); // new
-        assert_eq!(enhancer.replacement(5), 3..4); // york
-        assert_eq!(enhancer.replacement(6), 4..5); // city
-        assert_eq!(enhancer.replacement(7), 5..6); // underground
-        assert_eq!(enhancer.replacement(8), 6..7); // train
-    }
-
-    #[test]
-    fn multiple_probable_growings() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // subway = underground train
-        builder.declare(3..4, 7, &["underground", "train"]);
-        //                    ^          7           8
-
-        // great awesome = good
-        builder.declare(0..2, 9, &["good"]);
-        //                    ^       9
-
-        // awesome NYC = NY
-        builder.declare(1..3, 10, &["NY"]);
-        //                    ^^     10
-
-        // NYC subway = metro
-        builder.declare(2..4, 11, &["metro"]);
-        //                    ^^      11
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0),  0..1); // great
-        assert_eq!(enhancer.replacement(1),  1..2); // awesome
-        assert_eq!(enhancer.replacement(2),  2..5); // NYC
-        assert_eq!(enhancer.replacement(3),  5..7); // subway
-        assert_eq!(enhancer.replacement(4),  2..3); // new
-        assert_eq!(enhancer.replacement(5),  3..4); // york
-        assert_eq!(enhancer.replacement(6),  4..5); // city
-        assert_eq!(enhancer.replacement(7),  5..6); // underground
-        assert_eq!(enhancer.replacement(8),  6..7); // train
-        assert_eq!(enhancer.replacement(9),  0..2); // good
-        assert_eq!(enhancer.replacement(10), 1..5); // NY
-        assert_eq!(enhancer.replacement(11), 2..5); // metro
-    }
-}

meilisearch-core/src/query_tree.rs

@@ -0,0 +1,560 @@
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::hash::{Hash, Hasher};
+use std::ops::Range;
+use std::time::Instant;
+use std::{cmp, fmt, iter::once};
+
+use fst::{IntoStreamer, Streamer};
+use itertools::{EitherOrBoth, merge_join_by};
+use meilisearch_tokenizer::split_query_string;
+use sdset::{Set, SetBuf, SetOperation};
+use log::debug;
+
+use crate::database::MainT;
+use crate::{store, DocumentId, DocIndex, MResult};
+use crate::automaton::{normalize_str, build_dfa, build_prefix_dfa, build_exact_dfa};
+use crate::QueryWordsMapper;
+
+#[derive(Clone, PartialEq, Eq, Hash)]
+pub enum Operation {
+    And(Vec<Operation>),
+    Or(Vec<Operation>),
+    Query(Query),
+}
+
+impl fmt::Debug for Operation {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fn pprint_tree(f: &mut fmt::Formatter<'_>, op: &Operation, depth: usize) -> fmt::Result {
+            match op {
+                Operation::And(children) => {
+                    writeln!(f, "{:1$}AND", "", depth * 2)?;
+                    children.iter().try_for_each(|c| pprint_tree(f, c, depth + 1))
+                },
+                Operation::Or(children) => {
+                    writeln!(f, "{:1$}OR", "", depth * 2)?;
+                    children.iter().try_for_each(|c| pprint_tree(f, c, depth + 1))
+                },
+                Operation::Query(query) => writeln!(f, "{:2$}{:?}", "", query, depth * 2),
+            }
+        }
+
+        pprint_tree(f, self, 0)
+    }
+}
+
+impl Operation {
+    fn tolerant(id: QueryId, prefix: bool, s: &str) -> Operation {
+        Operation::Query(Query { id, prefix, exact: true, kind: QueryKind::Tolerant(s.to_string()) })
+    }
+
+    fn non_tolerant(id: QueryId, prefix: bool, s: &str) -> Operation {
+        Operation::Query(Query { id, prefix, exact: true, kind: QueryKind::NonTolerant(s.to_string()) })
+    }
+
+    fn phrase2(id: QueryId, prefix: bool, (left, right): (&str, &str)) -> Operation {
+        let kind = QueryKind::Phrase(vec![left.to_owned(), right.to_owned()]);
+        Operation::Query(Query { id, prefix, exact: true, kind })
+    }
+}
+
+pub type QueryId = usize;
+
+#[derive(Clone, Eq)]
+pub struct Query {
+    pub id: QueryId,
+    pub prefix: bool,
+    pub exact: bool,
+    pub kind: QueryKind,
+}
+
+impl PartialEq for Query {
+    fn eq(&self, other: &Self) -> bool {
+        self.prefix == other.prefix && self.kind == other.kind
+    }
+}
+
+impl Hash for Query {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.prefix.hash(state);
+        self.kind.hash(state);
+    }
+}
+
+#[derive(Clone, PartialEq, Eq, Hash)]
+pub enum QueryKind {
+    Tolerant(String),
+    NonTolerant(String),
+    Phrase(Vec<String>),
+}
+
+impl fmt::Debug for Query {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let Query { id, prefix, kind, .. } = self;
+        let prefix = if *prefix { String::from("Prefix") } else { String::default() };
+        match kind {
+            QueryKind::NonTolerant(word) => {
+                f.debug_struct(&(prefix + "NonTolerant")).field("id", &id).field("word", &word).finish()
+            },
+            QueryKind::Tolerant(word) => {
+                f.debug_struct(&(prefix + "Tolerant")).field("id", &id).field("word", &word).finish()
+            },
+            QueryKind::Phrase(words) => {
+                f.debug_struct(&(prefix + "Phrase")).field("id", &id).field("words", &words).finish()
+            },
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct PostingsList {
+    docids: SetBuf<DocumentId>,
+    matches: SetBuf<DocIndex>,
+}
+
+pub struct Context {
+    pub words_set: fst::Set,
+    pub stop_words: fst::Set,
+    pub synonyms: store::Synonyms,
+    pub postings_lists: store::PostingsLists,
+    pub prefix_postings_lists: store::PrefixPostingsListsCache,
+}
+
+fn split_best_frequency<'a>(reader: &heed::RoTxn<MainT>, ctx: &Context, word: &'a str) -> MResult<Option<(&'a str, &'a str)>> {
+    let chars = word.char_indices().skip(1);
+    let mut best = None;
+
+    for (i, _) in chars {
+        let (left, right) = word.split_at(i);
+
+        let left_freq = ctx.postings_lists
+            .postings_list(reader, left.as_bytes())?
+            .map(|p| p.docids.len())
+            .unwrap_or(0);
+        let right_freq = ctx.postings_lists
+            .postings_list(reader, right.as_bytes())?
+            .map(|p| p.docids.len())
+            .unwrap_or(0);
+
+        let min_freq = cmp::min(left_freq, right_freq);
+        if min_freq != 0 && best.map_or(true, |(old, _, _)| min_freq > old) {
+            best = Some((min_freq, left, right));
+        }
+    }
+
+    Ok(best.map(|(_, l, r)| (l, r)))
+}
+
+fn fetch_synonyms(reader: &heed::RoTxn<MainT>, ctx: &Context, words: &[&str]) -> MResult<Vec<Vec<String>>> {
+    let words = normalize_str(&words.join(" "));
+    let set = ctx.synonyms.synonyms(reader, words.as_bytes())?.unwrap_or_default();
+
+    let mut strings = Vec::new();
+    let mut stream = set.stream();
+    while let Some(input) = stream.next() {
+        if let Ok(input) = std::str::from_utf8(input) {
+            let alts = input.split_ascii_whitespace().map(ToOwned::to_owned).collect();
+            strings.push(alts);
+        }
+    }
+
+    Ok(strings)
+}
+
+fn create_operation<I, F>(iter: I, f: F) -> Operation
+where I: IntoIterator<Item=Operation>,
+      F: Fn(Vec<Operation>) -> Operation,
+{
+    let mut iter = iter.into_iter();
+    match (iter.next(), iter.next()) {
+        (Some(first), None) => first,
+        (first, second) => f(first.into_iter().chain(second).chain(iter).collect()),
+    }
+}
+
+const MAX_NGRAM: usize = 3;
+
+pub fn create_query_tree(
+    reader: &heed::RoTxn<MainT>,
+    ctx: &Context,
+    query: &str,
+) -> MResult<(Operation, HashMap<QueryId, Range<usize>>)>
+{
+    let words = split_query_string(query).map(str::to_lowercase);
+    let words = words.filter(|w| !ctx.stop_words.contains(w));
+    let words: Vec<_> = words.enumerate().collect();
+
+    let mut mapper = QueryWordsMapper::new(words.iter().map(|(_, w)| w));
+
+    fn create_inner(
+        reader: &heed::RoTxn<MainT>,
+        ctx: &Context,
+        mapper: &mut QueryWordsMapper,
+        words: &[(usize, String)],
+    ) -> MResult<Vec<Operation>>
+    {
+        let mut alts = Vec::new();
+
+        for ngram in 1..=MAX_NGRAM {
+            if let Some(group) = words.get(..ngram) {
+                let mut group_ops = Vec::new();
+
+                let tail = &words[ngram..];
+                let is_last = tail.is_empty();
+
+                let mut group_alts = Vec::new();
+                match group {
+                    [(id, word)] => {
+                        let mut idgen = ((id + 1) * 100)..;
+                        let range = (*id)..id+1;
+
+                        let phrase = split_best_frequency(reader, ctx, word)?
+                            .map(|ws| {
+                                let id = idgen.next().unwrap();
+                                idgen.next().unwrap();
+                                mapper.declare(range.clone(), id, &[ws.0, ws.1]);
+                                Operation::phrase2(id, is_last, ws)
+                            });
+
+                        let synonyms = fetch_synonyms(reader, ctx, &[word])?
+                            .into_iter()
+                            .map(|alts| {
+                                let exact = alts.len() == 1;
+                                let id = idgen.next().unwrap();
+                                mapper.declare(range.clone(), id, &alts);
+
+                                let mut idgen = once(id).chain(&mut idgen);
+                                let iter = alts.into_iter().map(|w| {
+                                    let id = idgen.next().unwrap();
+                                    let kind = QueryKind::NonTolerant(w);
+                                    Operation::Query(Query { id, prefix: false, exact, kind })
+                                });
+
+                                create_operation(iter, Operation::And)
+                            });
+
+                        let original = Operation::tolerant(*id, is_last, word);
+
+                        group_alts.push(original);
+                        group_alts.extend(synonyms.chain(phrase));
+                    },
+                    words => {
+                        let id = words[0].0;
+                        let mut idgen = ((id + 1) * 100_usize.pow(ngram as u32))..;
+                        let range = id..id+ngram;
+
+                        let words: Vec<_> = words.iter().map(|(_, s)| s.as_str()).collect();
+
+                        for synonym in fetch_synonyms(reader, ctx, &words)? {
+                            let exact = synonym.len() == 1;
+                            let id = idgen.next().unwrap();
+                            mapper.declare(range.clone(), id, &synonym);
+
+                            let mut idgen = once(id).chain(&mut idgen);
+                            let synonym = synonym.into_iter().map(|s| {
+                                let id = idgen.next().unwrap();
+                                let kind = QueryKind::NonTolerant(s);
+                                Operation::Query(Query { id, prefix: false, exact, kind })
+                            });
+                            group_alts.push(create_operation(synonym, Operation::And));
+                        }
+
+                        let id = idgen.next().unwrap();
+                        let concat = words.concat();
+                        mapper.declare(range.clone(), id, &[&concat]);
+                        group_alts.push(Operation::non_tolerant(id, is_last, &concat));
+                    }
+                }
+
+                group_ops.push(create_operation(group_alts, Operation::Or));
+
+                if !tail.is_empty() {
+                    let tail_ops = create_inner(reader, ctx, mapper, tail)?;
+                    group_ops.push(create_operation(tail_ops, Operation::Or));
+                }
+
+                alts.push(create_operation(group_ops, Operation::And));
+            }
+        }
+
+        Ok(alts)
+    }
+
+    let alternatives = create_inner(reader, ctx, &mut mapper, &words)?;
+    let operation = Operation::Or(alternatives);
+    let mapping = mapper.mapping();
+
+    Ok((operation, mapping))
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct PostingsKey<'o> {
+    pub query: &'o Query,
+    pub input: Vec<u8>,
+    pub distance: u8,
+    pub is_exact: bool,
+}
+
+pub type Postings<'o, 'txn> = HashMap<PostingsKey<'o>, Cow<'txn, Set<DocIndex>>>;
+pub type Cache<'o, 'txn> = HashMap<&'o Operation, Cow<'txn, Set<DocumentId>>>;
+
+pub struct QueryResult<'o, 'txn> {
+    pub docids: Cow<'txn, Set<DocumentId>>,
+    pub queries: Postings<'o, 'txn>,
+}
+
+pub fn traverse_query_tree<'o, 'txn>(
+    reader: &'txn heed::RoTxn<MainT>,
+    ctx: &Context,
+    tree: &'o Operation,
+) -> MResult<QueryResult<'o, 'txn>>
+{
+    fn execute_and<'o, 'txn>(
+        reader: &'txn heed::RoTxn<MainT>,
+        ctx: &Context,
+        cache: &mut Cache<'o, 'txn>,
+        postings: &mut Postings<'o, 'txn>,
+        depth: usize,
+        operations: &'o [Operation],
+    ) -> MResult<Cow<'txn, Set<DocumentId>>>
+    {
+        debug!("{:1$}AND", "", depth * 2);
+
+        let before = Instant::now();
+        let mut results = Vec::new();
+
+        for op in operations {
+            if cache.get(op).is_none() {
+                let docids = match op {
+                    Operation::And(ops) => execute_and(reader, ctx, cache, postings, depth + 1, &ops)?,
+                    Operation::Or(ops) => execute_or(reader, ctx, cache, postings, depth + 1, &ops)?,
+                    Operation::Query(query) => execute_query(reader, ctx, postings, depth + 1, &query)?,
+                };
+                cache.insert(op, docids);
+            }
+        }
+
+        for op in operations {
+            if let Some(docids) = cache.get(op) {
+                results.push(docids.as_ref());
+            }
+        }
+
+        let op = sdset::multi::Intersection::new(results);
+        let docids = op.into_set_buf();
+
+        debug!("{:3$}--- AND fetched {} documents in {:.02?}", "", docids.len(), before.elapsed(), depth * 2);
+
+        Ok(Cow::Owned(docids))
+    }
+
+    fn execute_or<'o, 'txn>(
+        reader: &'txn heed::RoTxn<MainT>,
+        ctx: &Context,
+        cache: &mut Cache<'o, 'txn>,
+        postings: &mut Postings<'o, 'txn>,
+        depth: usize,
+        operations: &'o [Operation],
+    ) -> MResult<Cow<'txn, Set<DocumentId>>>
+    {
+        debug!("{:1$}OR", "", depth * 2);
+
+        let before = Instant::now();
+        let mut results = Vec::new();
+
+        for op in operations {
+            if cache.get(op).is_none() {
+                let docids = match op {
+                    Operation::And(ops) => execute_and(reader, ctx, cache, postings, depth + 1, &ops)?,
+                    Operation::Or(ops) => execute_or(reader, ctx, cache, postings, depth + 1, &ops)?,
+                    Operation::Query(query) => execute_query(reader, ctx, postings, depth + 1, &query)?,
+                };
+                cache.insert(op, docids);
+            }
+        }
+
+        for op in operations {
+            if let Some(docids) = cache.get(op) {
+                results.push(docids.as_ref());
+            }
+        }
+
+        let op = sdset::multi::Union::new(results);
+        let docids = op.into_set_buf();
+
+        debug!("{:3$}--- OR fetched {} documents in {:.02?}", "", docids.len(), before.elapsed(), depth * 2);
+
+        Ok(Cow::Owned(docids))
+    }
+
+    fn execute_query<'o, 'txn>(
+        reader: &'txn heed::RoTxn<MainT>,
+        ctx: &Context,
+        postings: &mut Postings<'o, 'txn>,
+        depth: usize,
+        query: &'o Query,
+    ) -> MResult<Cow<'txn, Set<DocumentId>>>
+    {
+        let before = Instant::now();
+
+        let Query { prefix, kind, exact, .. } = query;
+        let docids: Cow<Set<_>> = match kind {
+            QueryKind::Tolerant(word) => {
+                if *prefix && word.len() <= 2 {
+                    let prefix = {
+                        let mut array = [0; 4];
+                        let bytes = word.as_bytes();
+                        array[..bytes.len()].copy_from_slice(bytes);
+                        array
+                    };
+
+                    // We retrieve the cached postings lists for all
+                    // the words that starts with this short prefix.
+                    let result = ctx.prefix_postings_lists.prefix_postings_list(reader, prefix)?.unwrap_or_default();
+                    let key = PostingsKey { query, input: word.clone().into_bytes(), distance: 0, is_exact: false };
+                    postings.insert(key, result.matches);
+                    let prefix_docids = &result.docids;
+
+                    // We retrieve the exact postings list for the prefix,
+                    // because we must consider these matches as exact.
+                    let result = ctx.postings_lists.postings_list(reader, word.as_bytes())?.unwrap_or_default();
+                    let key = PostingsKey { query, input: word.clone().into_bytes(), distance: 0, is_exact: true };
+                    postings.insert(key, result.matches);
+                    let exact_docids = &result.docids;
+
+                    let before = Instant::now();
+                    let docids = sdset::duo::Union::new(prefix_docids, exact_docids).into_set_buf();
+                    debug!("{:4$}prefix docids ({} and {}) construction took {:.02?}",
+                        "", prefix_docids.len(), exact_docids.len(), before.elapsed(), depth * 2);
+
+                    Cow::Owned(docids)
+
+                } else {
+                    let dfa = if *prefix { build_prefix_dfa(word) } else { build_dfa(word) };
+
+                    let byte = word.as_bytes()[0];
+                    let mut stream = if byte == u8::max_value() {
+                        ctx.words_set.search(&dfa).ge(&[byte]).into_stream()
+                    } else {
+                        ctx.words_set.search(&dfa).ge(&[byte]).lt(&[byte + 1]).into_stream()
+                    };
+
+                    let before = Instant::now();
+                    let mut results = Vec::new();
+                    while let Some(input) = stream.next() {
+                        if let Some(result) = ctx.postings_lists.postings_list(reader, input)? {
+                            let distance = dfa.eval(input).to_u8();
+                            let is_exact = *exact && distance == 0 && input.len() == word.len();
+                            results.push(result.docids);
+                            let key = PostingsKey { query, input: input.to_owned(), distance, is_exact };
+                            postings.insert(key, result.matches);
+                        }
+                    }
+                    debug!("{:3$}docids retrieval ({:?}) took {:.02?}", "", results.len(), before.elapsed(), depth * 2);
+
+                    let before = Instant::now();
+                    let docids = if results.len() > 10 {
+                        let cap = results.iter().map(|dis| dis.len()).sum();
+                        let mut docids = Vec::with_capacity(cap);
+                        for dis in results {
+                            docids.extend_from_slice(&dis);
+                        }
+                        SetBuf::from_dirty(docids)
+                    } else {
+                        let sets = results.iter().map(AsRef::as_ref).collect();
+                        sdset::multi::Union::new(sets).into_set_buf()
+                    };
+                    debug!("{:2$}docids construction took {:.02?}", "", before.elapsed(), depth * 2);
+
+                    Cow::Owned(docids)
+                }
+            },
+            QueryKind::NonTolerant(word) => {
+                // TODO support prefix and non-prefix exact DFA
+                let dfa = build_exact_dfa(word);
+
+                let byte = word.as_bytes()[0];
+                let mut stream = if byte == u8::max_value() {
+                    ctx.words_set.search(&dfa).ge(&[byte]).into_stream()
+                } else {
+                    ctx.words_set.search(&dfa).ge(&[byte]).lt(&[byte + 1]).into_stream()
+                };
+
+                let before = Instant::now();
+                let mut results = Vec::new();
+                while let Some(input) = stream.next() {
+                    if let Some(result) = ctx.postings_lists.postings_list(reader, input)? {
+                        let distance = dfa.eval(input).to_u8();
+                        results.push(result.docids);
+                        let key = PostingsKey { query, input: input.to_owned(), distance, is_exact: *exact };
+                        postings.insert(key, result.matches);
+                    }
+                }
+                debug!("{:3$}docids retrieval ({:?}) took {:.02?}", "", results.len(), before.elapsed(), depth * 2);
+
+                let before = Instant::now();
+                let docids = if results.len() > 10 {
+                    let cap = results.iter().map(|dis| dis.len()).sum();
+                    let mut docids = Vec::with_capacity(cap);
+                    for dis in results {
+                        docids.extend_from_slice(&dis);
+                    }
+                    SetBuf::from_dirty(docids)
+                } else {
+                    let sets = results.iter().map(AsRef::as_ref).collect();
+                    sdset::multi::Union::new(sets).into_set_buf()
+                };
+                debug!("{:2$}docids construction took {:.02?}", "", before.elapsed(), depth * 2);
+
+                Cow::Owned(docids)
+            },
+            QueryKind::Phrase(words) => {
+                // TODO support prefix and non-prefix exact DFA
+                if let [first, second] = words.as_slice() {
+                    let first = ctx.postings_lists.postings_list(reader, first.as_bytes())?.unwrap_or_default();
+                    let second = ctx.postings_lists.postings_list(reader, second.as_bytes())?.unwrap_or_default();
+
+                    let iter = merge_join_by(first.matches.as_slice(), second.matches.as_slice(), |a, b| {
+                        let x = (a.document_id, a.attribute, (a.word_index as u32) + 1);
+                        let y = (b.document_id, b.attribute, b.word_index as u32);
+                        x.cmp(&y)
+                    });
+
+                    let matches: Vec<_> = iter
+                        .filter_map(EitherOrBoth::both)
+                        .flat_map(|(a, b)| once(*a).chain(Some(*b)))
+                        .collect();
+
+                    let before = Instant::now();
+                    let mut docids: Vec<_> = matches.iter().map(|m| m.document_id).collect();
+                    docids.dedup();
+                    let docids = SetBuf::new(docids).unwrap();
+                    debug!("{:2$}docids construction took {:.02?}", "", before.elapsed(), depth * 2);
+
+                    let matches = Cow::Owned(SetBuf::from_dirty(matches));
+                    let key = PostingsKey { query, input: vec![], distance: 0, is_exact: true };
+                    postings.insert(key, matches);
+
+                    Cow::Owned(docids)
+                } else {
+                    debug!("{:2$}{:?} skipped", "", words, depth * 2);
+                    Cow::default()
+                }
+            },
+        };
+
+        debug!("{:4$}{:?} fetched {:?} documents in {:.02?}", "", query, docids.len(), before.elapsed(), depth * 2);
+        Ok(docids)
+    }
+
+    let mut cache = Cache::new();
+    let mut postings = Postings::new();
+
+    let docids = match tree {
+        Operation::And(ops) => execute_and(reader, ctx, &mut cache, &mut postings, 0, &ops)?,
+        Operation::Or(ops) => execute_or(reader, ctx, &mut cache, &mut postings, 0, &ops)?,
+        Operation::Query(query) => execute_query(reader, ctx, &mut postings, 0, &query)?,
+    };
+
+    Ok(QueryResult { docids, queries: postings })
+}

meilisearch-core/src/query_words_mapper.rs

@@ -0,0 +1,415 @@
+use std::collections::HashMap;
+use std::iter::FromIterator;
+use std::ops::Range;
+use intervaltree::{Element, IntervalTree};
+
+pub type QueryId = usize;
+
+pub struct QueryWordsMapper {
+    originals: Vec<String>,
+    mappings: HashMap<QueryId, (Range<usize>, Vec<String>)>,
+}
+
+impl QueryWordsMapper {
+    pub fn new<I, A>(originals: I) -> QueryWordsMapper
+    where I: IntoIterator<Item = A>,
+          A: ToString,
+    {
+        let originals = originals.into_iter().map(|s| s.to_string()).collect();
+        QueryWordsMapper { originals, mappings: HashMap::new() }
+    }
+
+    pub fn declare<I, A>(&mut self, range: Range<usize>, id: QueryId, replacement: I)
+    where I: IntoIterator<Item = A>,
+          A: ToString,
+    {
+        assert!(range.len() != 0);
+        assert!(self.originals.get(range.clone()).is_some());
+        assert!(id >= self.originals.len());
+
+        let replacement: Vec<_> = replacement.into_iter().map(|s| s.to_string()).collect();
+
+        assert!(!replacement.is_empty());
+
+        // We detect words at the end and at the front of the
+        // replacement that are common with the originals:
+        //
+        //     x a b c d e f g
+        //       ^^^/   \^^^
+        //     a b x c d k j e f
+        //     ^^^           ^^^
+        //
+
+        let left = &self.originals[..range.start];
+        let right = &self.originals[range.end..];
+
+        let common_left = longest_common_prefix(left, &replacement);
+        let common_right = longest_common_prefix(&replacement, right);
+
+        for i in 0..common_left {
+            let range = range.start - common_left + i..range.start - common_left + i + 1;
+            let replacement = vec![replacement[i].clone()];
+            self.mappings.insert(id + i, (range, replacement));
+        }
+
+        {
+            let replacement = replacement[common_left..replacement.len() - common_right].iter().cloned().collect();
+            self.mappings.insert(id + common_left, (range.clone(), replacement));
+        }
+
+        for i in 0..common_right {
+            let id = id + replacement.len() - common_right + i;
+            let range = range.end + i..range.end + i + 1;
+            let replacement = vec![replacement[replacement.len() - common_right + i].clone()];
+            self.mappings.insert(id, (range, replacement));
+        }
+    }
+
+    pub fn mapping(self) -> HashMap<QueryId, Range<usize>> {
+        let mappings = self.mappings.into_iter().map(|(i, (r, v))| (r, (i, v)));
+        let intervals = IntervalTree::from_iter(mappings);
+
+        let mut output = HashMap::new();
+        let mut offset = 0;
+
+        // We map each original word to the biggest number of
+        // associated words.
+        for i in 0..self.originals.len() {
+            let max = intervals.query_point(i)
+                .filter_map(|e| {
+                    if e.range.end - 1 == i {
+                        let len = e.value.1.iter().skip(i - e.range.start).count();
+                        if len != 0 { Some(len) } else { None }
+                    } else { None }
+                })
+                .max()
+                .unwrap_or(1);
+
+            let range = i + offset..i + offset + max;
+            output.insert(i, range);
+            offset += max - 1;
+        }
+
+        // We retrieve the range that each original word
+        // is mapped to and apply it to each of the words.
+        for i in 0..self.originals.len() {
+
+            let iter = intervals.query_point(i).filter(|e| e.range.end - 1 == i);
+            for Element { range, value: (id, words) } in iter {
+
+                // We ask for the complete range mapped to the area we map.
+                let start = output.get(&range.start).map(|r| r.start).unwrap_or(range.start);
+                let end = output.get(&(range.end - 1)).map(|r| r.end).unwrap_or(range.end);
+                let range = start..end;
+
+                // We map each query id to one word until the last,
+                // we map it to the remainings words.
+                let add = range.len() - words.len();
+                for (j, x) in range.take(words.len()).enumerate() {
+                    let add = if j == words.len() - 1 { add } else { 0 }; // is last?
+                    let range = x..x + 1 + add;
+                    output.insert(id + j, range);
+                }
+            }
+        }
+
+        output
+    }
+}
+
+fn longest_common_prefix<T: Eq + std::fmt::Debug>(a: &[T], b: &[T]) -> usize {
+    let mut best = None;
+    for i in (0..a.len()).rev() {
+        let count = a[i..].iter().zip(b).take_while(|(a, b)| a == b).count();
+        best = match best {
+            Some(old) if count > old => Some(count),
+            Some(_) => break,
+            None => Some(count),
+        };
+    }
+    best.unwrap_or(0)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn original_unmodified() {
+        let query = ["new", "york", "city", "subway"];
+        //             0       1       2        3
+        let mut builder = QueryWordsMapper::new(&query);
+
+        // new york = new york city
+        builder.declare(0..2, 4, &["new", "york", "city"]);
+        //                    ^      4       5       6
+
+        // new = new york city
+        builder.declare(0..1, 7, &["new", "york", "city"]);
+        //                    ^      7       8       9
+
+        let mapping = builder.mapping();
+
+        assert_eq!(mapping[&0], 0..1); // new
+        assert_eq!(mapping[&1], 1..2); // york
+        assert_eq!(mapping[&2], 2..3); // city
+        assert_eq!(mapping[&3], 3..4); // subway
+
+        assert_eq!(mapping[&4], 0..1); // new
+        assert_eq!(mapping[&5], 1..2); // york
+        assert_eq!(mapping[&6], 2..3); // city
+
+        assert_eq!(mapping[&7], 0..1); // new
+        assert_eq!(mapping[&8], 1..2); // york
+        assert_eq!(mapping[&9], 2..3); // city
+    }
+
+    #[test]
+    fn original_unmodified2() {
+        let query = ["new", "york", "city", "subway"];
+        //             0       1       2        3
+        let mut builder = QueryWordsMapper::new(&query);
+
+        // city subway = new york city underground train
+        builder.declare(2..4, 4, &["new", "york", "city", "underground", "train"]);
+        //                    ^      4      5       6           7           8
+
+        let mapping = builder.mapping();
+
+        assert_eq!(mapping[&0], 0..1); // new
+        assert_eq!(mapping[&1], 1..2); // york
+        assert_eq!(mapping[&2], 2..3); // city
+        assert_eq!(mapping[&3], 3..5); // subway
+
+        assert_eq!(mapping[&4], 0..1); // new
+        assert_eq!(mapping[&5], 1..2); // york
+        assert_eq!(mapping[&6], 2..3); // city
+        assert_eq!(mapping[&7], 3..4); // underground
+        assert_eq!(mapping[&8], 4..5); // train
+    }
+
+    #[test]
+    fn original_unmodified3() {
+        let query = ["a", "b", "x", "x", "a", "b", "c", "d", "e", "f", "g"];
+        //            0    1    2    3    4    5    6    7    8    9    10
+        let mut builder = QueryWordsMapper::new(&query);
+
+        // c d = a b x c d k j e f
+        builder.declare(6..8, 11, &["a", "b", "x", "c", "d", "k", "j", "e", "f"]);
+        //                    ^^    11   12   13   14   15   16   17   18   19
+
+        let mapping = builder.mapping();
+
+        assert_eq!(mapping[&0],  0..1); // a
+        assert_eq!(mapping[&1],  1..2); // b
+        assert_eq!(mapping[&2],  2..3); // x
+        assert_eq!(mapping[&3],  3..4); // x
+        assert_eq!(mapping[&4],  4..5); // a
+        assert_eq!(mapping[&5],  5..6); // b
+        assert_eq!(mapping[&6],  6..7); // c
+        assert_eq!(mapping[&7],  7..11); // d
+        assert_eq!(mapping[&8],  11..12); // e
+        assert_eq!(mapping[&9],  12..13); // f
+        assert_eq!(mapping[&10], 13..14); // g
+
+        assert_eq!(mapping[&11], 4..5); // a
+        assert_eq!(mapping[&12], 5..6); // b
+        assert_eq!(mapping[&13], 6..7); // x
+        assert_eq!(mapping[&14], 7..8); // c
+        assert_eq!(mapping[&15], 8..9); // d
+        assert_eq!(mapping[&16], 9..10); // k
+        assert_eq!(mapping[&17], 10..11); // j
+        assert_eq!(mapping[&18], 11..12); // e
+        assert_eq!(mapping[&19], 12..13); // f
+    }
+
+    #[test]
+    fn simple_growing() {
+        let query = ["new", "york", "subway"];
+        //             0       1        2
+        let mut builder = QueryWordsMapper::new(&query);
+
+        // new york = new york city
+        builder.declare(0..2, 3, &["new", "york", "city"]);
+        //                    ^      3       4       5
+
+        let mapping = builder.mapping();
+
+        assert_eq!(mapping[&0], 0..1); // new
+        assert_eq!(mapping[&1], 1..3); // york
+        assert_eq!(mapping[&2], 3..4); // subway
+        assert_eq!(mapping[&3], 0..1); // new
+        assert_eq!(mapping[&4], 1..2); // york
+        assert_eq!(mapping[&5], 2..3); // city
+    }
+
+    #[test]
+    fn same_place_growings() {
+        let query = ["NY", "subway"];
+        //             0       1
+        let mut builder = QueryWordsMapper::new(&query);
+
+        // NY = new york
+        builder.declare(0..1, 2, &["new", "york"]);
+        //                    ^      2       3
+
+        // NY = new york city
+        builder.declare(0..1, 4, &["new", "york", "city"]);
+        //                    ^      4       5       6
+
+        // NY = NYC
+        builder.declare(0..1, 7, &["NYC"]);
+        //                    ^      7
+
+        // NY = new york city
+        builder.declare(0..1, 8, &["new", "york", "city"]);
+        //                    ^      8       9      10
+
+        // subway = underground train
+        builder.declare(1..2, 11, &["underground", "train"]);
+        //                    ^          11          12
+
+        let mapping = builder.mapping();
+
+        assert_eq!(mapping[&0], 0..3); // NY
+        assert_eq!(mapping[&1], 3..5); // subway
+        assert_eq!(mapping[&2], 0..1); // new
+        assert_eq!(mapping[&3], 1..3); // york
+        assert_eq!(mapping[&4], 0..1); // new
+        assert_eq!(mapping[&5], 1..2); // york
+        assert_eq!(mapping[&6], 2..3); // city
+        assert_eq!(mapping[&7], 0..3); // NYC
+        assert_eq!(mapping[&8], 0..1); // new
+        assert_eq!(mapping[&9], 1..2); // york
+        assert_eq!(mapping[&10], 2..3); // city
+        assert_eq!(mapping[&11], 3..4); // underground
+        assert_eq!(mapping[&12], 4..5); // train
+    }
+
+    #[test]
+    fn bigger_growing() {
+        let query = ["NYC", "subway"];
+        //             0        1
+        let mut builder = QueryWordsMapper::new(&query);
+
+        // NYC = new york city
+        builder.declare(0..1, 2, &["new", "york", "city"]);
+        //                    ^      2       3       4
+
+        let mapping = builder.mapping();
+
+        assert_eq!(mapping[&0], 0..3); // NYC
+        assert_eq!(mapping[&1], 3..4); // subway
+        assert_eq!(mapping[&2], 0..1); // new
+        assert_eq!(mapping[&3], 1..2); // york
+        assert_eq!(mapping[&4], 2..3); // city
+    }
+
+    #[test]
+    fn middle_query_growing() {
+        let query = ["great", "awesome", "NYC", "subway"];
+        //              0         1        2        3
+        let mut builder = QueryWordsMapper::new(&query);
+
+        // NYC = new york city
+        builder.declare(2..3, 4, &["new", "york", "city"]);
+        //                    ^      4       5       6
+
+        let mapping = builder.mapping();
+
+        assert_eq!(mapping[&0], 0..1); // great
+        assert_eq!(mapping[&1], 1..2); // awesome
+        assert_eq!(mapping[&2], 2..5); // NYC
+        assert_eq!(mapping[&3], 5..6); // subway
+        assert_eq!(mapping[&4], 2..3); // new
+        assert_eq!(mapping[&5], 3..4); // york
+        assert_eq!(mapping[&6], 4..5); // city
+    }
+
+    #[test]
+    fn end_query_growing() {
+        let query = ["NYC", "subway"];
+        //             0        1
+        let mut builder = QueryWordsMapper::new(&query);
+
+        // NYC = new york city
+        builder.declare(1..2, 2, &["underground", "train"]);
+        //                    ^         2            3
+
+        let mapping = builder.mapping();
+
+        assert_eq!(mapping[&0], 0..1); // NYC
+        assert_eq!(mapping[&1], 1..3); // subway
+        assert_eq!(mapping[&2], 1..2); // underground
+        assert_eq!(mapping[&3], 2..3); // train
+    }
+
+    #[test]
+    fn multiple_growings() {
+        let query = ["great", "awesome", "NYC", "subway"];
+        //              0         1        2        3
+        let mut builder = QueryWordsMapper::new(&query);
+
+        // NYC = new york city
+        builder.declare(2..3, 4, &["new", "york", "city"]);
+        //                    ^      4       5       6
+
+        // subway = underground train
+        builder.declare(3..4, 7, &["underground", "train"]);
+        //                    ^          7           8
+
+        let mapping = builder.mapping();
+
+        assert_eq!(mapping[&0], 0..1); // great
+        assert_eq!(mapping[&1], 1..2); // awesome
+        assert_eq!(mapping[&2], 2..5); // NYC
+        assert_eq!(mapping[&3], 5..7); // subway
+        assert_eq!(mapping[&4], 2..3); // new
+        assert_eq!(mapping[&5], 3..4); // york
+        assert_eq!(mapping[&6], 4..5); // city
+        assert_eq!(mapping[&7], 5..6); // underground
+        assert_eq!(mapping[&8], 6..7); // train
+    }
+
+    #[test]
+    fn multiple_probable_growings() {
+        let query = ["great", "awesome", "NYC", "subway"];
+        //              0         1        2        3
+        let mut builder = QueryWordsMapper::new(&query);
+
+        // NYC = new york city
+        builder.declare(2..3, 4, &["new", "york", "city"]);
+        //                    ^      4       5       6
+
+        // subway = underground train
+        builder.declare(3..4, 7, &["underground", "train"]);
+        //                    ^          7           8
+
+        // great awesome = good
+        builder.declare(0..2, 9, &["good"]);
+        //                    ^       9
+
+        // awesome NYC = NY
+        builder.declare(1..3, 10, &["NY"]);
+        //                    ^^     10
+
+        // NYC subway = metro
+        builder.declare(2..4, 11, &["metro"]);
+        //                    ^^      11
+
+        let mapping = builder.mapping();
+
+        assert_eq!(mapping[&0], 0..1); // great
+        assert_eq!(mapping[&1], 1..2); // awesome
+        assert_eq!(mapping[&2], 2..5); // NYC
+        assert_eq!(mapping[&3], 5..7); // subway
+        assert_eq!(mapping[&4], 2..3); // new
+        assert_eq!(mapping[&5], 3..4); // york
+        assert_eq!(mapping[&6], 4..5); // city
+        assert_eq!(mapping[&7], 5..6); // underground
+        assert_eq!(mapping[&8], 6..7); // train
+        assert_eq!(mapping[&9], 0..2); // good
+        assert_eq!(mapping[&10], 1..5); // NY
+        assert_eq!(mapping[&11], 2..7); // metro
+    }
+}

meilisearch-core/src/ranked_map.rs

@@ -1,14 +1,14 @@
 use std::io::{Read, Write};
 
 use hashbrown::HashMap;
-use meilisearch_schema::SchemaAttr;
+use meilisearch_schema::FieldId;
 use serde::{Deserialize, Serialize};
 
 use crate::{DocumentId, Number};
 
 #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(transparent)]
-pub struct RankedMap(HashMap<(DocumentId, SchemaAttr), Number>);
+pub struct RankedMap(HashMap<(DocumentId, FieldId), Number>);
 
 impl RankedMap {
     pub fn len(&self) -> usize {
@@ -19,16 +19,16 @@ impl RankedMap {
         self.0.is_empty()
     }
 
-    pub fn insert(&mut self, document: DocumentId, attribute: SchemaAttr, number: Number) {
-        self.0.insert((document, attribute), number);
+    pub fn insert(&mut self, document: DocumentId, field: FieldId, number: Number) {
+        self.0.insert((document, field), number);
     }
 
-    pub fn remove(&mut self, document: DocumentId, attribute: SchemaAttr) {
-        self.0.remove(&(document, attribute));
+    pub fn remove(&mut self, document: DocumentId, field: FieldId) {
+        self.0.remove(&(document, field));
     }
 
-    pub fn get(&self, document: DocumentId, attribute: SchemaAttr) -> Option<Number> {
-        self.0.get(&(document, attribute)).cloned()
+    pub fn get(&self, document: DocumentId, field: FieldId) -> Option<Number> {
+        self.0.get(&(document, field)).cloned()
     }
 
     pub fn read_from_bin<R: Read>(reader: R) -> bincode::Result<RankedMap> {

meilisearch-core/src/raw_document.rs

@@ -1,186 +1,51 @@
-use std::fmt;
-use std::sync::Arc;
-
-use meilisearch_schema::SchemaAttr;
+use compact_arena::SmallArena;
 use sdset::SetBuf;
-use slice_group_by::GroupBy;
+use crate::DocIndex;
+use crate::bucket_sort::{SimpleMatch, BareMatch, PostingsListView};
+use crate::reordered_attrs::ReorderedAttrs;
 
-use crate::{DocumentId, Highlight, TmpMatch};
-
-#[derive(Clone)]
-pub struct RawDocument {
-    pub id: DocumentId,
-    pub matches: SharedMatches,
-    pub highlights: Vec<Highlight>,
-    pub fields_counts: SetBuf<(SchemaAttr, u64)>,
+pub struct RawDocument<'a, 'tag> {
+    pub id: crate::DocumentId,
+    pub bare_matches: &'a mut [BareMatch<'tag>],
+    pub processed_matches: Vec<SimpleMatch>,
+    /// The list of minimum `distance` found
+    pub processed_distances: Vec<Option<u8>>,
+    /// Does this document contains a field
+    /// with one word that is exactly matching
+    pub contains_one_word_field: bool,
 }
 
-impl RawDocument {
-    pub fn query_index(&self) -> &[u32] {
-        let r = self.matches.range;
-        // it is safe because construction/modifications
-        // can only be done in this module
-        unsafe {
-            &self
-                .matches
-                .matches
-                .query_index
-                .get_unchecked(r.start..r.end)
-        }
-    }
+impl<'a, 'tag> RawDocument<'a, 'tag> {
+    pub fn new<'txn>(
+        bare_matches: &'a mut [BareMatch<'tag>],
+        postings_lists: &mut SmallArena<'tag, PostingsListView<'txn>>,
+        searchable_attrs: Option<&ReorderedAttrs>,
+    ) -> RawDocument<'a, 'tag>
+    {
+        if let Some(reordered_attrs) = searchable_attrs {
+            for bm in bare_matches.iter() {
+                let postings_list = &postings_lists[bm.postings_list];
 
-    pub fn distance(&self) -> &[u8] {
-        let r = self.matches.range;
-        // it is safe because construction/modifications
-        // can only be done in this module
-        unsafe { &self.matches.matches.distance.get_unchecked(r.start..r.end) }
-    }
+                let mut rewritten = Vec::new();
+                for di in postings_list.iter() {
+                    if let Some(attribute) = reordered_attrs.get(di.attribute) {
+                        rewritten.push(DocIndex { attribute, ..*di });
+                    }
+                }
 
-    pub fn attribute(&self) -> &[u16] {
-        let r = self.matches.range;
-        // it is safe because construction/modifications
-        // can only be done in this module
-        unsafe { &self.matches.matches.attribute.get_unchecked(r.start..r.end) }
-    }
-
-    pub fn word_index(&self) -> &[u16] {
-        let r = self.matches.range;
-        // it is safe because construction/modifications
-        // can only be done in this module
-        unsafe {
-            &self
-                .matches
-                .matches
-                .word_index
-                .get_unchecked(r.start..r.end)
-        }
-    }
-
-    pub fn is_exact(&self) -> &[bool] {
-        let r = self.matches.range;
-        // it is safe because construction/modifications
-        // can only be done in this module
-        unsafe { &self.matches.matches.is_exact.get_unchecked(r.start..r.end) }
-    }
-}
-
-impl fmt::Debug for RawDocument {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        f.write_str("RawDocument {\r\n")?;
-        f.write_fmt(format_args!("{:>15}: {:?},\r\n", "id", self.id))?;
-        f.write_fmt(format_args!(
-            "{:>15}: {:^5?},\r\n",
-            "query_index",
-            self.query_index()
-        ))?;
-        f.write_fmt(format_args!(
-            "{:>15}: {:^5?},\r\n",
-            "distance",
-            self.distance()
-        ))?;
-        f.write_fmt(format_args!(
-            "{:>15}: {:^5?},\r\n",
-            "attribute",
-            self.attribute()
-        ))?;
-        f.write_fmt(format_args!(
-            "{:>15}: {:^5?},\r\n",
-            "word_index",
-            self.word_index()
-        ))?;
-        f.write_fmt(format_args!(
-            "{:>15}: {:^5?},\r\n",
-            "is_exact",
-            self.is_exact()
-        ))?;
-        f.write_str("}")?;
-        Ok(())
-    }
-}
-
-pub fn raw_documents_from(
-    matches: SetBuf<(DocumentId, TmpMatch)>,
-    highlights: SetBuf<(DocumentId, Highlight)>,
-    fields_counts: SetBuf<(DocumentId, SchemaAttr, u64)>,
-) -> Vec<RawDocument> {
-    let mut docs_ranges: Vec<(_, Range, _, _)> = Vec::new();
-    let mut matches2 = Matches::with_capacity(matches.len());
-
-    let matches = matches.linear_group_by_key(|(id, _)| *id);
-    let highlights = highlights.linear_group_by_key(|(id, _)| *id);
-    let fields_counts = fields_counts.linear_group_by_key(|(id, _, _)| *id);
-
-    for ((mgroup, hgroup), fgroup) in matches.zip(highlights).zip(fields_counts) {
-        debug_assert_eq!(mgroup[0].0, hgroup[0].0);
-        debug_assert_eq!(mgroup[0].0, fgroup[0].0);
-
-        let document_id = mgroup[0].0;
-        let start = docs_ranges.last().map(|(_, r, _, _)| r.end).unwrap_or(0);
-        let end = start + mgroup.len();
-        let highlights = hgroup.iter().map(|(_, h)| *h).collect();
-        let fields_counts = SetBuf::new(fgroup.iter().map(|(_, a, c)| (*a, *c)).collect()).unwrap();
-
-        docs_ranges.push((document_id, Range { start, end }, highlights, fields_counts));
-        matches2.extend_from_slice(mgroup);
-    }
-
-    let matches = Arc::new(matches2);
-    docs_ranges
-        .into_iter()
-        .map(|(id, range, highlights, fields_counts)| {
-            let matches = SharedMatches {
-                range,
-                matches: matches.clone(),
-            };
-            RawDocument {
-                id,
-                matches,
-                highlights,
-                fields_counts,
+                let new_postings = SetBuf::from_dirty(rewritten);
+                postings_lists[bm.postings_list].rewrite_with(new_postings);
             }
-        })
-        .collect()
-}
-
-#[derive(Debug, Copy, Clone)]
-struct Range {
-    start: usize,
-    end: usize,
-}
-
-#[derive(Clone)]
-pub struct SharedMatches {
-    range: Range,
-    matches: Arc<Matches>,
-}
-
-#[derive(Clone)]
-struct Matches {
-    query_index: Vec<u32>,
-    distance: Vec<u8>,
-    attribute: Vec<u16>,
-    word_index: Vec<u16>,
-    is_exact: Vec<bool>,
-}
-
-impl Matches {
-    fn with_capacity(cap: usize) -> Matches {
-        Matches {
-            query_index: Vec::with_capacity(cap),
-            distance: Vec::with_capacity(cap),
-            attribute: Vec::with_capacity(cap),
-            word_index: Vec::with_capacity(cap),
-            is_exact: Vec::with_capacity(cap),
         }
-    }
 
-    fn extend_from_slice(&mut self, matches: &[(DocumentId, TmpMatch)]) {
-        for (_, match_) in matches {
-            self.query_index.push(match_.query_index);
-            self.distance.push(match_.distance);
-            self.attribute.push(match_.attribute);
-            self.word_index.push(match_.word_index);
-            self.is_exact.push(match_.is_exact);
+        bare_matches.sort_unstable_by_key(|m| m.query_index);
+
+        RawDocument {
+            id: bare_matches[0].document_id,
+            bare_matches,
+            processed_matches: Vec::new(),
+            processed_distances: Vec::new(),
+            contains_one_word_field: false,
         }
     }
 }

meilisearch-core/src/raw_indexer.rs

@@ -3,7 +3,7 @@ use std::convert::TryFrom;
 
 use crate::{DocIndex, DocumentId};
 use deunicode::deunicode_with_tofu;
-use meilisearch_schema::SchemaAttr;
+use meilisearch_schema::IndexedPos;
 use meilisearch_tokenizer::{is_cjk, SeqTokenizer, Token, Tokenizer};
 use sdset::SetBuf;
 
@@ -37,14 +37,14 @@ impl RawIndexer {
         }
     }
 
-    pub fn index_text(&mut self, id: DocumentId, attr: SchemaAttr, text: &str) -> usize {
+    pub fn index_text(&mut self, id: DocumentId, indexed_pos: IndexedPos, text: &str) -> usize {
         let mut number_of_words = 0;
 
         for token in Tokenizer::new(text) {
             let must_continue = index_token(
                 token,
                 id,
-                attr,
+                indexed_pos,
                 self.word_limit,
                 &self.stop_words,
                 &mut self.words_doc_indexes,
@@ -61,7 +61,7 @@ impl RawIndexer {
         number_of_words
     }
 
-    pub fn index_text_seq<'a, I>(&mut self, id: DocumentId, attr: SchemaAttr, iter: I)
+    pub fn index_text_seq<'a, I>(&mut self, id: DocumentId, indexed_pos: IndexedPos, iter: I)
     where
         I: IntoIterator<Item = &'a str>,
     {
@@ -70,7 +70,7 @@ impl RawIndexer {
             let must_continue = index_token(
                 token,
                 id,
-                attr,
+                indexed_pos,
                 self.word_limit,
                 &self.stop_words,
                 &mut self.words_doc_indexes,
@@ -110,7 +110,7 @@ impl RawIndexer {
 fn index_token(
     token: Token,
     id: DocumentId,
-    attr: SchemaAttr,
+    indexed_pos: IndexedPos,
     word_limit: usize,
     stop_words: &fst::Set,
     words_doc_indexes: &mut BTreeMap<Word, Vec<DocIndex>>,
@@ -127,7 +127,7 @@ fn index_token(
     };
 
     if !stop_words.contains(&token.word) {
-        match token_to_docindex(id, attr, token) {
+        match token_to_docindex(id, indexed_pos, token) {
             Some(docindex) => {
                 let word = Vec::from(token.word);
 
@@ -160,14 +160,14 @@ fn index_token(
     true
 }
 
-fn token_to_docindex(id: DocumentId, attr: SchemaAttr, token: Token) -> Option<DocIndex> {
+fn token_to_docindex(id: DocumentId, indexed_pos: IndexedPos, token: Token) -> Option<DocIndex> {
     let word_index = u16::try_from(token.word_index).ok()?;
     let char_index = u16::try_from(token.char_index).ok()?;
     let char_length = u16::try_from(token.word.chars().count()).ok()?;
 
     let docindex = DocIndex {
         document_id: id,
-        attribute: attr.0,
+        attribute: indexed_pos.0,
         word_index,
         char_index,
         char_length,
@@ -179,15 +179,16 @@ fn token_to_docindex(id: DocumentId, attr: SchemaAttr, token: Token) -> Option<D
 #[cfg(test)]
 mod tests {
     use super::*;
+    use meilisearch_schema::IndexedPos;
 
     #[test]
     fn strange_apostrophe() {
         let mut indexer = RawIndexer::new(fst::Set::default());
 
         let docid = DocumentId(0);
-        let attr = SchemaAttr(0);
+        let indexed_pos = IndexedPos(0);
         let text = "Zut, l’aspirateur, j’ai oublié de l’éteindre !";
-        indexer.index_text(docid, attr, text);
+        indexer.index_text(docid, indexed_pos, text);
 
         let Indexed {
             words_doc_indexes, ..
@@ -207,9 +208,9 @@ mod tests {
         let mut indexer = RawIndexer::new(fst::Set::default());
 
         let docid = DocumentId(0);
-        let attr = SchemaAttr(0);
+        let indexed_pos = IndexedPos(0);
         let text = vec!["Zut, l’aspirateur, j’ai oublié de l’éteindre !"];
-        indexer.index_text_seq(docid, attr, text);
+        indexer.index_text_seq(docid, indexed_pos, text);
 
         let Indexed {
             words_doc_indexes, ..
@@ -232,9 +233,9 @@ mod tests {
         let mut indexer = RawIndexer::new(stop_words);
 
         let docid = DocumentId(0);
-        let attr = SchemaAttr(0);
+        let indexed_pos = IndexedPos(0);
         let text = "Zut, l’aspirateur, j’ai oublié de l’éteindre !";
-        indexer.index_text(docid, attr, text);
+        indexer.index_text(docid, indexed_pos, text);
 
         let Indexed {
             words_doc_indexes, ..
@@ -256,9 +257,9 @@ mod tests {
         let mut indexer = RawIndexer::new(fst::Set::default());
 
         let docid = DocumentId(0);
-        let attr = SchemaAttr(0);
+        let indexed_pos = IndexedPos(0);
         let text = "🇯🇵";
-        indexer.index_text(docid, attr, text);
+        indexer.index_text(docid, indexed_pos, text);
 
         let Indexed {
             words_doc_indexes, ..

meilisearch-core/src/reordered_attrs.rs

@@ -1,27 +1,31 @@
+use std::cmp;
+
 #[derive(Default, Clone)]
 pub struct ReorderedAttrs {
-    count: usize,
     reorders: Vec<Option<u16>>,
+    reverse: Vec<u16>,
 }
 
 impl ReorderedAttrs {
     pub fn new() -> ReorderedAttrs {
-        ReorderedAttrs {
-            count: 0,
-            reorders: Vec::new(),
-        }
+        ReorderedAttrs { reorders: Vec::new(), reverse: Vec::new() }
     }
 
     pub fn insert_attribute(&mut self, attribute: u16) {
-        self.reorders.resize(attribute as usize + 1, None);
-        self.reorders[attribute as usize] = Some(self.count as u16);
-        self.count += 1;
+        let new_len = cmp::max(attribute as usize + 1, self.reorders.len());
+        self.reorders.resize(new_len, None);
+        self.reorders[attribute as usize] = Some(self.reverse.len() as u16);
+        self.reverse.push(attribute);
     }
 
     pub fn get(&self, attribute: u16) -> Option<u16> {
-        match self.reorders.get(attribute as usize) {
-            Some(Some(attribute)) => Some(*attribute),
-            _ => None,
+        match self.reorders.get(attribute as usize)? {
+            Some(attribute) => Some(*attribute),
+            None => None,
         }
     }
+
+    pub fn reverse(&self, attribute: u16) -> Option<u16> {
+        self.reverse.get(attribute as usize).copied()
+    }
 }

meilisearch-core/src/serde/convert_to_string.rs

@@ -8,7 +8,7 @@ pub struct ConvertToString;
 impl ser::Serializer for ConvertToString {
     type Ok = String;
     type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
+    type SerializeSeq = SeqConvertToString;
     type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
     type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
     type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
@@ -16,10 +16,8 @@ impl ser::Serializer for ConvertToString {
     type SerializeStruct = StructConvertToString;
     type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
 
-    fn serialize_bool(self, _value: bool) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "boolean",
-        })
+    fn serialize_bool(self, value: bool) -> Result<Self::Ok, Self::Error> {
+        Ok(value.to_string())
     }
 
     fn serialize_char(self, value: char) -> Result<Self::Ok, Self::Error> {
@@ -90,7 +88,7 @@ impl ser::Serializer for ConvertToString {
     }
 
     fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "()" })
+        Ok(String::new())
     }
 
     fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
@@ -137,8 +135,8 @@ impl ser::Serializer for ConvertToString {
     }
 
     fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "sequence",
+        Ok(SeqConvertToString {
+            text: String::new(),
         })
     }
 
@@ -256,3 +254,26 @@ impl ser::SerializeStruct for StructConvertToString {
         Ok(self.text)
     }
 }
+
+pub struct SeqConvertToString {
+    text: String,
+}
+
+impl ser::SerializeSeq for SeqConvertToString {
+    type Ok = String;
+    type Error = SerializerError;
+
+    fn serialize_element<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
+    where
+        T: ser::Serialize,
+    {
+        let text = key.serialize(ConvertToString)?;
+        self.text.push_str(&text);
+        self.text.push_str(" ");
+        Ok(())
+    }
+
+    fn end(self) -> Result<Self::Ok, Self::Error> {
+        Ok(self.text)
+    }
+}

meilisearch-core/src/serde/deserializer.rs

@@ -2,7 +2,7 @@ use std::collections::HashSet;
 use std::io::Cursor;
 use std::{error::Error, fmt};
 
-use meilisearch_schema::{Schema, SchemaAttr};
+use meilisearch_schema::{Schema, FieldId};
 use serde::{de, forward_to_deserialize_any};
 use serde_json::de::IoRead as SerdeJsonIoRead;
 use serde_json::Deserializer as SerdeJsonDeserializer;
@@ -54,7 +54,7 @@ pub struct Deserializer<'a> {
     pub reader: &'a heed::RoTxn<MainT>,
     pub documents_fields: DocumentsFields,
     pub schema: &'a Schema,
-    pub attributes: Option<&'a HashSet<SchemaAttr>>,
+    pub fields: Option<&'a HashSet<FieldId>>,
 }
 
 impl<'de, 'a, 'b> de::Deserializer<'de> for &'b mut Deserializer<'a> {
@@ -92,15 +92,17 @@ impl<'de, 'a, 'b> de::Deserializer<'de> for &'b mut Deserializer<'a> {
                     }
                 };
 
-                let is_displayed = self.schema.props(attr).is_displayed();
-                if is_displayed && self.attributes.map_or(true, |f| f.contains(&attr)) {
-                    let attribute_name = self.schema.attribute_name(attr);
+                let is_displayed = self.schema.is_displayed(attr);
+                if is_displayed && self.fields.map_or(true, |f| f.contains(&attr)) {
+                    if let Some(attribute_name) = self.schema.name(attr) {
+                        let cursor = Cursor::new(value.to_owned());
+                        let ioread = SerdeJsonIoRead::new(cursor);
+                        let value = Value(SerdeJsonDeserializer::new(ioread));
 
-                    let cursor = Cursor::new(value.to_owned());
-                    let ioread = SerdeJsonIoRead::new(cursor);
-                    let value = Value(SerdeJsonDeserializer::new(ioread));
-
-                    Some((attribute_name, value))
+                        Some((attribute_name, value))
+                    } else {
+                        None
+                    }
                 } else {
                     None
                 }

meilisearch-core/src/serde/extract_document_id.rs

@@ -2,28 +2,43 @@ use std::hash::{Hash, Hasher};
 
 use crate::DocumentId;
 use serde::{ser, Serialize};
-use serde_json::Value;
+use serde_json::{Value, Number};
 use siphasher::sip::SipHasher;
 
 use super::{ConvertToString, SerializerError};
 
 pub fn extract_document_id<D>(
-    identifier: &str,
+    primary_key: &str,
     document: &D,
 ) -> Result<Option<DocumentId>, SerializerError>
 where
     D: serde::Serialize,
 {
-    let serializer = ExtractDocumentId { identifier };
+    let serializer = ExtractDocumentId { primary_key };
     document.serialize(serializer)
 }
 
+fn validate_number(value: &Number) -> Option<String> {
+    if value.is_f64() {
+        return None
+    }
+    Some(value.to_string())
+}
+
+fn validate_string(value: &str) -> Option<String> {
+    if value.chars().all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') {
+        Some(value.to_string())
+    } else {
+        None
+    }
+}
+
 pub fn value_to_string(value: &Value) -> Option<String> {
     match value {
         Value::Null => None,
         Value::Bool(_) => None,
-        Value::Number(value) => Some(value.to_string()),
-        Value::String(value) => Some(value.to_string()),
+        Value::Number(value) => validate_number(value),
+        Value::String(value) => validate_string(value),
         Value::Array(_) => None,
         Value::Object(_) => None,
     }
@@ -37,7 +52,7 @@ pub fn compute_document_id<H: Hash>(t: H) -> DocumentId {
 }
 
 struct ExtractDocumentId<'a> {
-    identifier: &'a str,
+    primary_key: &'a str,
 }
 
 impl<'a> ser::Serializer for ExtractDocumentId<'a> {
@@ -173,7 +188,7 @@ impl<'a> ser::Serializer for ExtractDocumentId<'a> {
 
     fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
         let serializer = ExtractDocumentIdMapSerializer {
-            identifier: self.identifier,
+            primary_key: self.primary_key,
             document_id: None,
             current_key_name: None,
         };
@@ -187,7 +202,7 @@ impl<'a> ser::Serializer for ExtractDocumentId<'a> {
         _len: usize,
     ) -> Result<Self::SerializeStruct, Self::Error> {
         let serializer = ExtractDocumentIdStructSerializer {
-            identifier: self.identifier,
+            primary_key: self.primary_key,
             document_id: None,
         };
 
@@ -208,7 +223,7 @@ impl<'a> ser::Serializer for ExtractDocumentId<'a> {
 }
 
 pub struct ExtractDocumentIdMapSerializer<'a> {
-    identifier: &'a str,
+    primary_key: &'a str,
     document_id: Option<DocumentId>,
     current_key_name: Option<String>,
 }
@@ -245,7 +260,7 @@ impl<'a> ser::SerializeMap for ExtractDocumentIdMapSerializer<'a> {
     {
         let key = key.serialize(ConvertToString)?;
 
-        if self.identifier == key {
+        if self.primary_key == key {
             let value = serde_json::to_string(value).and_then(|s| serde_json::from_str(&s))?;
             match value_to_string(&value).map(|s| compute_document_id(&s)) {
                 Some(document_id) => self.document_id = Some(document_id),
@@ -262,7 +277,7 @@ impl<'a> ser::SerializeMap for ExtractDocumentIdMapSerializer<'a> {
 }
 
 pub struct ExtractDocumentIdStructSerializer<'a> {
-    identifier: &'a str,
+    primary_key: &'a str,
     document_id: Option<DocumentId>,
 }
 
@@ -278,7 +293,7 @@ impl<'a> ser::SerializeStruct for ExtractDocumentIdStructSerializer<'a> {
     where
         T: Serialize,
     {
-        if self.identifier == key {
+        if self.primary_key == key {
             let value = serde_json::to_string(value).and_then(|s| serde_json::from_str(&s))?;
             match value_to_string(&value).map(compute_document_id) {
                 Some(document_id) => self.document_id = Some(document_id),

meilisearch-core/src/serde/indexer.rs

@@ -1,4 +1,4 @@
-use meilisearch_schema::SchemaAttr;
+use meilisearch_schema::IndexedPos;
 use serde::ser;
 use serde::Serialize;
 
@@ -7,7 +7,7 @@ use crate::raw_indexer::RawIndexer;
 use crate::DocumentId;
 
 pub struct Indexer<'a> {
-    pub attribute: SchemaAttr,
+    pub pos: IndexedPos,
     pub indexer: &'a mut RawIndexer,
     pub document_id: DocumentId,
 }
@@ -24,9 +24,7 @@ impl<'a> ser::Serializer for Indexer<'a> {
     type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
 
     fn serialize_bool(self, _value: bool) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "boolean",
-        })
+        Ok(None)
     }
 
     fn serialize_char(self, value: char) -> Result<Self::Ok, Self::Error> {
@@ -87,7 +85,7 @@ impl<'a> ser::Serializer for Indexer<'a> {
     fn serialize_str(self, text: &str) -> Result<Self::Ok, Self::Error> {
         let number_of_words = self
             .indexer
-            .index_text(self.document_id, self.attribute, text);
+            .index_text(self.document_id, self.pos, text);
         Ok(Some(number_of_words))
     }
 
@@ -96,9 +94,7 @@ impl<'a> ser::Serializer for Indexer<'a> {
     }
 
     fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "Option",
-        })
+        Ok(None)
     }
 
     fn serialize_some<T: ?Sized>(self, value: &T) -> Result<Self::Ok, Self::Error>
@@ -108,18 +104,16 @@ impl<'a> ser::Serializer for Indexer<'a> {
         let text = value.serialize(ConvertToString)?;
         let number_of_words = self
             .indexer
-            .index_text(self.document_id, self.attribute, &text);
+            .index_text(self.document_id, self.pos, &text);
         Ok(Some(number_of_words))
     }
 
     fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType { type_name: "()" })
+        Ok(None)
     }
 
     fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "unit struct",
-        })
+        Ok(None)
     }
 
     fn serialize_unit_variant(
@@ -128,9 +122,7 @@ impl<'a> ser::Serializer for Indexer<'a> {
         _variant_index: u32,
         _variant: &'static str,
     ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "unit variant",
-        })
+        Ok(None)
     }
 
     fn serialize_newtype_struct<T: ?Sized>(
@@ -161,7 +153,7 @@ impl<'a> ser::Serializer for Indexer<'a> {
 
     fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
         let indexer = SeqIndexer {
-            attribute: self.attribute,
+            pos: self.pos,
             document_id: self.document_id,
             indexer: self.indexer,
             texts: Vec::new(),
@@ -172,7 +164,7 @@ impl<'a> ser::Serializer for Indexer<'a> {
 
     fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
         let indexer = TupleIndexer {
-            attribute: self.attribute,
+            pos: self.pos,
             document_id: self.document_id,
             indexer: self.indexer,
             texts: Vec::new(),
@@ -205,7 +197,7 @@ impl<'a> ser::Serializer for Indexer<'a> {
 
     fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
         let indexer = MapIndexer {
-            attribute: self.attribute,
+            pos: self.pos,
             document_id: self.document_id,
             indexer: self.indexer,
             texts: Vec::new(),
@@ -219,9 +211,14 @@ impl<'a> ser::Serializer for Indexer<'a> {
         _name: &'static str,
         _len: usize,
     ) -> Result<Self::SerializeStruct, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "struct",
-        })
+        let indexer = StructIndexer {
+            pos: self.pos,
+            document_id: self.document_id,
+            indexer: self.indexer,
+            texts: Vec::new(),
+        };
+
+        Ok(indexer)
     }
 
     fn serialize_struct_variant(
@@ -238,7 +235,7 @@ impl<'a> ser::Serializer for Indexer<'a> {
 }
 
 pub struct SeqIndexer<'a> {
-    attribute: SchemaAttr,
+    pos: IndexedPos,
     document_id: DocumentId,
     indexer: &'a mut RawIndexer,
     texts: Vec<String>,
@@ -260,13 +257,13 @@ impl<'a> ser::SerializeSeq for SeqIndexer<'a> {
     fn end(self) -> Result<Self::Ok, Self::Error> {
         let texts = self.texts.iter().map(String::as_str);
         self.indexer
-            .index_text_seq(self.document_id, self.attribute, texts);
+            .index_text_seq(self.document_id, self.pos, texts);
         Ok(None)
     }
 }
 
 pub struct MapIndexer<'a> {
-    attribute: SchemaAttr,
+    pos: IndexedPos,
     document_id: DocumentId,
     indexer: &'a mut RawIndexer,
     texts: Vec<String>,
@@ -297,13 +294,13 @@ impl<'a> ser::SerializeMap for MapIndexer<'a> {
     fn end(self) -> Result<Self::Ok, Self::Error> {
         let texts = self.texts.iter().map(String::as_str);
         self.indexer
-            .index_text_seq(self.document_id, self.attribute, texts);
+            .index_text_seq(self.document_id, self.pos, texts);
         Ok(None)
     }
 }
 
 pub struct StructIndexer<'a> {
-    attribute: SchemaAttr,
+    pos: IndexedPos,
     document_id: DocumentId,
     indexer: &'a mut RawIndexer,
     texts: Vec<String>,
@@ -331,13 +328,13 @@ impl<'a> ser::SerializeStruct for StructIndexer<'a> {
     fn end(self) -> Result<Self::Ok, Self::Error> {
         let texts = self.texts.iter().map(String::as_str);
         self.indexer
-            .index_text_seq(self.document_id, self.attribute, texts);
+            .index_text_seq(self.document_id, self.pos, texts);
         Ok(None)
     }
 }
 
 pub struct TupleIndexer<'a> {
-    attribute: SchemaAttr,
+    pos: IndexedPos,
     document_id: DocumentId,
     indexer: &'a mut RawIndexer,
     texts: Vec<String>,
@@ -359,7 +356,7 @@ impl<'a> ser::SerializeTuple for TupleIndexer<'a> {
     fn end(self) -> Result<Self::Ok, Self::Error> {
         let texts = self.texts.iter().map(String::as_str);
         self.indexer
-            .index_text_seq(self.document_id, self.attribute, texts);
+            .index_text_seq(self.document_id, self.pos, texts);
         Ok(None)
     }
 }

meilisearch-core/src/serde/mod.rs

@@ -20,12 +20,13 @@ pub use self::convert_to_string::ConvertToString;
 pub use self::deserializer::{Deserializer, DeserializerError};
 pub use self::extract_document_id::{compute_document_id, extract_document_id, value_to_string};
 pub use self::indexer::Indexer;
-pub use self::serializer::{serialize_value, Serializer};
+pub use self::serializer::{serialize_value, serialize_value_with_id, Serializer};
 
 use std::{error::Error, fmt};
 
 use serde::ser;
 use serde_json::Error as SerdeJsonError;
+use meilisearch_schema::Error as SchemaError;
 
 use crate::ParseNumberError;
 
@@ -36,6 +37,7 @@ pub enum SerializerError {
     Zlmdb(heed::Error),
     SerdeJson(SerdeJsonError),
     ParseNumber(ParseNumberError),
+    Schema(SchemaError),
     UnserializableType { type_name: &'static str },
     UnindexableType { type_name: &'static str },
     UnrankableType { type_name: &'static str },
@@ -55,13 +57,14 @@ impl fmt::Display for SerializerError {
                 f.write_str("serialized document does not have an id according to the schema")
             }
             SerializerError::InvalidDocumentIdType => {
-                f.write_str("document identifier can only be of type string or number")
+                f.write_str("a document primary key can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_).")
             }
             SerializerError::Zlmdb(e) => write!(f, "heed related error: {}", e),
             SerializerError::SerdeJson(e) => write!(f, "serde json error: {}", e),
             SerializerError::ParseNumber(e) => {
                 write!(f, "error while trying to parse a number: {}", e)
             }
+            SerializerError::Schema(e) => write!(f, "impossible to update schema: {}", e),
             SerializerError::UnserializableType { type_name } => {
                 write!(f, "{} is not a serializable type", type_name)
             }
@@ -101,3 +104,9 @@ impl From<ParseNumberError> for SerializerError {
         SerializerError::ParseNumber(error)
     }
 }
+
+impl From<SchemaError> for SerializerError {
+    fn from(error: SchemaError) -> SerializerError {
+        SerializerError::Schema(error)
+   }
+}

meilisearch-core/src/serde/serializer.rs

@@ -1,4 +1,4 @@
-use meilisearch_schema::{Schema, SchemaAttr, SchemaProps};
+use meilisearch_schema::{Schema, FieldId};
 use serde::ser;
 
 use crate::database::MainT;
@@ -10,7 +10,7 @@ use super::{ConvertToNumber, ConvertToString, Indexer, SerializerError};
 
 pub struct Serializer<'a, 'b> {
     pub txn: &'a mut heed::RwTxn<'b, MainT>,
-    pub schema: &'a Schema,
+    pub schema: &'a mut Schema,
     pub document_store: DocumentsFields,
     pub document_fields_counts: DocumentsFieldsCounts,
     pub indexer: &'a mut RawIndexer,
@@ -193,7 +193,7 @@ impl<'a, 'b> ser::Serializer for Serializer<'a, 'b> {
 
 pub struct MapSerializer<'a, 'b> {
     txn: &'a mut heed::RwTxn<'b, MainT>,
-    schema: &'a Schema,
+    schema: &'a mut Schema,
     document_id: DocumentId,
     document_store: DocumentsFields,
     document_fields_counts: DocumentsFieldsCounts,
@@ -233,20 +233,17 @@ impl<'a, 'b> ser::SerializeMap for MapSerializer<'a, 'b> {
         V: ser::Serialize,
     {
         let key = key.serialize(ConvertToString)?;
-        match self.schema.attribute(&key) {
-            Some(attribute) => serialize_value(
-                self.txn,
-                attribute,
-                self.schema.props(attribute),
-                self.document_id,
-                self.document_store,
-                self.document_fields_counts,
-                self.indexer,
-                self.ranked_map,
-                value,
-            ),
-            None => Ok(()),
-        }
+        serialize_value(
+            self.txn,
+            key.as_str(),
+            self.schema,
+            self.document_id,
+            self.document_store,
+            self.document_fields_counts,
+            self.indexer,
+            self.ranked_map,
+            value,
+        )
     }
 
     fn end(self) -> Result<Self::Ok, Self::Error> {
@@ -256,7 +253,7 @@ impl<'a, 'b> ser::SerializeMap for MapSerializer<'a, 'b> {
 
 pub struct StructSerializer<'a, 'b> {
     txn: &'a mut heed::RwTxn<'b, MainT>,
-    schema: &'a Schema,
+    schema: &'a mut Schema,
     document_id: DocumentId,
     document_store: DocumentsFields,
     document_fields_counts: DocumentsFieldsCounts,
@@ -276,20 +273,17 @@ impl<'a, 'b> ser::SerializeStruct for StructSerializer<'a, 'b> {
     where
         T: ser::Serialize,
     {
-        match self.schema.attribute(key) {
-            Some(attribute) => serialize_value(
-                self.txn,
-                attribute,
-                self.schema.props(attribute),
-                self.document_id,
-                self.document_store,
-                self.document_fields_counts,
-                self.indexer,
-                self.ranked_map,
-                value,
-            ),
-            None => Ok(()),
-        }
+        serialize_value(
+            self.txn,
+            key,
+            self.schema,
+            self.document_id,
+            self.document_store,
+            self.document_fields_counts,
+            self.indexer,
+            self.ranked_map,
+            value,
+        )
     }
 
     fn end(self) -> Result<Self::Ok, Self::Error> {
@@ -297,10 +291,38 @@ impl<'a, 'b> ser::SerializeStruct for StructSerializer<'a, 'b> {
     }
 }
 
-pub fn serialize_value<T: ?Sized>(
+pub fn serialize_value<'a, T: ?Sized>(
     txn: &mut heed::RwTxn<MainT>,
-    attribute: SchemaAttr,
-    props: SchemaProps,
+    attribute: &str,
+    schema: &'a mut Schema,
+    document_id: DocumentId,
+    document_store: DocumentsFields,
+    documents_fields_counts: DocumentsFieldsCounts,
+    indexer: &mut RawIndexer,
+    ranked_map: &mut RankedMap,
+    value: &T,
+) -> Result<(), SerializerError>
+where
+    T: ser::Serialize,
+{
+    let field_id = schema.insert_and_index(&attribute)?;
+    serialize_value_with_id(
+        txn,
+        field_id,
+        schema,
+        document_id,
+        document_store,
+        documents_fields_counts,
+        indexer,
+        ranked_map,
+        value,
+    )
+}
+
+pub fn serialize_value_with_id<'a, T: ?Sized>(
+    txn: &mut heed::RwTxn<MainT>,
+    field_id: FieldId,
+    schema: &'a Schema,
     document_id: DocumentId,
     document_store: DocumentsFields,
     documents_fields_counts: DocumentsFieldsCounts,
@@ -312,11 +334,11 @@ where
     T: ser::Serialize,
 {
     let serialized = serde_json::to_vec(value)?;
-    document_store.put_document_field(txn, document_id, attribute, &serialized)?;
+    document_store.put_document_field(txn, document_id, field_id, &serialized)?;
 
-    if props.is_indexed() {
+    if let Some(indexed_pos) = schema.is_indexed(field_id) {
         let indexer = Indexer {
-            attribute,
+            pos: *indexed_pos,
             indexer,
             document_id,
         };
@@ -324,15 +346,15 @@ where
             documents_fields_counts.put_document_field_count(
                 txn,
                 document_id,
-                attribute,
-                number_of_words as u64,
+                *indexed_pos,
+                number_of_words as u16,
             )?;
         }
     }
 
-    if props.is_ranked() {
-        let number = value.serialize(ConvertToNumber)?;
-        ranked_map.insert(document_id, attribute, number);
+    if schema.is_ranked(field_id) {
+        let number = value.serialize(ConvertToNumber).unwrap_or_default();
+        ranked_map.insert(document_id, field_id, number);
     }
 
     Ok(())

meilisearch-core/src/settings.rs

@@ -0,0 +1,184 @@
+use std::collections::{BTreeMap, BTreeSet, HashSet};
+use std::str::FromStr;
+use std::iter::IntoIterator;
+
+use serde::{Deserialize, Deserializer, Serialize};
+use once_cell::sync::Lazy;
+
+use self::RankingRule::*;
+
+pub const DEFAULT_RANKING_RULES: [RankingRule; 6] = [Typo, Words, Proximity, Attribute, WordsPosition, Exactness];
+
+static RANKING_RULE_REGEX: Lazy<regex::Regex> = Lazy::new(|| {
+    let regex = regex::Regex::new(r"(asc|desc)\(([a-zA-Z0-9-_]*)\)").unwrap();
+    regex
+});
+
+#[derive(Default, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+pub struct Settings {
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub ranking_rules: Option<Option<Vec<String>>>,
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub distinct_attribute: Option<Option<String>>,
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub searchable_attributes: Option<Option<Vec<String>>>,
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub displayed_attributes: Option<Option<HashSet<String>>>,
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub stop_words: Option<Option<BTreeSet<String>>>,
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub synonyms: Option<Option<BTreeMap<String, Vec<String>>>>,
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub accept_new_fields: Option<Option<bool>>,
+}
+
+// Any value that is present is considered Some value, including null.
+fn deserialize_some<'de, T, D>(deserializer: D) -> Result<Option<T>, D::Error>
+    where T: Deserialize<'de>,
+          D: Deserializer<'de>
+{
+    Deserialize::deserialize(deserializer).map(Some)
+}
+
+impl Settings {
+    pub fn into_update(&self) -> Result<SettingsUpdate, RankingRuleConversionError> {
+        let settings = self.clone();
+
+        let ranking_rules = match settings.ranking_rules {
+            Some(Some(rules)) => UpdateState::Update(RankingRule::from_iter(rules.iter())?),
+            Some(None) => UpdateState::Clear,
+            None => UpdateState::Nothing,
+        };
+
+        Ok(SettingsUpdate {
+            ranking_rules,
+            distinct_attribute: settings.distinct_attribute.into(),
+            primary_key: UpdateState::Nothing,
+            searchable_attributes: settings.searchable_attributes.into(),
+            displayed_attributes: settings.displayed_attributes.into(),
+            stop_words: settings.stop_words.into(),
+            synonyms: settings.synonyms.into(),
+            accept_new_fields: settings.accept_new_fields.into(),
+        })
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum UpdateState<T> {
+    Update(T),
+    Clear,
+    Nothing,
+}
+
+impl <T> From<Option<Option<T>>> for UpdateState<T> {
+    fn from(opt: Option<Option<T>>) -> UpdateState<T> {
+        match opt {
+            Some(Some(t)) => UpdateState::Update(t),
+            Some(None) => UpdateState::Clear,
+            None => UpdateState::Nothing,
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct RankingRuleConversionError;
+
+impl std::fmt::Display for RankingRuleConversionError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "impossible to convert into RankingRule")
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum RankingRule {
+    Typo,
+    Words,
+    Proximity,
+    Attribute,
+    WordsPosition,
+    Exactness,
+    Asc(String),
+    Desc(String),
+}
+
+impl std::fmt::Display for RankingRule {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match self {
+            RankingRule::Typo => f.write_str("typo"),
+            RankingRule::Words => f.write_str("words"),
+            RankingRule::Proximity => f.write_str("proximity"),
+            RankingRule::Attribute => f.write_str("attribute"),
+            RankingRule::WordsPosition => f.write_str("wordsPosition"),
+            RankingRule::Exactness => f.write_str("exactness"),
+            RankingRule::Asc(field) => write!(f, "asc({})", field),
+            RankingRule::Desc(field) => write!(f, "desc({})", field),
+        }
+    }
+}
+
+impl FromStr for RankingRule {
+    type Err = RankingRuleConversionError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let rule = match s {
+            "typo" => RankingRule::Typo,
+            "words" => RankingRule::Words,
+            "proximity" => RankingRule::Proximity,
+            "attribute" => RankingRule::Attribute,
+            "wordsPosition" => RankingRule::WordsPosition,
+            "exactness" => RankingRule::Exactness,
+            _ => {
+                let captures = RANKING_RULE_REGEX.captures(s).ok_or(RankingRuleConversionError)?;
+                match (captures.get(1).map(|m| m.as_str()), captures.get(2)) {
+                    (Some("asc"), Some(field)) => RankingRule::Asc(field.as_str().to_string()),
+                    (Some("desc"), Some(field)) => RankingRule::Desc(field.as_str().to_string()),
+                    _ => return Err(RankingRuleConversionError)
+                }
+            }
+        };
+        Ok(rule)
+    }
+}
+
+impl RankingRule {
+    pub fn field(&self) -> Option<&str> {
+        match self {
+            RankingRule::Asc(field) | RankingRule::Desc(field) => Some(field),
+            _ => None,
+        }
+    }
+
+    pub fn from_iter(rules: impl IntoIterator<Item = impl AsRef<str>>) -> Result<Vec<RankingRule>, RankingRuleConversionError> {
+        rules.into_iter()
+            .map(|s| RankingRule::from_str(s.as_ref()))
+            .collect()
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SettingsUpdate {
+    pub ranking_rules: UpdateState<Vec<RankingRule>>,
+    pub distinct_attribute: UpdateState<String>,
+    pub primary_key: UpdateState<String>,
+    pub searchable_attributes: UpdateState<Vec<String>>,
+    pub displayed_attributes: UpdateState<HashSet<String>>,
+    pub stop_words: UpdateState<BTreeSet<String>>,
+    pub synonyms: UpdateState<BTreeMap<String, Vec<String>>>,
+    pub accept_new_fields: UpdateState<bool>,
+}
+
+impl Default for SettingsUpdate {
+    fn default() -> Self {
+        Self {
+            ranking_rules: UpdateState::Nothing,
+            distinct_attribute: UpdateState::Nothing,
+            primary_key: UpdateState::Nothing,
+            searchable_attributes: UpdateState::Nothing,
+            displayed_attributes: UpdateState::Nothing,
+            stop_words: UpdateState::Nothing,
+            synonyms: UpdateState::Nothing,
+            accept_new_fields: UpdateState::Nothing,
+        }
+    }
+}

meilisearch-core/src/store/documents_fields.rs

@@ -1,14 +1,14 @@
 use heed::types::{ByteSlice, OwnedType};
 use crate::database::MainT;
 use heed::Result as ZResult;
-use meilisearch_schema::SchemaAttr;
+use meilisearch_schema::FieldId;
 
-use super::DocumentAttrKey;
+use super::DocumentFieldStoredKey;
 use crate::DocumentId;
 
 #[derive(Copy, Clone)]
 pub struct DocumentsFields {
-    pub(crate) documents_fields: heed::Database<OwnedType<DocumentAttrKey>, ByteSlice>,
+    pub(crate) documents_fields: heed::Database<OwnedType<DocumentFieldStoredKey>, ByteSlice>,
 }
 
 impl DocumentsFields {
@@ -16,10 +16,10 @@ impl DocumentsFields {
         self,
         writer: &mut heed::RwTxn<MainT>,
         document_id: DocumentId,
-        attribute: SchemaAttr,
+        field: FieldId,
         value: &[u8],
     ) -> ZResult<()> {
-        let key = DocumentAttrKey::new(document_id, attribute);
+        let key = DocumentFieldStoredKey::new(document_id, field);
         self.documents_fields.put(writer, &key, value)
     }
 
@@ -28,8 +28,8 @@ impl DocumentsFields {
         writer: &mut heed::RwTxn<MainT>,
         document_id: DocumentId,
     ) -> ZResult<usize> {
-        let start = DocumentAttrKey::new(document_id, SchemaAttr::min());
-        let end = DocumentAttrKey::new(document_id, SchemaAttr::max());
+        let start = DocumentFieldStoredKey::new(document_id, FieldId::min());
+        let end = DocumentFieldStoredKey::new(document_id, FieldId::max());
         self.documents_fields.delete_range(writer, &(start..=end))
     }
 
@@ -41,9 +41,9 @@ impl DocumentsFields {
         self,
         reader: &'txn heed::RoTxn<MainT>,
         document_id: DocumentId,
-        attribute: SchemaAttr,
+        field: FieldId,
     ) -> ZResult<Option<&'txn [u8]>> {
-        let key = DocumentAttrKey::new(document_id, attribute);
+        let key = DocumentFieldStoredKey::new(document_id, field);
         self.documents_fields.get(reader, &key)
     }
 
@@ -52,25 +52,25 @@ impl DocumentsFields {
         reader: &'txn heed::RoTxn<MainT>,
         document_id: DocumentId,
     ) -> ZResult<DocumentFieldsIter<'txn>> {
-        let start = DocumentAttrKey::new(document_id, SchemaAttr::min());
-        let end = DocumentAttrKey::new(document_id, SchemaAttr::max());
+        let start = DocumentFieldStoredKey::new(document_id, FieldId::min());
+        let end = DocumentFieldStoredKey::new(document_id, FieldId::max());
         let iter = self.documents_fields.range(reader, &(start..=end))?;
         Ok(DocumentFieldsIter { iter })
     }
 }
 
 pub struct DocumentFieldsIter<'txn> {
-    iter: heed::RoRange<'txn, OwnedType<DocumentAttrKey>, ByteSlice>,
+    iter: heed::RoRange<'txn, OwnedType<DocumentFieldStoredKey>, ByteSlice>,
 }
 
 impl<'txn> Iterator for DocumentFieldsIter<'txn> {
-    type Item = ZResult<(SchemaAttr, &'txn [u8])>;
+    type Item = ZResult<(FieldId, &'txn [u8])>;
 
     fn next(&mut self) -> Option<Self::Item> {
         match self.iter.next() {
             Some(Ok((key, bytes))) => {
-                let attr = SchemaAttr(key.attr.get());
-                Some(Ok((attr, bytes)))
+                let field_id = FieldId(key.field_id.get());
+                Some(Ok((field_id, bytes)))
             }
             Some(Err(e)) => Some(Err(e)),
             None => None,

meilisearch-core/src/store/documents_fields_counts.rs

@@ -1,13 +1,13 @@
-use super::DocumentAttrKey;
+use super::DocumentFieldIndexedKey;
 use crate::database::MainT;
 use crate::DocumentId;
 use heed::types::OwnedType;
 use heed::Result as ZResult;
-use meilisearch_schema::SchemaAttr;
+use meilisearch_schema::IndexedPos;
 
 #[derive(Copy, Clone)]
 pub struct DocumentsFieldsCounts {
-    pub(crate) documents_fields_counts: heed::Database<OwnedType<DocumentAttrKey>, OwnedType<u64>>,
+    pub(crate) documents_fields_counts: heed::Database<OwnedType<DocumentFieldIndexedKey>, OwnedType<u16>>,
 }
 
 impl DocumentsFieldsCounts {
@@ -15,10 +15,10 @@ impl DocumentsFieldsCounts {
         self,
         writer: &mut heed::RwTxn<MainT>,
         document_id: DocumentId,
-        attribute: SchemaAttr,
-        value: u64,
+        attribute: IndexedPos,
+        value: u16,
     ) -> ZResult<()> {
-        let key = DocumentAttrKey::new(document_id, attribute);
+        let key = DocumentFieldIndexedKey::new(document_id, attribute);
         self.documents_fields_counts.put(writer, &key, &value)
     }
 
@@ -27,10 +27,9 @@ impl DocumentsFieldsCounts {
         writer: &mut heed::RwTxn<MainT>,
         document_id: DocumentId,
     ) -> ZResult<usize> {
-        let start = DocumentAttrKey::new(document_id, SchemaAttr::min());
-        let end = DocumentAttrKey::new(document_id, SchemaAttr::max());
-        self.documents_fields_counts
-            .delete_range(writer, &(start..=end))
+        let start = DocumentFieldIndexedKey::new(document_id, IndexedPos::min());
+        let end = DocumentFieldIndexedKey::new(document_id, IndexedPos::max());
+        self.documents_fields_counts.delete_range(writer, &(start..=end))
     }
 
     pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
@@ -41,9 +40,9 @@ impl DocumentsFieldsCounts {
         self,
         reader: &heed::RoTxn<MainT>,
         document_id: DocumentId,
-        attribute: SchemaAttr,
-    ) -> ZResult<Option<u64>> {
-        let key = DocumentAttrKey::new(document_id, attribute);
+        attribute: IndexedPos,
+    ) -> ZResult<Option<u16>> {
+        let key = DocumentFieldIndexedKey::new(document_id, attribute);
         match self.documents_fields_counts.get(reader, &key)? {
             Some(count) => Ok(Some(count)),
             None => Ok(None),
@@ -55,8 +54,8 @@ impl DocumentsFieldsCounts {
         reader: &'txn heed::RoTxn<MainT>,
         document_id: DocumentId,
     ) -> ZResult<DocumentFieldsCountsIter<'txn>> {
-        let start = DocumentAttrKey::new(document_id, SchemaAttr::min());
-        let end = DocumentAttrKey::new(document_id, SchemaAttr::max());
+        let start = DocumentFieldIndexedKey::new(document_id, IndexedPos::min());
+        let end = DocumentFieldIndexedKey::new(document_id, IndexedPos::max());
         let iter = self.documents_fields_counts.range(reader, &(start..=end))?;
         Ok(DocumentFieldsCountsIter { iter })
     }
@@ -79,17 +78,17 @@ impl DocumentsFieldsCounts {
 }
 
 pub struct DocumentFieldsCountsIter<'txn> {
-    iter: heed::RoRange<'txn, OwnedType<DocumentAttrKey>, OwnedType<u64>>,
+    iter: heed::RoRange<'txn, OwnedType<DocumentFieldIndexedKey>, OwnedType<u16>>,
 }
 
 impl Iterator for DocumentFieldsCountsIter<'_> {
-    type Item = ZResult<(SchemaAttr, u64)>;
+    type Item = ZResult<(IndexedPos, u16)>;
 
     fn next(&mut self) -> Option<Self::Item> {
         match self.iter.next() {
             Some(Ok((key, count))) => {
-                let attr = SchemaAttr(key.attr.get());
-                Some(Ok((attr, count)))
+                let indexed_pos = IndexedPos(key.indexed_pos.get());
+                Some(Ok((indexed_pos, count)))
             }
             Some(Err(e)) => Some(Err(e)),
             None => None,
@@ -99,7 +98,7 @@ impl Iterator for DocumentFieldsCountsIter<'_> {
 
 pub struct DocumentsIdsIter<'txn> {
     last_seen_id: Option<DocumentId>,
-    iter: heed::RoIter<'txn, OwnedType<DocumentAttrKey>, OwnedType<u64>>,
+    iter: heed::RoIter<'txn, OwnedType<DocumentFieldIndexedKey>, OwnedType<u16>>,
 }
 
 impl Iterator for DocumentsIdsIter<'_> {
@@ -123,18 +122,18 @@ impl Iterator for DocumentsIdsIter<'_> {
 }
 
 pub struct AllDocumentsFieldsCountsIter<'txn> {
-    iter: heed::RoIter<'txn, OwnedType<DocumentAttrKey>, OwnedType<u64>>,
+    iter: heed::RoIter<'txn, OwnedType<DocumentFieldIndexedKey>, OwnedType<u16>>,
 }
 
 impl Iterator for AllDocumentsFieldsCountsIter<'_> {
-    type Item = ZResult<(DocumentId, SchemaAttr, u64)>;
+    type Item = ZResult<(DocumentId, IndexedPos, u16)>;
 
     fn next(&mut self) -> Option<Self::Item> {
         match self.iter.next() {
             Some(Ok((key, count))) => {
                 let docid = DocumentId(key.docid.get());
-                let attr = SchemaAttr(key.attr.get());
-                Some(Ok((docid, attr, count)))
+                let indexed_pos = IndexedPos(key.indexed_pos.get());
+                Some(Ok((docid, indexed_pos, count)))
             }
             Some(Err(e)) => Some(Err(e)),
             None => None,

meilisearch-core/src/store/main.rs

@@ -1,21 +1,26 @@
-use crate::database::MainT;
-use crate::RankedMap;
+use std::sync::Arc;
+use std::collections::HashMap;
+
 use chrono::{DateTime, Utc};
 use heed::types::{ByteSlice, OwnedType, SerdeBincode, Str};
 use heed::Result as ZResult;
 use meilisearch_schema::Schema;
-use std::collections::HashMap;
-use std::sync::Arc;
+
+use crate::database::MainT;
+use crate::RankedMap;
+use crate::settings::RankingRule;
 
 const CREATED_AT_KEY: &str = "created-at";
-const CUSTOMS_KEY: &str = "customs-key";
+const RANKING_RULES_KEY: &str = "ranking-rules";
+const DISTINCT_ATTRIBUTE_KEY: &str = "distinct-attribute";
+const STOP_WORDS_KEY: &str = "stop-words";
+const SYNONYMS_KEY: &str = "synonyms";
+const CUSTOMS_KEY: &str = "customs";
 const FIELDS_FREQUENCY_KEY: &str = "fields-frequency";
 const NAME_KEY: &str = "name";
 const NUMBER_OF_DOCUMENTS_KEY: &str = "number-of-documents";
 const RANKED_MAP_KEY: &str = "ranked-map";
 const SCHEMA_KEY: &str = "schema";
-const STOP_WORDS_KEY: &str = "stop-words";
-const SYNONYMS_KEY: &str = "synonyms";
 const UPDATED_AT_KEY: &str = "updated-at";
 const WORDS_KEY: &str = "words";
 
@@ -67,6 +72,17 @@ impl Main {
         self.main.put::<_, Str, ByteSlice>(writer, WORDS_KEY, bytes)
     }
 
+    pub unsafe fn static_words_fst(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<fst::Set>> {
+        match self.main.get::<_, Str, ByteSlice>(reader, WORDS_KEY)? {
+            Some(bytes) => {
+                let bytes: &'static [u8] = std::mem::transmute(bytes);
+                let set = fst::Set::from_static_slice(bytes).unwrap();
+                Ok(Some(set))
+            }
+            None => Ok(None),
+        }
+    }
+
     pub fn words_fst(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<fst::Set>> {
         match self.main.get::<_, Str, ByteSlice>(reader, WORDS_KEY)? {
             Some(bytes) => {
@@ -80,23 +96,23 @@ impl Main {
     }
 
     pub fn put_schema(self, writer: &mut heed::RwTxn<MainT>, schema: &Schema) -> ZResult<()> {
-        self.main
-            .put::<_, Str, SerdeBincode<Schema>>(writer, SCHEMA_KEY, schema)
+        self.main.put::<_, Str, SerdeBincode<Schema>>(writer, SCHEMA_KEY, schema)
     }
 
     pub fn schema(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<Schema>> {
-        self.main
-            .get::<_, Str, SerdeBincode<Schema>>(reader, SCHEMA_KEY)
+        self.main.get::<_, Str, SerdeBincode<Schema>>(reader, SCHEMA_KEY)
+    }
+
+    pub fn delete_schema(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<bool> {
+        self.main.delete::<_, Str>(writer, SCHEMA_KEY)
     }
 
     pub fn put_ranked_map(self, writer: &mut heed::RwTxn<MainT>, ranked_map: &RankedMap) -> ZResult<()> {
-        self.main
-            .put::<_, Str, SerdeBincode<RankedMap>>(writer, RANKED_MAP_KEY, &ranked_map)
+        self.main.put::<_, Str, SerdeBincode<RankedMap>>(writer, RANKED_MAP_KEY, &ranked_map)
     }
 
     pub fn ranked_map(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<RankedMap>> {
-        self.main
-            .get::<_, Str, SerdeBincode<RankedMap>>(reader, RANKED_MAP_KEY)
+        self.main.get::<_, Str, SerdeBincode<RankedMap>>(reader, RANKED_MAP_KEY)
     }
 
     pub fn put_synonyms_fst(self, writer: &mut heed::RwTxn<MainT>, fst: &fst::Set) -> ZResult<()> {
@@ -118,8 +134,7 @@ impl Main {
 
     pub fn put_stop_words_fst(self, writer: &mut heed::RwTxn<MainT>, fst: &fst::Set) -> ZResult<()> {
         let bytes = fst.as_fst().as_bytes();
-        self.main
-            .put::<_, Str, ByteSlice>(writer, STOP_WORDS_KEY, bytes)
+        self.main.put::<_, Str, ByteSlice>(writer, STOP_WORDS_KEY, bytes)
     }
 
     pub fn stop_words_fst(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<fst::Set>> {
@@ -173,6 +188,33 @@ impl Main {
         }
     }
 
+    pub fn ranking_rules(&self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<Vec<RankingRule>>> {
+        self.main.get::<_, Str, SerdeBincode<Vec<RankingRule>>>(reader, RANKING_RULES_KEY)
+    }
+
+    pub fn put_ranking_rules(self, writer: &mut heed::RwTxn<MainT>, value: &[RankingRule]) -> ZResult<()> {
+        self.main.put::<_, Str, SerdeBincode<Vec<RankingRule>>>(writer, RANKING_RULES_KEY, &value.to_vec())
+    }
+
+    pub fn delete_ranking_rules(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<bool> {
+        self.main.delete::<_, Str>(writer, RANKING_RULES_KEY)
+    }
+
+    pub fn distinct_attribute(&self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<String>> {
+        if let Some(value) = self.main.get::<_, Str, Str>(reader, DISTINCT_ATTRIBUTE_KEY)? {
+            return Ok(Some(value.to_owned()))
+        }
+        return Ok(None)
+    }
+
+    pub fn put_distinct_attribute(self, writer: &mut heed::RwTxn<MainT>, value: &str) -> ZResult<()> {
+        self.main.put::<_, Str, Str>(writer, DISTINCT_ATTRIBUTE_KEY, value)
+    }
+
+    pub fn delete_distinct_attribute(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<bool> {
+        self.main.delete::<_, Str>(writer, DISTINCT_ATTRIBUTE_KEY)
+    }
+
     pub fn put_customs(self, writer: &mut heed::RwTxn<MainT>, customs: &[u8]) -> ZResult<()> {
         self.main
             .put::<_, Str, ByteSlice>(writer, CUSTOMS_KEY, customs)

meilisearch-core/src/store/mod.rs

@@ -1,4 +1,6 @@
 mod docs_words;
+mod prefix_documents_cache;
+mod prefix_postings_lists_cache;
 mod documents_fields;
 mod documents_fields_counts;
 mod main;
@@ -8,6 +10,8 @@ mod updates;
 mod updates_results;
 
 pub use self::docs_words::DocsWords;
+pub use self::prefix_documents_cache::PrefixDocumentsCache;
+pub use self::prefix_postings_lists_cache::PrefixPostingsListsCache;
 pub use self::documents_fields::{DocumentFieldsIter, DocumentsFields};
 pub use self::documents_fields_counts::{
     DocumentFieldsCountsIter, DocumentsFieldsCounts, DocumentsIdsIter,
@@ -18,38 +22,141 @@ pub use self::synonyms::Synonyms;
 pub use self::updates::Updates;
 pub use self::updates_results::UpdatesResults;
 
+use std::borrow::Cow;
 use std::collections::HashSet;
+use std::convert::TryInto;
+use std::{mem, ptr};
 
 use heed::Result as ZResult;
-use meilisearch_schema::{Schema, SchemaAttr};
+use heed::{BytesEncode, BytesDecode};
+use meilisearch_schema::{IndexedPos, FieldId};
+use sdset::{Set, SetBuf};
 use serde::de::{self, Deserialize};
 use zerocopy::{AsBytes, FromBytes};
 
 use crate::criterion::Criteria;
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
 use crate::database::{MainT, UpdateT};
+use crate::database::{UpdateEvent, UpdateEventsEmitter};
 use crate::serde::Deserializer;
-use crate::{query_builder::QueryBuilder, update, DocumentId, Error, MResult};
+use crate::settings::SettingsUpdate;
+use crate::{query_builder::QueryBuilder, update, DocIndex, DocumentId, Error, MResult};
 
 type BEU64 = zerocopy::U64<byteorder::BigEndian>;
 type BEU16 = zerocopy::U16<byteorder::BigEndian>;
 
 #[derive(Debug, Copy, Clone, AsBytes, FromBytes)]
 #[repr(C)]
-pub struct DocumentAttrKey {
+pub struct DocumentFieldIndexedKey {
     docid: BEU64,
-    attr: BEU16,
+    indexed_pos: BEU16,
 }
 
-impl DocumentAttrKey {
-    fn new(docid: DocumentId, attr: SchemaAttr) -> DocumentAttrKey {
-        DocumentAttrKey {
+impl DocumentFieldIndexedKey {
+    fn new(docid: DocumentId, indexed_pos: IndexedPos) -> DocumentFieldIndexedKey {
+        DocumentFieldIndexedKey {
             docid: BEU64::new(docid.0),
-            attr: BEU16::new(attr.0),
+            indexed_pos: BEU16::new(indexed_pos.0),
         }
     }
 }
 
+#[derive(Debug, Copy, Clone, AsBytes, FromBytes)]
+#[repr(C)]
+pub struct DocumentFieldStoredKey {
+    docid: BEU64,
+    field_id: BEU16,
+}
+
+impl DocumentFieldStoredKey {
+    fn new(docid: DocumentId, field_id: FieldId) -> DocumentFieldStoredKey {
+        DocumentFieldStoredKey {
+            docid: BEU64::new(docid.0),
+            field_id: BEU16::new(field_id.0),
+        }
+    }
+}
+
+#[derive(Default, Debug)]
+pub struct Postings<'a> {
+    pub docids: Cow<'a, Set<DocumentId>>,
+    pub matches: Cow<'a, Set<DocIndex>>,
+}
+
+pub struct PostingsCodec;
+
+impl<'a> BytesEncode<'a> for PostingsCodec {
+    type EItem = Postings<'a>;
+
+    fn bytes_encode(item: &'a Self::EItem) -> Option<Cow<'a, [u8]>> {
+        let u64_size = mem::size_of::<u64>();
+        let docids_size = item.docids.len() * mem::size_of::<DocumentId>();
+        let matches_size = item.matches.len() * mem::size_of::<DocIndex>();
+
+        let mut buffer = Vec::with_capacity(u64_size + docids_size + matches_size);
+
+        let docids_len = item.docids.len();
+        buffer.extend_from_slice(&docids_len.to_be_bytes());
+        buffer.extend_from_slice(item.docids.as_bytes());
+        buffer.extend_from_slice(item.matches.as_bytes());
+
+        Some(Cow::Owned(buffer))
+    }
+}
+
+fn aligned_to(bytes: &[u8], align: usize) -> bool {
+    (bytes as *const _ as *const () as usize) % align == 0
+}
+
+fn from_bytes_to_set<'a, T: 'a>(bytes: &'a [u8]) -> Option<Cow<'a, Set<T>>>
+where T: Clone + FromBytes
+{
+    match zerocopy::LayoutVerified::<_, [T]>::new_slice(bytes) {
+        Some(layout) => Some(Cow::Borrowed(Set::new_unchecked(layout.into_slice()))),
+        None => {
+            let len = bytes.len();
+            let elem_size = mem::size_of::<T>();
+
+            // ensure that it is the alignment that is wrong
+            // and the length is valid
+            if len % elem_size == 0 && !aligned_to(bytes, mem::align_of::<T>()) {
+                let elems = len / elem_size;
+                let mut vec = Vec::<T>::with_capacity(elems);
+
+                unsafe {
+                    let dst = vec.as_mut_ptr() as *mut u8;
+                    ptr::copy_nonoverlapping(bytes.as_ptr(), dst, len);
+                    vec.set_len(elems);
+                }
+
+                return Some(Cow::Owned(SetBuf::new_unchecked(vec)));
+            }
+
+            None
+        }
+    }
+}
+
+impl<'a> BytesDecode<'a> for PostingsCodec {
+    type DItem = Postings<'a>;
+
+    fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
+        let u64_size = mem::size_of::<u64>();
+        let docid_size = mem::size_of::<DocumentId>();
+
+        let (len_bytes, bytes) = bytes.split_at(u64_size);
+        let docids_len = len_bytes.try_into().ok().map(u64::from_be_bytes)? as usize;
+        let docids_size = docids_len * docid_size;
+
+        let docids_bytes = &bytes[..docids_size];
+        let matches_bytes = &bytes[docids_size..];
+
+        let docids = from_bytes_to_set(docids_bytes)?;
+        let matches = from_bytes_to_set(matches_bytes)?;
+
+        Some(Postings { docids, matches })
+    }
+}
+
 fn main_name(name: &str) -> String {
     format!("store-{}", name)
 }
@@ -74,6 +181,14 @@ fn docs_words_name(name: &str) -> String {
     format!("store-{}-docs-words", name)
 }
 
+fn prefix_documents_cache_name(name: &str) -> String {
+    format!("store-{}-prefix-documents-cache", name)
+}
+
+fn prefix_postings_lists_cache_name(name: &str) -> String {
+    format!("store-{}-prefix-postings-lists-cache", name)
+}
+
 fn updates_name(name: &str) -> String {
     format!("store-{}-updates", name)
 }
@@ -90,6 +205,8 @@ pub struct Index {
     pub documents_fields_counts: DocumentsFieldsCounts,
     pub synonyms: Synonyms,
     pub docs_words: DocsWords,
+    pub prefix_documents_cache: PrefixDocumentsCache,
+    pub prefix_postings_lists_cache: PrefixPostingsListsCache,
 
     pub updates: Updates,
     pub updates_results: UpdatesResults,
@@ -107,10 +224,7 @@ impl Index {
         let schema = schema.ok_or(Error::SchemaMissing)?;
 
         let attributes = match attributes {
-            Some(attributes) => attributes
-                .iter()
-                .map(|name| schema.attribute(name))
-                .collect(),
+            Some(attributes) => Some(attributes.iter().filter_map(|name| schema.id(*name)).collect()),
             None => None,
         };
 
@@ -119,7 +233,7 @@ impl Index {
             reader,
             documents_fields: self.documents_fields,
             schema: &schema,
-            attributes: attributes.as_ref(),
+            fields: attributes.as_ref(),
         };
 
         Ok(Option::<T>::deserialize(&mut deserializer)?)
@@ -129,7 +243,7 @@ impl Index {
         &self,
         reader: &heed::RoTxn<MainT>,
         document_id: DocumentId,
-        attribute: SchemaAttr,
+        attribute: FieldId,
     ) -> MResult<Option<T>> {
         let bytes = self
             .documents_fields
@@ -140,9 +254,19 @@ impl Index {
         }
     }
 
-    pub fn schema_update(&self, writer: &mut heed::RwTxn<UpdateT>, schema: Schema) -> MResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        update::push_schema_update(writer, self.updates, self.updates_results, schema)
+    pub fn document_attribute_bytes<'txn>(
+        &self,
+        reader: &'txn heed::RoTxn<MainT>,
+        document_id: DocumentId,
+        attribute: FieldId,
+    ) -> MResult<Option<&'txn [u8]>> {
+        let bytes = self
+            .documents_fields
+            .document_attribute(reader, document_id, attribute)?;
+        match bytes {
+            Some(bytes) => Ok(Some(bytes)),
+            None => Ok(None),
+        }
     }
 
     pub fn customs_update(&self, writer: &mut heed::RwTxn<UpdateT>, customs: Vec<u8>) -> ZResult<u64> {
@@ -150,6 +274,11 @@ impl Index {
         update::push_customs_update(writer, self.updates, self.updates_results, customs)
     }
 
+    pub fn settings_update(&self, writer: &mut heed::RwTxn<UpdateT>, update: SettingsUpdate) -> ZResult<u64> {
+        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
+        update::push_settings_update(writer, self.updates, self.updates_results, update)
+    }
+
     pub fn documents_addition<D>(&self) -> update::DocumentsAddition<D> {
         update::DocumentsAddition::new(
             self.updates,
@@ -179,38 +308,6 @@ impl Index {
         update::push_clear_all(writer, self.updates, self.updates_results)
     }
 
-    pub fn synonyms_addition(&self) -> update::SynonymsAddition {
-        update::SynonymsAddition::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn synonyms_deletion(&self) -> update::SynonymsDeletion {
-        update::SynonymsDeletion::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn stop_words_addition(&self) -> update::StopWordsAddition {
-        update::StopWordsAddition::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn stop_words_deletion(&self) -> update::StopWordsDeletion {
-        update::StopWordsDeletion::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
     pub fn current_update_id(&self, reader: &heed::RoTxn<UpdateT>) -> MResult<Option<u64>> {
         match self.updates.last_update(reader)? {
             Some((id, _)) => Ok(Some(id)),
@@ -237,14 +334,14 @@ impl Index {
             for id in 0..=last_id {
                 if let Some(update) = self.update_status(reader, id)? {
                     updates.push(update);
-                    last_update_result_id = id;
+                    last_update_result_id = id + 1;
                 }
             }
         }
 
         // retrieve all enqueued updates
         if let Some((last_id, _)) = self.updates.last_update(reader)? {
-            for id in last_update_result_id + 1..=last_id {
+            for id in last_update_result_id..=last_id {
                 if let Some(update) = self.update_status(reader, id)? {
                     updates.push(update);
                 }
@@ -260,6 +357,8 @@ impl Index {
             self.postings_lists,
             self.documents_fields_counts,
             self.synonyms,
+            self.prefix_documents_cache,
+            self.prefix_postings_lists_cache,
         )
     }
 
@@ -272,6 +371,8 @@ impl Index {
             self.postings_lists,
             self.documents_fields_counts,
             self.synonyms,
+            self.prefix_documents_cache,
+            self.prefix_postings_lists_cache,
             criteria,
         )
     }
@@ -290,6 +391,8 @@ pub fn create(
     let documents_fields_counts_name = documents_fields_counts_name(name);
     let synonyms_name = synonyms_name(name);
     let docs_words_name = docs_words_name(name);
+    let prefix_documents_cache_name = prefix_documents_cache_name(name);
+    let prefix_postings_lists_cache_name = prefix_postings_lists_cache_name(name);
     let updates_name = updates_name(name);
     let updates_results_name = updates_results_name(name);
 
@@ -300,6 +403,8 @@ pub fn create(
     let documents_fields_counts = env.create_database(Some(&documents_fields_counts_name))?;
     let synonyms = env.create_database(Some(&synonyms_name))?;
     let docs_words = env.create_database(Some(&docs_words_name))?;
+    let prefix_documents_cache = env.create_database(Some(&prefix_documents_cache_name))?;
+    let prefix_postings_lists_cache = env.create_database(Some(&prefix_postings_lists_cache_name))?;
     let updates = update_env.create_database(Some(&updates_name))?;
     let updates_results = update_env.create_database(Some(&updates_results_name))?;
 
@@ -307,11 +412,11 @@ pub fn create(
         main: Main { main },
         postings_lists: PostingsLists { postings_lists },
         documents_fields: DocumentsFields { documents_fields },
-        documents_fields_counts: DocumentsFieldsCounts {
-            documents_fields_counts,
-        },
+        documents_fields_counts: DocumentsFieldsCounts { documents_fields_counts },
         synonyms: Synonyms { synonyms },
         docs_words: DocsWords { docs_words },
+        prefix_postings_lists_cache: PrefixPostingsListsCache { prefix_postings_lists_cache },
+        prefix_documents_cache: PrefixDocumentsCache { prefix_documents_cache },
         updates: Updates { updates },
         updates_results: UpdatesResults { updates_results },
         updates_notifier,
@@ -331,6 +436,8 @@ pub fn open(
     let documents_fields_counts_name = documents_fields_counts_name(name);
     let synonyms_name = synonyms_name(name);
     let docs_words_name = docs_words_name(name);
+    let prefix_documents_cache_name = prefix_documents_cache_name(name);
+    let prefix_postings_lists_cache_name = prefix_postings_lists_cache_name(name);
     let updates_name = updates_name(name);
     let updates_results_name = updates_results_name(name);
 
@@ -359,6 +466,14 @@ pub fn open(
         Some(docs_words) => docs_words,
         None => return Ok(None),
     };
+    let prefix_documents_cache = match env.open_database(Some(&prefix_documents_cache_name))? {
+        Some(prefix_documents_cache) => prefix_documents_cache,
+        None => return Ok(None),
+    };
+    let prefix_postings_lists_cache = match env.open_database(Some(&prefix_postings_lists_cache_name))? {
+        Some(prefix_postings_lists_cache) => prefix_postings_lists_cache,
+        None => return Ok(None),
+    };
     let updates = match update_env.open_database(Some(&updates_name))? {
         Some(updates) => updates,
         None => return Ok(None),
@@ -372,11 +487,11 @@ pub fn open(
         main: Main { main },
         postings_lists: PostingsLists { postings_lists },
         documents_fields: DocumentsFields { documents_fields },
-        documents_fields_counts: DocumentsFieldsCounts {
-            documents_fields_counts,
-        },
+        documents_fields_counts: DocumentsFieldsCounts { documents_fields_counts },
         synonyms: Synonyms { synonyms },
         docs_words: DocsWords { docs_words },
+        prefix_documents_cache: PrefixDocumentsCache { prefix_documents_cache },
+        prefix_postings_lists_cache: PrefixPostingsListsCache { prefix_postings_lists_cache },
         updates: Updates { updates },
         updates_results: UpdatesResults { updates_results },
         updates_notifier,
@@ -395,6 +510,8 @@ pub fn clear(
     index.documents_fields_counts.clear(writer)?;
     index.synonyms.clear(writer)?;
     index.docs_words.clear(writer)?;
+    index.prefix_documents_cache.clear(writer)?;
+    index.prefix_postings_lists_cache.clear(writer)?;
     index.updates.clear(update_writer)?;
     index.updates_results.clear(update_writer)?;
     Ok(())

meilisearch-core/src/store/postings_lists.rs

@@ -1,13 +1,17 @@
-use crate::DocIndex;
-use crate::database::MainT;
-use heed::types::{ByteSlice, CowSlice};
-use heed::Result as ZResult;
-use sdset::{Set, SetBuf};
 use std::borrow::Cow;
 
+use heed::Result as ZResult;
+use heed::types::ByteSlice;
+use sdset::{Set, SetBuf};
+use slice_group_by::GroupBy;
+
+use crate::database::MainT;
+use crate::DocIndex;
+use crate::store::{Postings, PostingsCodec};
+
 #[derive(Copy, Clone)]
 pub struct PostingsLists {
-    pub(crate) postings_lists: heed::Database<ByteSlice, CowSlice<DocIndex>>,
+    pub(crate) postings_lists: heed::Database<ByteSlice, PostingsCodec>,
 }
 
 impl PostingsLists {
@@ -15,9 +19,14 @@ impl PostingsLists {
         self,
         writer: &mut heed::RwTxn<MainT>,
         word: &[u8],
-        words_indexes: &Set<DocIndex>,
+        matches: &Set<DocIndex>,
     ) -> ZResult<()> {
-        self.postings_lists.put(writer, word, words_indexes)
+        let docids = matches.linear_group_by_key(|m| m.document_id).map(|g| g[0].document_id).collect();
+        let docids = Cow::Owned(SetBuf::new_unchecked(docids));
+        let matches = Cow::Borrowed(matches);
+        let postings = Postings { docids, matches };
+
+        self.postings_lists.put(writer, word, &postings)
     }
 
     pub fn del_postings_list(self, writer: &mut heed::RwTxn<MainT>, word: &[u8]) -> ZResult<bool> {
@@ -32,11 +41,7 @@ impl PostingsLists {
         self,
         reader: &'txn heed::RoTxn<MainT>,
         word: &[u8],
-    ) -> ZResult<Option<Cow<'txn, Set<DocIndex>>>> {
-        match self.postings_lists.get(reader, word)? {
-            Some(Cow::Borrowed(slice)) => Ok(Some(Cow::Borrowed(Set::new_unchecked(slice)))),
-            Some(Cow::Owned(vec)) => Ok(Some(Cow::Owned(SetBuf::new_unchecked(vec)))),
-            None => Ok(None),
-        }
+    ) -> ZResult<Option<Postings<'txn>>> {
+        self.postings_lists.get(reader, word)
     }
 }

meilisearch-core/src/store/prefix_documents_cache.rs

@@ -0,0 +1,80 @@
+use std::borrow::Cow;
+
+use heed::types::{OwnedType, CowSlice};
+use heed::Result as ZResult;
+use zerocopy::{AsBytes, FromBytes};
+
+use super::BEU64;
+use crate::{DocumentId, Highlight};
+use crate::database::MainT;
+
+#[derive(Debug, Copy, Clone, AsBytes, FromBytes)]
+#[repr(C)]
+pub struct PrefixKey {
+    prefix: [u8; 4],
+    index: BEU64,
+    docid: BEU64,
+}
+
+impl PrefixKey {
+    pub fn new(prefix: [u8; 4], index: u64, docid: u64) -> PrefixKey {
+        PrefixKey {
+            prefix,
+            index: BEU64::new(index),
+            docid: BEU64::new(docid),
+        }
+    }
+}
+
+#[derive(Copy, Clone)]
+pub struct PrefixDocumentsCache {
+    pub(crate) prefix_documents_cache: heed::Database<OwnedType<PrefixKey>, CowSlice<Highlight>>,
+}
+
+impl PrefixDocumentsCache {
+    pub fn put_prefix_document(
+        self,
+        writer: &mut heed::RwTxn<MainT>,
+        prefix: [u8; 4],
+        index: usize,
+        docid: DocumentId,
+        highlights: &[Highlight],
+    ) -> ZResult<()> {
+        let key = PrefixKey::new(prefix, index as u64, docid.0);
+        self.prefix_documents_cache.put(writer, &key, highlights)
+    }
+
+    pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
+        self.prefix_documents_cache.clear(writer)
+    }
+
+    pub fn prefix_documents<'txn>(
+        self,
+        reader: &'txn heed::RoTxn<MainT>,
+        prefix: [u8; 4],
+    ) -> ZResult<PrefixDocumentsIter<'txn>> {
+        let start = PrefixKey::new(prefix, 0, 0);
+        let end = PrefixKey::new(prefix, u64::max_value(), u64::max_value());
+        let iter = self.prefix_documents_cache.range(reader, &(start..=end))?;
+        Ok(PrefixDocumentsIter { iter })
+    }
+}
+
+pub struct PrefixDocumentsIter<'txn> {
+    iter: heed::RoRange<'txn, OwnedType<PrefixKey>, CowSlice<Highlight>>,
+}
+
+impl<'txn> Iterator for PrefixDocumentsIter<'txn> {
+    type Item = ZResult<(DocumentId, Cow<'txn, [Highlight]>)>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.iter.next() {
+            Some(Ok((key, highlights))) => {
+                let docid = DocumentId(key.docid.get());
+                Some(Ok((docid, highlights)))
+            }
+            Some(Err(e)) => Some(Err(e)),
+            None => None,
+        }
+    }
+}

meilisearch-core/src/store/prefix_postings_lists_cache.rs

@@ -0,0 +1,45 @@
+use std::borrow::Cow;
+
+use heed::Result as ZResult;
+use heed::types::OwnedType;
+use sdset::{Set, SetBuf};
+use slice_group_by::GroupBy;
+
+use crate::database::MainT;
+use crate::DocIndex;
+use crate::store::{PostingsCodec, Postings};
+
+#[derive(Copy, Clone)]
+pub struct PrefixPostingsListsCache {
+    pub(crate) prefix_postings_lists_cache: heed::Database<OwnedType<[u8; 4]>, PostingsCodec>,
+}
+
+impl PrefixPostingsListsCache {
+    pub fn put_prefix_postings_list(
+        self,
+        writer: &mut heed::RwTxn<MainT>,
+        prefix: [u8; 4],
+        matches: &Set<DocIndex>,
+    ) -> ZResult<()>
+    {
+        let docids = matches.linear_group_by_key(|m| m.document_id).map(|g| g[0].document_id).collect();
+        let docids = Cow::Owned(SetBuf::new_unchecked(docids));
+        let matches = Cow::Borrowed(matches);
+        let postings = Postings { docids, matches };
+
+        self.prefix_postings_lists_cache.put(writer, &prefix, &postings)
+    }
+
+    pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
+        self.prefix_postings_lists_cache.clear(writer)
+    }
+
+    pub fn prefix_postings_list<'txn>(
+        self,
+        reader: &'txn heed::RoTxn<MainT>,
+        prefix: [u8; 4],
+    ) -> ZResult<Option<Postings<'txn>>>
+    {
+        self.prefix_postings_lists_cache.get(reader, &prefix)
+    }
+}

meilisearch-core/src/update/clear_all.rs

@@ -4,19 +4,17 @@ use crate::{store, MResult, RankedMap};
 
 pub fn apply_clear_all(
     writer: &mut heed::RwTxn<MainT>,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
+    index: &store::Index,
 ) -> MResult<()> {
-    main_store.put_words_fst(writer, &fst::Set::default())?;
-    main_store.put_ranked_map(writer, &RankedMap::default())?;
-    main_store.put_number_of_documents(writer, |_| 0)?;
-    documents_fields_store.clear(writer)?;
-    documents_fields_counts_store.clear(writer)?;
-    postings_lists_store.clear(writer)?;
-    docs_words_store.clear(writer)?;
+    index.main.put_words_fst(writer, &fst::Set::default())?;
+    index.main.put_ranked_map(writer, &RankedMap::default())?;
+    index.main.put_number_of_documents(writer, |_| 0)?;
+    index.documents_fields.clear(writer)?;
+    index.documents_fields_counts.clear(writer)?;
+    index.postings_lists.clear(writer)?;
+    index.docs_words.clear(writer)?;
+    index.prefix_documents_cache.clear(writer)?;
+    index.prefix_postings_lists_cache.clear(writer)?;
 
     Ok(())
 }

meilisearch-core/src/update/documents_addition.rs

@@ -1,15 +1,16 @@
 use std::collections::HashMap;
 
 use fst::{set::OpBuilder, SetBuilder};
+use indexmap::IndexMap;
 use sdset::{duo::Union, SetOperation};
 use serde::{Deserialize, Serialize};
 
 use crate::database::{MainT, UpdateT};
 use crate::database::{UpdateEvent, UpdateEventsEmitter};
 use crate::raw_indexer::RawIndexer;
-use crate::serde::{extract_document_id, serialize_value, Deserializer, Serializer};
+use crate::serde::{extract_document_id, serialize_value_with_id, Deserializer, Serializer};
 use crate::store;
-use crate::update::{apply_documents_deletion, next_update_id, Update};
+use crate::update::{apply_documents_deletion, compute_short_prefixes, next_update_id, Update};
 use crate::{Error, MResult, RankedMap};
 
 pub struct DocumentsAddition<D> {
@@ -104,25 +105,21 @@ pub fn push_documents_addition<D: serde::Serialize>(
 
 pub fn apply_documents_addition<'a, 'b>(
     writer: &'a mut heed::RwTxn<'b, MainT>,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-    addition: Vec<HashMap<String, serde_json::Value>>,
+    index: &store::Index,
+    addition: Vec<IndexMap<String, serde_json::Value>>,
 ) -> MResult<()> {
     let mut documents_additions = HashMap::new();
 
-    let schema = match main_store.schema(writer)? {
+    let mut schema = match index.main.schema(writer)? {
         Some(schema) => schema,
         None => return Err(Error::SchemaMissing),
     };
 
-    let identifier = schema.identifier_name();
+    let primary_key = schema.primary_key().ok_or(Error::MissingPrimaryKey)?;
 
     // 1. store documents ids for future deletion
     for document in addition {
-        let document_id = match extract_document_id(identifier, &document)? {
+        let document_id = match extract_document_id(&primary_key, &document)? {
             Some(id) => id,
             None => return Err(Error::MissingDocumentId),
         };
@@ -133,22 +130,14 @@ pub fn apply_documents_addition<'a, 'b>(
     // 2. remove the documents posting lists
     let number_of_inserted_documents = documents_additions.len();
     let documents_ids = documents_additions.iter().map(|(id, _)| *id).collect();
-    apply_documents_deletion(
-        writer,
-        main_store,
-        documents_fields_store,
-        documents_fields_counts_store,
-        postings_lists_store,
-        docs_words_store,
-        documents_ids,
-    )?;
+    apply_documents_deletion(writer, index, documents_ids)?;
 
-    let mut ranked_map = match main_store.ranked_map(writer)? {
+    let mut ranked_map = match index.main.ranked_map(writer)? {
         Some(ranked_map) => ranked_map,
         None => RankedMap::default(),
     };
 
-    let stop_words = match main_store.stop_words_fst(writer)? {
+    let stop_words = match index.main.stop_words_fst(writer)? {
         Some(stop_words) => stop_words,
         None => fst::Set::default(),
     };
@@ -159,9 +148,9 @@ pub fn apply_documents_addition<'a, 'b>(
     for (document_id, document) in documents_additions {
         let serializer = Serializer {
             txn: writer,
-            schema: &schema,
-            document_store: documents_fields_store,
-            document_fields_counts: documents_fields_counts_store,
+            schema: &mut schema,
+            document_store: index.documents_fields,
+            document_fields_counts: index.documents_fields_counts,
             indexer: &mut indexer,
             ranked_map: &mut ranked_map,
             document_id,
@@ -172,36 +161,34 @@ pub fn apply_documents_addition<'a, 'b>(
 
     write_documents_addition_index(
         writer,
-        main_store,
-        postings_lists_store,
-        docs_words_store,
+        index,
         &ranked_map,
         number_of_inserted_documents,
         indexer,
-    )
+    )?;
+
+    index.main.put_schema(writer, &schema)?;
+
+    Ok(())
 }
 
 pub fn apply_documents_partial_addition<'a, 'b>(
     writer: &'a mut heed::RwTxn<'b, MainT>,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-    addition: Vec<HashMap<String, serde_json::Value>>,
+    index: &store::Index,
+    addition: Vec<IndexMap<String, serde_json::Value>>,
 ) -> MResult<()> {
     let mut documents_additions = HashMap::new();
 
-    let schema = match main_store.schema(writer)? {
+    let mut schema = match index.main.schema(writer)? {
         Some(schema) => schema,
         None => return Err(Error::SchemaMissing),
     };
 
-    let identifier = schema.identifier_name();
+    let primary_key = schema.primary_key().ok_or(Error::MissingPrimaryKey)?;
 
     // 1. store documents ids for future deletion
     for mut document in addition {
-        let document_id = match extract_document_id(identifier, &document)? {
+        let document_id = match extract_document_id(&primary_key, &document)? {
             Some(id) => id,
             None => return Err(Error::MissingDocumentId),
         };
@@ -209,9 +196,9 @@ pub fn apply_documents_partial_addition<'a, 'b>(
         let mut deserializer = Deserializer {
             document_id,
             reader: writer,
-            documents_fields: documents_fields_store,
+            documents_fields: index.documents_fields,
             schema: &schema,
-            attributes: None,
+            fields: None,
         };
 
         // retrieve the old document and
@@ -229,22 +216,14 @@ pub fn apply_documents_partial_addition<'a, 'b>(
     // 2. remove the documents posting lists
     let number_of_inserted_documents = documents_additions.len();
     let documents_ids = documents_additions.iter().map(|(id, _)| *id).collect();
-    apply_documents_deletion(
-        writer,
-        main_store,
-        documents_fields_store,
-        documents_fields_counts_store,
-        postings_lists_store,
-        docs_words_store,
-        documents_ids,
-    )?;
+    apply_documents_deletion(writer, index, documents_ids)?;
 
-    let mut ranked_map = match main_store.ranked_map(writer)? {
+    let mut ranked_map = match index.main.ranked_map(writer)? {
         Some(ranked_map) => ranked_map,
         None => RankedMap::default(),
     };
 
-    let stop_words = match main_store.stop_words_fst(writer)? {
+    let stop_words = match index.main.stop_words_fst(writer)? {
         Some(stop_words) => stop_words,
         None => fst::Set::default(),
     };
@@ -255,9 +234,9 @@ pub fn apply_documents_partial_addition<'a, 'b>(
     for (document_id, document) in documents_additions {
         let serializer = Serializer {
             txn: writer,
-            schema: &schema,
-            document_store: documents_fields_store,
-            document_fields_counts: documents_fields_counts_store,
+            schema: &mut schema,
+            document_store: index.documents_fields,
+            document_fields_counts: index.documents_fields_counts,
             indexer: &mut indexer,
             ranked_map: &mut ranked_map,
             document_id,
@@ -268,24 +247,19 @@ pub fn apply_documents_partial_addition<'a, 'b>(
 
     write_documents_addition_index(
         writer,
-        main_store,
-        postings_lists_store,
-        docs_words_store,
+        index,
         &ranked_map,
         number_of_inserted_documents,
         indexer,
-    )
+    )?;
+
+    index.main.put_schema(writer, &schema)?;
+
+    Ok(())
 }
 
-pub fn reindex_all_documents(
-    writer: &mut heed::RwTxn<MainT>,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-) -> MResult<()> {
-    let schema = match main_store.schema(writer)? {
+pub fn reindex_all_documents(writer: &mut heed::RwTxn<MainT>, index: &store::Index) -> MResult<()> {
+    let schema = match index.main.schema(writer)? {
         Some(schema) => schema,
         None => return Err(Error::SchemaMissing),
     };
@@ -294,71 +268,66 @@ pub fn reindex_all_documents(
 
     // 1. retrieve all documents ids
     let mut documents_ids_to_reindex = Vec::new();
-    for result in documents_fields_counts_store.documents_ids(writer)? {
+    for result in index.documents_fields_counts.documents_ids(writer)? {
         let document_id = result?;
         documents_ids_to_reindex.push(document_id);
     }
 
     // 2. remove the documents posting lists
-    main_store.put_words_fst(writer, &fst::Set::default())?;
-    main_store.put_ranked_map(writer, &ranked_map)?;
-    main_store.put_number_of_documents(writer, |_| 0)?;
-    postings_lists_store.clear(writer)?;
-    docs_words_store.clear(writer)?;
+    index.main.put_words_fst(writer, &fst::Set::default())?;
+    index.main.put_ranked_map(writer, &ranked_map)?;
+    index.main.put_number_of_documents(writer, |_| 0)?;
+    index.postings_lists.clear(writer)?;
+    index.docs_words.clear(writer)?;
 
-    // 3. re-index chunks of documents (otherwise we make the borrow checker unhappy)
-    for documents_ids in documents_ids_to_reindex.chunks(100) {
-        let stop_words = match main_store.stop_words_fst(writer)? {
-            Some(stop_words) => stop_words,
-            None => fst::Set::default(),
-        };
+    let stop_words = match index.main.stop_words_fst(writer)? {
+        Some(stop_words) => stop_words,
+        None => fst::Set::default(),
+    };
 
-        let number_of_inserted_documents = documents_ids.len();
-        let mut indexer = RawIndexer::new(stop_words);
-        let mut ram_store = HashMap::new();
+    let number_of_inserted_documents = documents_ids_to_reindex.len();
+    let mut indexer = RawIndexer::new(stop_words);
+    let mut ram_store = HashMap::new();
 
-        for document_id in documents_ids {
-            for result in documents_fields_store.document_fields(writer, *document_id)? {
-                let (attr, bytes) = result?;
-                let value: serde_json::Value = serde_json::from_slice(bytes)?;
-                ram_store.insert((document_id, attr), value);
-            }
-
-            for ((docid, attr), value) in ram_store.drain() {
-                serialize_value(
-                    writer,
-                    attr,
-                    schema.props(attr),
-                    *docid,
-                    documents_fields_store,
-                    documents_fields_counts_store,
-                    &mut indexer,
-                    &mut ranked_map,
-                    &value,
-                )?;
-            }
+    for document_id in documents_ids_to_reindex {
+        for result in index.documents_fields.document_fields(writer, document_id)? {
+            let (field_id, bytes) = result?;
+            let value: serde_json::Value = serde_json::from_slice(bytes)?;
+            ram_store.insert((document_id, field_id), value);
         }
 
-        // 4. write the new index in the main store
-        write_documents_addition_index(
-            writer,
-            main_store,
-            postings_lists_store,
-            docs_words_store,
-            &ranked_map,
-            number_of_inserted_documents,
-            indexer,
-        )?;
+        for ((docid, field_id), value) in ram_store.drain() {
+            serialize_value_with_id(
+                writer,
+                field_id,
+                &schema,
+                docid,
+                index.documents_fields,
+                index.documents_fields_counts,
+                &mut indexer,
+                &mut ranked_map,
+                &value
+            )?;
+        }
     }
 
+    // 4. write the new index in the main store
+    write_documents_addition_index(
+        writer,
+        index,
+        &ranked_map,
+        number_of_inserted_documents,
+        indexer,
+    )?;
+
+    index.main.put_schema(writer, &schema)?;
+
     Ok(())
 }
 
 pub fn write_documents_addition_index(
     writer: &mut heed::RwTxn<MainT>,
-    main_store: store::Main,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
+    index: &store::Index,
     ranked_map: &RankedMap,
     number_of_inserted_documents: usize,
     indexer: RawIndexer,
@@ -369,16 +338,16 @@ pub fn write_documents_addition_index(
     for (word, delta_set) in indexed.words_doc_indexes {
         delta_words_builder.insert(&word).unwrap();
 
-        let set = match postings_lists_store.postings_list(writer, &word)? {
-            Some(set) => Union::new(&set, &delta_set).into_set_buf(),
+        let set = match index.postings_lists.postings_list(writer, &word)? {
+            Some(postings) => Union::new(&postings.matches, &delta_set).into_set_buf(),
             None => delta_set,
         };
 
-        postings_lists_store.put_postings_list(writer, &word, &set)?;
+        index.postings_lists.put_postings_list(writer, &word, &set)?;
     }
 
     for (id, words) in indexed.docs_words {
-        docs_words_store.put_doc_words(writer, id, &words)?;
+        index.docs_words.put_doc_words(writer, id, &words)?;
     }
 
     let delta_words = delta_words_builder
@@ -386,7 +355,7 @@ pub fn write_documents_addition_index(
         .and_then(fst::Set::from_bytes)
         .unwrap();
 
-    let words = match main_store.words_fst(writer)? {
+    let words = match index.main.words_fst(writer)? {
         Some(words) => {
             let op = OpBuilder::new()
                 .add(words.stream())
@@ -403,9 +372,11 @@ pub fn write_documents_addition_index(
         None => delta_words,
     };
 
-    main_store.put_words_fst(writer, &words)?;
-    main_store.put_ranked_map(writer, ranked_map)?;
-    main_store.put_number_of_documents(writer, |old| old + number_of_inserted_documents as u64)?;
+    index.main.put_words_fst(writer, &words)?;
+    index.main.put_ranked_map(writer, ranked_map)?;
+    index.main.put_number_of_documents(writer, |old| old + number_of_inserted_documents as u64)?;
+
+    compute_short_prefixes(writer, index)?;
 
     Ok(())
 }

meilisearch-core/src/update/documents_deletion.rs

@@ -8,7 +8,7 @@ use crate::database::{MainT, UpdateT};
 use crate::database::{UpdateEvent, UpdateEventsEmitter};
 use crate::serde::extract_document_id;
 use crate::store;
-use crate::update::{next_update_id, Update};
+use crate::update::{next_update_id, compute_short_prefixes, Update};
 use crate::{DocumentId, Error, MResult, RankedMap};
 
 pub struct DocumentsDeletion {
@@ -40,8 +40,8 @@ impl DocumentsDeletion {
     where
         D: serde::Serialize,
     {
-        let identifier = schema.identifier_name();
-        let document_id = match extract_document_id(identifier, &document)? {
+        let primary_key = schema.primary_key().ok_or(Error::MissingPrimaryKey)?;
+        let document_id = match extract_document_id(&primary_key, &document)? {
             Some(id) => id,
             None => return Err(Error::MissingDocumentId),
         };
@@ -85,47 +85,32 @@ pub fn push_documents_deletion(
 
 pub fn apply_documents_deletion(
     writer: &mut heed::RwTxn<MainT>,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
+    index: &store::Index,
     deletion: Vec<DocumentId>,
 ) -> MResult<()> {
     let idset = SetBuf::from_dirty(deletion);
 
-    let schema = match main_store.schema(writer)? {
+    let schema = match index.main.schema(writer)? {
         Some(schema) => schema,
         None => return Err(Error::SchemaMissing),
     };
 
-    let mut ranked_map = match main_store.ranked_map(writer)? {
+    let mut ranked_map = match index.main.ranked_map(writer)? {
         Some(ranked_map) => ranked_map,
         None => RankedMap::default(),
     };
 
     // collect the ranked attributes according to the schema
-    let ranked_attrs: Vec<_> = schema
-        .iter()
-        .filter_map(
-            |(_, attr, prop)| {
-                if prop.is_ranked() {
-                    Some(attr)
-                } else {
-                    None
-                }
-            },
-        )
-        .collect();
+    let ranked_fields = schema.ranked();
 
     let mut words_document_ids = HashMap::new();
     for id in idset {
         // remove all the ranked attributes from the ranked_map
-        for ranked_attr in &ranked_attrs {
+        for ranked_attr in ranked_fields {
             ranked_map.remove(id, *ranked_attr);
         }
 
-        if let Some(words) = docs_words_store.doc_words(writer, id)? {
+        if let Some(words) = index.docs_words.doc_words(writer, id)? {
             let mut stream = words.stream();
             while let Some(word) = stream.next() {
                 let word = word.to_vec();
@@ -142,21 +127,21 @@ pub fn apply_documents_deletion(
     for (word, document_ids) in words_document_ids {
         let document_ids = SetBuf::from_dirty(document_ids);
 
-        if let Some(doc_indexes) = postings_lists_store.postings_list(writer, &word)? {
-            let op = DifferenceByKey::new(&doc_indexes, &document_ids, |d| d.document_id, |id| *id);
+        if let Some(postings) = index.postings_lists.postings_list(writer, &word)? {
+            let op = DifferenceByKey::new(&postings.matches, &document_ids, |d| d.document_id, |id| *id);
             let doc_indexes = op.into_set_buf();
 
             if !doc_indexes.is_empty() {
-                postings_lists_store.put_postings_list(writer, &word, &doc_indexes)?;
+                index.postings_lists.put_postings_list(writer, &word, &doc_indexes)?;
             } else {
-                postings_lists_store.del_postings_list(writer, &word)?;
+                index.postings_lists.del_postings_list(writer, &word)?;
                 removed_words.insert(word);
             }
         }
 
         for id in document_ids {
-            documents_fields_counts_store.del_all_document_fields_counts(writer, id)?;
-            if documents_fields_store.del_all_document_fields(writer, id)? != 0 {
+            index.documents_fields_counts.del_all_document_fields_counts(writer, id)?;
+            if index.documents_fields.del_all_document_fields(writer, id)? != 0 {
                 deleted_documents.insert(id);
             }
         }
@@ -164,11 +149,11 @@ pub fn apply_documents_deletion(
 
     let deleted_documents_len = deleted_documents.len() as u64;
     for id in deleted_documents {
-        docs_words_store.del_doc_words(writer, id)?;
+        index.docs_words.del_doc_words(writer, id)?;
     }
 
     let removed_words = fst::Set::from_iter(removed_words).unwrap();
-    let words = match main_store.words_fst(writer)? {
+    let words = match index.main.words_fst(writer)? {
         Some(words_set) => {
             let op = fst::set::OpBuilder::new()
                 .add(words_set.stream())
@@ -185,9 +170,11 @@ pub fn apply_documents_deletion(
         None => fst::Set::default(),
     };
 
-    main_store.put_words_fst(writer, &words)?;
-    main_store.put_ranked_map(writer, &ranked_map)?;
-    main_store.put_number_of_documents(writer, |old| old - deleted_documents_len)?;
+    index.main.put_words_fst(writer, &words)?;
+    index.main.put_ranked_map(writer, &ranked_map)?;
+    index.main.put_number_of_documents(writer, |old| old - deleted_documents_len)?;
+
+    compute_short_prefixes(writer, index)?;
 
     Ok(())
 }

meilisearch-core/src/update/mod.rs

@@ -2,11 +2,7 @@ mod clear_all;
 mod customs_update;
 mod documents_addition;
 mod documents_deletion;
-mod schema_update;
-mod stop_words_addition;
-mod stop_words_deletion;
-mod synonyms_addition;
-mod synonyms_deletion;
+mod settings_update;
 
 pub use self::clear_all::{apply_clear_all, push_clear_all};
 pub use self::customs_update::{apply_customs_update, push_customs_update};
@@ -14,24 +10,22 @@ pub use self::documents_addition::{
     apply_documents_addition, apply_documents_partial_addition, DocumentsAddition,
 };
 pub use self::documents_deletion::{apply_documents_deletion, DocumentsDeletion};
-pub use self::schema_update::{apply_schema_update, push_schema_update};
-pub use self::stop_words_addition::{apply_stop_words_addition, StopWordsAddition};
-pub use self::stop_words_deletion::{apply_stop_words_deletion, StopWordsDeletion};
-pub use self::synonyms_addition::{apply_synonyms_addition, SynonymsAddition};
-pub use self::synonyms_deletion::{apply_synonyms_deletion, SynonymsDeletion};
+pub use self::settings_update::{apply_settings_update, push_settings_update};
 
 use std::cmp;
-use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::time::Instant;
 
 use chrono::{DateTime, Utc};
+use fst::{IntoStreamer, Streamer};
 use heed::Result as ZResult;
+use indexmap::IndexMap;
 use log::debug;
+use sdset::Set;
 use serde::{Deserialize, Serialize};
 
 use crate::{store, DocumentId, MResult};
 use crate::database::{MainT, UpdateT};
-use meilisearch_schema::Schema;
+use crate::settings::SettingsUpdate;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Update {
@@ -47,13 +41,6 @@ impl Update {
         }
     }
 
-    fn schema(data: Schema) -> Update {
-        Update {
-            data: UpdateData::Schema(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
     fn customs(data: Vec<u8>) -> Update {
         Update {
             data: UpdateData::Customs(data),
@@ -61,14 +48,14 @@ impl Update {
         }
     }
 
-    fn documents_addition(data: Vec<HashMap<String, serde_json::Value>>) -> Update {
+    fn documents_addition(data: Vec<IndexMap<String, serde_json::Value>>) -> Update {
         Update {
             data: UpdateData::DocumentsAddition(data),
             enqueued_at: Utc::now(),
         }
     }
 
-    fn documents_partial(data: Vec<HashMap<String, serde_json::Value>>) -> Update {
+    fn documents_partial(data: Vec<IndexMap<String, serde_json::Value>>) -> Update {
         Update {
             data: UpdateData::DocumentsPartial(data),
             enqueued_at: Utc::now(),
@@ -82,30 +69,9 @@ impl Update {
         }
     }
 
-    fn synonyms_addition(data: BTreeMap<String, Vec<String>>) -> Update {
+    fn settings(data: SettingsUpdate) -> Update {
         Update {
-            data: UpdateData::SynonymsAddition(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn synonyms_deletion(data: BTreeMap<String, Option<Vec<String>>>) -> Update {
-        Update {
-            data: UpdateData::SynonymsDeletion(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn stop_words_addition(data: BTreeSet<String>) -> Update {
-        Update {
-            data: UpdateData::StopWordsAddition(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn stop_words_deletion(data: BTreeSet<String>) -> Update {
-        Update {
-            data: UpdateData::StopWordsDeletion(data),
+            data: UpdateData::Settings(data),
             enqueued_at: Utc::now(),
         }
     }
@@ -114,22 +80,17 @@ impl Update {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum UpdateData {
     ClearAll,
-    Schema(Schema),
     Customs(Vec<u8>),
-    DocumentsAddition(Vec<HashMap<String, serde_json::Value>>),
-    DocumentsPartial(Vec<HashMap<String, serde_json::Value>>),
+    DocumentsAddition(Vec<IndexMap<String, serde_json::Value>>),
+    DocumentsPartial(Vec<IndexMap<String, serde_json::Value>>),
     DocumentsDeletion(Vec<DocumentId>),
-    SynonymsAddition(BTreeMap<String, Vec<String>>),
-    SynonymsDeletion(BTreeMap<String, Option<Vec<String>>>),
-    StopWordsAddition(BTreeSet<String>),
-    StopWordsDeletion(BTreeSet<String>),
+    Settings(SettingsUpdate)
 }
 
 impl UpdateData {
     pub fn update_type(&self) -> UpdateType {
         match self {
             UpdateData::ClearAll => UpdateType::ClearAll,
-            UpdateData::Schema(_) => UpdateType::Schema,
             UpdateData::Customs(_) => UpdateType::Customs,
             UpdateData::DocumentsAddition(addition) => UpdateType::DocumentsAddition {
                 number: addition.len(),
@@ -140,17 +101,8 @@ impl UpdateData {
             UpdateData::DocumentsDeletion(deletion) => UpdateType::DocumentsDeletion {
                 number: deletion.len(),
             },
-            UpdateData::SynonymsAddition(addition) => UpdateType::SynonymsAddition {
-                number: addition.len(),
-            },
-            UpdateData::SynonymsDeletion(deletion) => UpdateType::SynonymsDeletion {
-                number: deletion.len(),
-            },
-            UpdateData::StopWordsAddition(addition) => UpdateType::StopWordsAddition {
-                number: addition.len(),
-            },
-            UpdateData::StopWordsDeletion(deletion) => UpdateType::StopWordsDeletion {
-                number: deletion.len(),
+            UpdateData::Settings(update) => UpdateType::Settings {
+                settings: update.clone(),
             },
         }
     }
@@ -160,18 +112,15 @@ impl UpdateData {
 #[serde(tag = "name")]
 pub enum UpdateType {
     ClearAll,
-    Schema,
     Customs,
     DocumentsAddition { number: usize },
     DocumentsPartial { number: usize },
     DocumentsDeletion { number: usize },
-    SynonymsAddition { number: usize },
-    SynonymsDeletion { number: usize },
-    StopWordsAddition { number: usize },
-    StopWordsDeletion { number: usize },
+    Settings { settings: SettingsUpdate },
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
 pub struct ProcessedUpdateResult {
     pub update_id: u64,
     #[serde(rename = "type")]
@@ -184,8 +133,10 @@ pub struct ProcessedUpdateResult {
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
 pub struct EnqueuedUpdateResult {
     pub update_id: u64,
+    #[serde(rename = "type")]
     pub update_type: UpdateType,
     pub enqueued_at: DateTime<Utc>,
 }
@@ -197,6 +148,10 @@ pub enum UpdateStatus {
         #[serde(flatten)]
         content: EnqueuedUpdateResult,
     },
+    Failed {
+        #[serde(flatten)]
+        content: ProcessedUpdateResult,
+    },
     Processed {
         #[serde(flatten)]
         content: ProcessedUpdateResult,
@@ -210,7 +165,13 @@ pub fn update_status(
     update_id: u64,
 ) -> MResult<Option<UpdateStatus>> {
     match updates_results_store.update_result(update_reader, update_id)? {
-        Some(result) => Ok(Some(UpdateStatus::Processed { content: result })),
+        Some(result) => {
+            if result.error.is_some() {
+                Ok(Some(UpdateStatus::Failed { content: result }))
+            } else {
+                Ok(Some(UpdateStatus::Processed { content: result }))
+            }
+        },
         None => match updates_store.get(update_reader, update_id)? {
             Some(update) => Ok(Some(UpdateStatus::Enqueued {
                 content: EnqueuedUpdateResult {
@@ -256,30 +217,7 @@ pub fn update_task<'a, 'b>(
             let start = Instant::now();
 
             let update_type = UpdateType::ClearAll;
-            let result = apply_clear_all(
-                writer,
-                index.main,
-                index.documents_fields,
-                index.documents_fields_counts,
-                index.postings_lists,
-                index.docs_words,
-            );
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::Schema(schema) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::Schema;
-            let result = apply_schema_update(
-                writer,
-                &schema,
-                index.main,
-                index.documents_fields,
-                index.documents_fields_counts,
-                index.postings_lists,
-                index.docs_words,
-            );
+            let result = apply_clear_all(writer, index);
 
             (update_type, result, start.elapsed())
         }
@@ -298,15 +236,7 @@ pub fn update_task<'a, 'b>(
                 number: documents.len(),
             };
 
-            let result = apply_documents_addition(
-                writer,
-                index.main,
-                index.documents_fields,
-                index.documents_fields_counts,
-                index.postings_lists,
-                index.docs_words,
-                documents,
-            );
+            let result = apply_documents_addition(writer, index, documents);
 
             (update_type, result, start.elapsed())
         }
@@ -317,15 +247,7 @@ pub fn update_task<'a, 'b>(
                 number: documents.len(),
             };
 
-            let result = apply_documents_partial_addition(
-                writer,
-                index.main,
-                index.documents_fields,
-                index.documents_fields_counts,
-                index.postings_lists,
-                index.docs_words,
-                documents,
-            );
+            let result = apply_documents_partial_addition(writer, index, documents);
 
             (update_type, result, start.elapsed())
         }
@@ -336,67 +258,21 @@ pub fn update_task<'a, 'b>(
                 number: documents.len(),
             };
 
-            let result = apply_documents_deletion(
+            let result = apply_documents_deletion(writer, index, documents);
+
+            (update_type, result, start.elapsed())
+        }
+        UpdateData::Settings(settings) => {
+            let start = Instant::now();
+
+            let update_type = UpdateType::Settings {
+                settings: settings.clone(),
+            };
+
+            let result = apply_settings_update(
                 writer,
-                index.main,
-                index.documents_fields,
-                index.documents_fields_counts,
-                index.postings_lists,
-                index.docs_words,
-                documents,
-            );
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::SynonymsAddition(synonyms) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::SynonymsAddition {
-                number: synonyms.len(),
-            };
-
-            let result = apply_synonyms_addition(writer, index.main, index.synonyms, synonyms);
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::SynonymsDeletion(synonyms) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::SynonymsDeletion {
-                number: synonyms.len(),
-            };
-
-            let result = apply_synonyms_deletion(writer, index.main, index.synonyms, synonyms);
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::StopWordsAddition(stop_words) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::StopWordsAddition {
-                number: stop_words.len(),
-            };
-
-            let result =
-                apply_stop_words_addition(writer, index.main, index.postings_lists, stop_words);
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::StopWordsDeletion(stop_words) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::StopWordsDeletion {
-                number: stop_words.len(),
-            };
-
-            let result = apply_stop_words_deletion(
-                writer,
-                index.main,
-                index.documents_fields,
-                index.documents_fields_counts,
-                index.postings_lists,
-                index.docs_words,
-                stop_words,
+                index,
+                settings,
             );
 
             (update_type, result, start.elapsed())
@@ -419,3 +295,67 @@ pub fn update_task<'a, 'b>(
 
     Ok(status)
 }
+
+fn compute_short_prefixes(writer: &mut heed::RwTxn<MainT>, index: &store::Index) -> MResult<()> {
+    // retrieve the words fst to compute all those prefixes
+    let words_fst = match index.main.words_fst(writer)? {
+        Some(fst) => fst,
+        None => return Ok(()),
+    };
+
+    // clear the prefixes
+    let pplc_store = index.prefix_postings_lists_cache;
+    pplc_store.clear(writer)?;
+
+    for prefix_len in 1..=2 {
+        // compute prefixes and store those in the PrefixPostingsListsCache store.
+        let mut previous_prefix: Option<([u8; 4], Vec<_>)> = None;
+        let mut stream = words_fst.into_stream();
+        while let Some(input) = stream.next() {
+
+            // We skip the prefixes that are shorter than the current length
+            // we want to cache (<). We must ignore the input when it is exactly the
+            // same word as the prefix because if we match exactly on it we need
+            // to consider it as an exact match and not as a prefix (=).
+            if input.len() <= prefix_len { continue }
+
+            if let Some(postings_list) = index.postings_lists.postings_list(writer, input)?.map(|p| p.matches.into_owned()) {
+                let prefix = &input[..prefix_len];
+
+                let mut arr_prefix = [0; 4];
+                arr_prefix[..prefix_len].copy_from_slice(prefix);
+
+                match previous_prefix {
+                    Some((ref mut prev_prefix, ref mut prev_pl)) if *prev_prefix != arr_prefix => {
+                        prev_pl.sort_unstable();
+                        prev_pl.dedup();
+
+                        if let Ok(prefix) = std::str::from_utf8(&prev_prefix[..prefix_len]) {
+                            debug!("writing the prefix of {:?} of length {}", prefix, prev_pl.len());
+                        }
+
+                        let pls = Set::new_unchecked(&prev_pl);
+                        pplc_store.put_prefix_postings_list(writer, *prev_prefix, &pls)?;
+
+                        *prev_prefix = arr_prefix;
+                        prev_pl.clear();
+                        prev_pl.extend_from_slice(&postings_list);
+                    },
+                    Some((_, ref mut prev_pl)) => prev_pl.extend_from_slice(&postings_list),
+                    None => previous_prefix = Some((arr_prefix, postings_list.to_vec())),
+                }
+            }
+        }
+
+        // write the last prefix postings lists
+        if let Some((prev_prefix, mut prev_pl)) = previous_prefix.take() {
+            prev_pl.sort_unstable();
+            prev_pl.dedup();
+
+            let pls = Set::new_unchecked(&prev_pl);
+            pplc_store.put_prefix_postings_list(writer, prev_prefix, &pls)?;
+        }
+    }
+
+    Ok(())
+}

meilisearch-core/src/update/schema_update.rs

@@ -1,76 +0,0 @@
-use meilisearch_schema::{Diff, Schema};
-
-use crate::database::{MainT, UpdateT};
-use crate::update::documents_addition::reindex_all_documents;
-use crate::update::{next_update_id, Update};
-use crate::{error::UnsupportedOperation, store, MResult};
-
-pub fn apply_schema_update(
-    writer: &mut heed::RwTxn<MainT>,
-    new_schema: &Schema,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-) -> MResult<()> {
-    use UnsupportedOperation::{
-        CanOnlyIntroduceNewSchemaAttributesAtEnd, CannotRemoveSchemaAttribute,
-        CannotReorderSchemaAttribute, CannotUpdateSchemaIdentifier,
-    };
-
-    let mut need_full_reindexing = false;
-
-    if let Some(old_schema) = main_store.schema(writer)? {
-        for diff in meilisearch_schema::diff(&old_schema, new_schema) {
-            match diff {
-                Diff::IdentChange { .. } => return Err(CannotUpdateSchemaIdentifier.into()),
-                Diff::AttrMove { .. } => return Err(CannotReorderSchemaAttribute.into()),
-                Diff::AttrPropsChange { old, new, .. } => {
-                    if new.indexed != old.indexed {
-                        need_full_reindexing = true;
-                    }
-                    if new.ranked != old.ranked {
-                        need_full_reindexing = true;
-                    }
-                }
-                Diff::NewAttr { pos, .. } => {
-                    // new attribute not at the end of the schema
-                    if pos < old_schema.number_of_attributes() {
-                        return Err(CanOnlyIntroduceNewSchemaAttributesAtEnd.into());
-                    }
-                }
-                Diff::RemovedAttr { .. } => return Err(CannotRemoveSchemaAttribute.into()),
-            }
-        }
-    }
-
-    main_store.put_schema(writer, new_schema)?;
-
-    if need_full_reindexing {
-        reindex_all_documents(
-            writer,
-            main_store,
-            documents_fields_store,
-            documents_fields_counts_store,
-            postings_lists_store,
-            docs_words_store,
-        )?
-    }
-
-    Ok(())
-}
-
-pub fn push_schema_update(
-    writer: &mut heed::RwTxn<UpdateT>,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    schema: Schema,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::schema(schema);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}

meilisearch-core/src/update/settings_update.rs

@@ -0,0 +1,301 @@
+use std::collections::{BTreeMap, BTreeSet};
+
+use heed::Result as ZResult;
+use fst::{set::OpBuilder, SetBuilder};
+use sdset::SetBuf;
+use meilisearch_schema::Schema;
+
+use crate::database::{MainT, UpdateT};
+use crate::settings::{UpdateState, SettingsUpdate, RankingRule};
+use crate::update::documents_addition::reindex_all_documents;
+use crate::update::{next_update_id, Update};
+use crate::{store, MResult, Error};
+
+pub fn push_settings_update(
+    writer: &mut heed::RwTxn<UpdateT>,
+    updates_store: store::Updates,
+    updates_results_store: store::UpdatesResults,
+    settings: SettingsUpdate,
+) -> ZResult<u64> {
+    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
+
+    let update = Update::settings(settings);
+    updates_store.put_update(writer, last_update_id, &update)?;
+
+    Ok(last_update_id)
+}
+
+pub fn apply_settings_update(
+    writer: &mut heed::RwTxn<MainT>,
+    index: &store::Index,
+    settings: SettingsUpdate,
+) -> MResult<()> {
+    let mut must_reindex = false;
+
+    let mut schema = match index.main.schema(writer)? {
+        Some(schema) => schema,
+        None => {
+            match settings.primary_key.clone() {
+                UpdateState::Update(id) => Schema::with_primary_key(&id),
+                _ => return Err(Error::MissingPrimaryKey)
+            }
+        }
+    };
+
+    match settings.ranking_rules {
+        UpdateState::Update(v) => {
+            let ranked_field: Vec<&str> = v.iter().filter_map(RankingRule::field).collect();
+            schema.update_ranked(&ranked_field)?;
+            for name in ranked_field {
+                if schema.accept_new_fields() {
+                    schema.set_indexed(name.as_ref())?;
+                    schema.set_displayed(name.as_ref())?;
+                }
+            }
+            index.main.put_ranking_rules(writer, &v)?;
+            must_reindex = true;
+        },
+        UpdateState::Clear => {
+            index.main.delete_ranking_rules(writer)?;
+            schema.clear_ranked();
+            must_reindex = true;
+        },
+        UpdateState::Nothing => (),
+    }
+
+    match settings.distinct_attribute {
+        UpdateState::Update(v) => {
+            index.main.put_distinct_attribute(writer, &v)?;
+        },
+        UpdateState::Clear => {
+            index.main.delete_distinct_attribute(writer)?;
+        },
+        UpdateState::Nothing => (),
+    }
+
+    match settings.accept_new_fields {
+        UpdateState::Update(v) => {
+            schema.set_accept_new_fields(v);
+        },
+        UpdateState::Clear => {
+            schema.set_accept_new_fields(true);
+        },
+        UpdateState::Nothing => (),
+    }
+
+    match settings.searchable_attributes.clone() {
+        UpdateState::Update(v) => {
+            schema.update_indexed(v)?;
+            must_reindex = true;
+        },
+        UpdateState::Clear => {
+            schema.set_all_fields_as_indexed();
+            must_reindex = true;
+        },
+        UpdateState::Nothing => (),
+    }
+    match settings.displayed_attributes.clone() {
+        UpdateState::Update(v) => schema.update_displayed(v)?,
+        UpdateState::Clear => {
+            schema.set_all_fields_as_displayed();
+        },
+        UpdateState::Nothing => (),
+    }
+
+    index.main.put_schema(writer, &schema)?;
+
+    match settings.stop_words {
+        UpdateState::Update(stop_words) => {
+            if apply_stop_words_update(writer, index, stop_words)? {
+                must_reindex = true;
+            }
+        },
+        UpdateState::Clear => {
+            if apply_stop_words_update(writer, index, BTreeSet::new())? {
+                must_reindex = true;
+            }
+        },
+        UpdateState::Nothing => (),
+    }
+
+    match settings.synonyms {
+        UpdateState::Update(synonyms) => apply_synonyms_update(writer, index, synonyms)?,
+        UpdateState::Clear => apply_synonyms_update(writer, index, BTreeMap::new())?,
+        UpdateState::Nothing => (),
+    }
+
+    if must_reindex {
+        reindex_all_documents(writer, index)?;
+    }
+
+    Ok(())
+}
+
+pub fn apply_stop_words_update(
+    writer: &mut heed::RwTxn<MainT>,
+    index: &store::Index,
+    stop_words: BTreeSet<String>,
+) -> MResult<bool>
+{
+    let mut must_reindex = false;
+
+    let old_stop_words: BTreeSet<String> = index.main
+        .stop_words_fst(writer)?
+        .unwrap_or_default()
+        .stream()
+        .into_strs()?
+        .into_iter()
+        .collect();
+
+    let deletion: BTreeSet<String> = old_stop_words.difference(&stop_words).cloned().collect();
+    let addition: BTreeSet<String> = stop_words.difference(&old_stop_words).cloned().collect();
+
+    if !addition.is_empty() {
+        apply_stop_words_addition(writer, index, addition)?;
+    }
+
+    if !deletion.is_empty() {
+        must_reindex = true;
+        apply_stop_words_deletion(writer, index, deletion)?;
+    }
+
+    if let Some(words_fst) = index.main.words_fst(writer)? {
+        let stop_words = fst::Set::from_iter(stop_words)?;
+        let op = OpBuilder::new()
+            .add(&words_fst)
+            .add(&stop_words)
+            .difference();
+
+        let mut builder = fst::SetBuilder::memory();
+        builder.extend_stream(op)?;
+        let words_fst = builder.into_inner().and_then(fst::Set::from_bytes)?;
+
+        index.main.put_words_fst(writer, &words_fst)?;
+        index.main.put_stop_words_fst(writer, &stop_words)?;
+    }
+
+    Ok(must_reindex)
+}
+
+fn apply_stop_words_addition(
+    writer: &mut heed::RwTxn<MainT>,
+    index: &store::Index,
+    addition: BTreeSet<String>,
+) -> MResult<()>
+{
+    let main_store = index.main;
+    let postings_lists_store = index.postings_lists;
+
+    let mut stop_words_builder = SetBuilder::memory();
+
+    for word in addition {
+        stop_words_builder.insert(&word)?;
+        // we remove every posting list associated to a new stop word
+        postings_lists_store.del_postings_list(writer, word.as_bytes())?;
+    }
+
+    // create the new delta stop words fst
+    let delta_stop_words = stop_words_builder
+        .into_inner()
+        .and_then(fst::Set::from_bytes)?;
+
+    // we also need to remove all the stop words from the main fst
+    if let Some(word_fst) = main_store.words_fst(writer)? {
+        let op = OpBuilder::new()
+            .add(&word_fst)
+            .add(&delta_stop_words)
+            .difference();
+
+        let mut word_fst_builder = SetBuilder::memory();
+        word_fst_builder.extend_stream(op)?;
+        let word_fst = word_fst_builder
+            .into_inner()
+            .and_then(fst::Set::from_bytes)?;
+
+        main_store.put_words_fst(writer, &word_fst)?;
+    }
+
+    // now we add all of these stop words from the main store
+    let stop_words_fst = main_store.stop_words_fst(writer)?.unwrap_or_default();
+
+    let op = OpBuilder::new()
+        .add(&stop_words_fst)
+        .add(&delta_stop_words)
+        .r#union();
+
+    let mut stop_words_builder = SetBuilder::memory();
+    stop_words_builder.extend_stream(op)?;
+    let stop_words_fst = stop_words_builder
+        .into_inner()
+        .and_then(fst::Set::from_bytes)?;
+
+    main_store.put_stop_words_fst(writer, &stop_words_fst)?;
+
+    Ok(())
+}
+
+fn apply_stop_words_deletion(
+    writer: &mut heed::RwTxn<MainT>,
+    index: &store::Index,
+    deletion: BTreeSet<String>,
+) -> MResult<()> {
+
+    let mut stop_words_builder = SetBuilder::memory();
+
+    for word in deletion {
+        stop_words_builder.insert(&word)?;
+    }
+
+    // create the new delta stop words fst
+    let delta_stop_words = stop_words_builder
+        .into_inner()
+        .and_then(fst::Set::from_bytes)?;
+
+    // now we delete all of these stop words from the main store
+    let stop_words_fst = index.main.stop_words_fst(writer)?.unwrap_or_default();
+
+    let op = OpBuilder::new()
+        .add(&stop_words_fst)
+        .add(&delta_stop_words)
+        .difference();
+
+    let mut stop_words_builder = SetBuilder::memory();
+    stop_words_builder.extend_stream(op)?;
+    let stop_words_fst = stop_words_builder.into_inner().and_then(fst::Set::from_bytes)?;
+
+    Ok(index.main.put_stop_words_fst(writer, &stop_words_fst)?)
+}
+
+pub fn apply_synonyms_update(
+    writer: &mut heed::RwTxn<MainT>,
+    index: &store::Index,
+    synonyms: BTreeMap<String, Vec<String>>,
+) -> MResult<()> {
+
+    let main_store = index.main;
+    let synonyms_store = index.synonyms;
+
+    let mut synonyms_builder = SetBuilder::memory();
+    synonyms_store.clear(writer)?;
+    for (word, alternatives) in synonyms.clone() {
+        synonyms_builder.insert(&word)?;
+
+        let alternatives = {
+            let alternatives = SetBuf::from_dirty(alternatives);
+            let mut alternatives_builder = SetBuilder::memory();
+            alternatives_builder.extend_iter(alternatives)?;
+            let bytes = alternatives_builder.into_inner()?;
+            fst::Set::from_bytes(bytes)?
+        };
+
+        synonyms_store.put_synonyms(writer, word.as_bytes(), &alternatives)?;
+    }
+
+    let synonyms_set = synonyms_builder
+        .into_inner()
+        .and_then(fst::Set::from_bytes)?;
+
+    main_store.put_synonyms_fst(writer, &synonyms_set)?;
+
+    Ok(())
+}

meilisearch-core/src/update/stop_words_addition.rs

@@ -1,118 +0,0 @@
-use std::collections::BTreeSet;
-
-use fst::{set::OpBuilder, SetBuilder};
-
-use crate::database::{MainT, UpdateT};
-use crate::automaton::normalize_str;
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult};
-
-pub struct StopWordsAddition {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: UpdateEventsEmitter,
-    stop_words: BTreeSet<String>,
-}
-
-impl StopWordsAddition {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: UpdateEventsEmitter,
-    ) -> StopWordsAddition {
-        StopWordsAddition {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            stop_words: BTreeSet::new(),
-        }
-    }
-
-    pub fn add_stop_word<S: AsRef<str>>(&mut self, stop_word: S) {
-        let stop_word = normalize_str(stop_word.as_ref());
-        self.stop_words.insert(stop_word);
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        let update_id = push_stop_words_addition(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.stop_words,
-        )?;
-        Ok(update_id)
-    }
-}
-
-pub fn push_stop_words_addition(
-    writer: &mut heed::RwTxn<UpdateT>,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    addition: BTreeSet<String>,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::stop_words_addition(addition);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_stop_words_addition(
-    writer: &mut heed::RwTxn<MainT>,
-    main_store: store::Main,
-    postings_lists_store: store::PostingsLists,
-    addition: BTreeSet<String>,
-) -> MResult<()> {
-    let mut stop_words_builder = SetBuilder::memory();
-
-    for word in addition {
-        stop_words_builder.insert(&word).unwrap();
-        // we remove every posting list associated to a new stop word
-        postings_lists_store.del_postings_list(writer, word.as_bytes())?;
-    }
-
-    // create the new delta stop words fst
-    let delta_stop_words = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    // we also need to remove all the stop words from the main fst
-    if let Some(word_fst) = main_store.words_fst(writer)? {
-        let op = OpBuilder::new()
-            .add(&word_fst)
-            .add(&delta_stop_words)
-            .difference();
-
-        let mut word_fst_builder = SetBuilder::memory();
-        word_fst_builder.extend_stream(op).unwrap();
-        let word_fst = word_fst_builder
-            .into_inner()
-            .and_then(fst::Set::from_bytes)
-            .unwrap();
-
-        main_store.put_words_fst(writer, &word_fst)?;
-    }
-
-    // now we add all of these stop words from the main store
-    let stop_words_fst = main_store.stop_words_fst(writer)?.unwrap_or_default();
-
-    let op = OpBuilder::new()
-        .add(&stop_words_fst)
-        .add(&delta_stop_words)
-        .r#union();
-
-    let mut stop_words_builder = SetBuilder::memory();
-    stop_words_builder.extend_stream(op).unwrap();
-    let stop_words_fst = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    main_store.put_stop_words_fst(writer, &stop_words_fst)?;
-
-    Ok(())
-}

meilisearch-core/src/update/stop_words_deletion.rs

@@ -1,114 +0,0 @@
-use std::collections::BTreeSet;
-
-use fst::{set::OpBuilder, SetBuilder};
-
-use crate::database::{MainT, UpdateT};
-use crate::automaton::normalize_str;
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::update::documents_addition::reindex_all_documents;
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult};
-
-pub struct StopWordsDeletion {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: UpdateEventsEmitter,
-    stop_words: BTreeSet<String>,
-}
-
-impl StopWordsDeletion {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: UpdateEventsEmitter,
-    ) -> StopWordsDeletion {
-        StopWordsDeletion {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            stop_words: BTreeSet::new(),
-        }
-    }
-
-    pub fn delete_stop_word<S: AsRef<str>>(&mut self, stop_word: S) {
-        let stop_word = normalize_str(stop_word.as_ref());
-        self.stop_words.insert(stop_word);
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        let update_id = push_stop_words_deletion(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.stop_words,
-        )?;
-        Ok(update_id)
-    }
-}
-
-pub fn push_stop_words_deletion(
-    writer: &mut heed::RwTxn<UpdateT>,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    deletion: BTreeSet<String>,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::stop_words_deletion(deletion);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_stop_words_deletion(
-    writer: &mut heed::RwTxn<MainT>,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-    deletion: BTreeSet<String>,
-) -> MResult<()> {
-    let mut stop_words_builder = SetBuilder::memory();
-
-    for word in deletion {
-        stop_words_builder.insert(&word).unwrap();
-    }
-
-    // create the new delta stop words fst
-    let delta_stop_words = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    // now we delete all of these stop words from the main store
-    let stop_words_fst = main_store.stop_words_fst(writer)?.unwrap_or_default();
-
-    let op = OpBuilder::new()
-        .add(&stop_words_fst)
-        .add(&delta_stop_words)
-        .difference();
-
-    let mut stop_words_builder = SetBuilder::memory();
-    stop_words_builder.extend_stream(op).unwrap();
-    let stop_words_fst = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    main_store.put_stop_words_fst(writer, &stop_words_fst)?;
-
-    // now that we have setup the stop words
-    // lets reindex everything...
-    reindex_all_documents(
-        writer,
-        main_store,
-        documents_fields_store,
-        documents_fields_counts_store,
-        postings_lists_store,
-        docs_words_store,
-    )?;
-
-    Ok(())
-}

meilisearch-core/src/update/synonyms_addition.rs

@@ -1,120 +0,0 @@
-use std::collections::BTreeMap;
-
-use fst::{set::OpBuilder, SetBuilder};
-use sdset::SetBuf;
-
-use crate::database::{MainT, UpdateT};
-use crate::automaton::normalize_str;
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult};
-
-pub struct SynonymsAddition {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: UpdateEventsEmitter,
-    synonyms: BTreeMap<String, Vec<String>>,
-}
-
-impl SynonymsAddition {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: UpdateEventsEmitter,
-    ) -> SynonymsAddition {
-        SynonymsAddition {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            synonyms: BTreeMap::new(),
-        }
-    }
-
-    pub fn add_synonym<S, T, I>(&mut self, synonym: S, alternatives: I)
-    where
-        S: AsRef<str>,
-        T: AsRef<str>,
-        I: IntoIterator<Item = T>,
-    {
-        let synonym = normalize_str(synonym.as_ref());
-        let alternatives = alternatives.into_iter().map(|s| s.as_ref().to_lowercase());
-        self.synonyms
-            .entry(synonym)
-            .or_insert_with(Vec::new)
-            .extend(alternatives);
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        let update_id = push_synonyms_addition(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.synonyms,
-        )?;
-        Ok(update_id)
-    }
-}
-
-pub fn push_synonyms_addition(
-    writer: &mut heed::RwTxn<UpdateT>,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    addition: BTreeMap<String, Vec<String>>,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::synonyms_addition(addition);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_synonyms_addition(
-    writer: &mut heed::RwTxn<MainT>,
-    main_store: store::Main,
-    synonyms_store: store::Synonyms,
-    addition: BTreeMap<String, Vec<String>>,
-) -> MResult<()> {
-    let mut synonyms_builder = SetBuilder::memory();
-
-    for (word, alternatives) in addition {
-        synonyms_builder.insert(&word).unwrap();
-
-        let alternatives = {
-            let alternatives = SetBuf::from_dirty(alternatives);
-            let mut alternatives_builder = SetBuilder::memory();
-            alternatives_builder.extend_iter(alternatives).unwrap();
-            let bytes = alternatives_builder.into_inner().unwrap();
-            fst::Set::from_bytes(bytes).unwrap()
-        };
-
-        synonyms_store.put_synonyms(writer, word.as_bytes(), &alternatives)?;
-    }
-
-    let delta_synonyms = synonyms_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    let synonyms = match main_store.synonyms_fst(writer)? {
-        Some(synonyms) => {
-            let op = OpBuilder::new()
-                .add(synonyms.stream())
-                .add(delta_synonyms.stream())
-                .r#union();
-
-            let mut synonyms_builder = SetBuilder::memory();
-            synonyms_builder.extend_stream(op).unwrap();
-            synonyms_builder
-                .into_inner()
-                .and_then(fst::Set::from_bytes)
-                .unwrap()
-        }
-        None => delta_synonyms,
-    };
-
-    main_store.put_synonyms_fst(writer, &synonyms)?;
-
-    Ok(())
-}

meilisearch-core/src/update/synonyms_deletion.rs

@@ -1,158 +0,0 @@
-use std::collections::BTreeMap;
-use std::iter::FromIterator;
-
-use fst::{set::OpBuilder, SetBuilder};
-use sdset::SetBuf;
-
-use crate::database::{MainT, UpdateT};
-use crate::automaton::normalize_str;
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult};
-
-pub struct SynonymsDeletion {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: UpdateEventsEmitter,
-    synonyms: BTreeMap<String, Option<Vec<String>>>,
-}
-
-impl SynonymsDeletion {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: UpdateEventsEmitter,
-    ) -> SynonymsDeletion {
-        SynonymsDeletion {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            synonyms: BTreeMap::new(),
-        }
-    }
-
-    pub fn delete_all_alternatives_of<S: AsRef<str>>(&mut self, synonym: S) {
-        let synonym = normalize_str(synonym.as_ref());
-        self.synonyms.insert(synonym, None);
-    }
-
-    pub fn delete_specific_alternatives_of<S, T, I>(&mut self, synonym: S, alternatives: I)
-    where
-        S: AsRef<str>,
-        T: AsRef<str>,
-        I: Iterator<Item = T>,
-    {
-        let synonym = normalize_str(synonym.as_ref());
-        let value = self.synonyms.entry(synonym).or_insert(None);
-        let alternatives = alternatives.map(|s| s.as_ref().to_lowercase());
-        match value {
-            Some(v) => v.extend(alternatives),
-            None => *value = Some(Vec::from_iter(alternatives)),
-        }
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        let update_id = push_synonyms_deletion(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.synonyms,
-        )?;
-        Ok(update_id)
-    }
-}
-
-pub fn push_synonyms_deletion(
-    writer: &mut heed::RwTxn<UpdateT>,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    deletion: BTreeMap<String, Option<Vec<String>>>,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::synonyms_deletion(deletion);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_synonyms_deletion(
-    writer: &mut heed::RwTxn<MainT>,
-    main_store: store::Main,
-    synonyms_store: store::Synonyms,
-    deletion: BTreeMap<String, Option<Vec<String>>>,
-) -> MResult<()> {
-    let mut delete_whole_synonym_builder = SetBuilder::memory();
-
-    for (synonym, alternatives) in deletion {
-        match alternatives {
-            Some(alternatives) => {
-                let prev_alternatives = synonyms_store.synonyms(writer, synonym.as_bytes())?;
-                let prev_alternatives = match prev_alternatives {
-                    Some(alternatives) => alternatives,
-                    None => continue,
-                };
-
-                let delta_alternatives = {
-                    let alternatives = SetBuf::from_dirty(alternatives);
-                    let mut builder = SetBuilder::memory();
-                    builder.extend_iter(alternatives).unwrap();
-                    builder.into_inner().and_then(fst::Set::from_bytes).unwrap()
-                };
-
-                let op = OpBuilder::new()
-                    .add(prev_alternatives.stream())
-                    .add(delta_alternatives.stream())
-                    .difference();
-
-                let (alternatives, empty_alternatives) = {
-                    let mut builder = SetBuilder::memory();
-                    let len = builder.get_ref().len();
-                    builder.extend_stream(op).unwrap();
-                    let is_empty = len == builder.get_ref().len();
-                    let bytes = builder.into_inner().unwrap();
-                    let alternatives = fst::Set::from_bytes(bytes).unwrap();
-
-                    (alternatives, is_empty)
-                };
-
-                if empty_alternatives {
-                    delete_whole_synonym_builder.insert(synonym.as_bytes())?;
-                } else {
-                    synonyms_store.put_synonyms(writer, synonym.as_bytes(), &alternatives)?;
-                }
-            }
-            None => {
-                delete_whole_synonym_builder.insert(&synonym).unwrap();
-                synonyms_store.del_synonyms(writer, synonym.as_bytes())?;
-            }
-        }
-    }
-
-    let delta_synonyms = delete_whole_synonym_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    let synonyms = match main_store.synonyms_fst(writer)? {
-        Some(synonyms) => {
-            let op = OpBuilder::new()
-                .add(synonyms.stream())
-                .add(delta_synonyms.stream())
-                .difference();
-
-            let mut synonyms_builder = SetBuilder::memory();
-            synonyms_builder.extend_stream(op).unwrap();
-            synonyms_builder
-                .into_inner()
-                .and_then(fst::Set::from_bytes)
-                .unwrap()
-        }
-        None => fst::Set::default(),
-    };
-
-    main_store.put_synonyms_fst(writer, &synonyms)?;
-
-    Ok(())
-}

meilisearch-http/Cargo.toml

@@ -1,6 +1,8 @@
 [package]
 name = "meilisearch-http"
-version = "0.8.1"
+description = "MeiliSearch HTTP server"
+version = "0.10.0"
+license = "MIT"
 authors = [
     "Quentin de Quelen <quentin@dequelen.me>",
     "Clément Renault <clement@meilisearch.com>",
@@ -12,53 +14,47 @@ name = "meilisearch"
 path = "src/main.rs"
 
 [dependencies]
-bincode = "1.2.0"
-chrono = { version = "0.4.9", features = ["serde"] }
-crossbeam-channel = "0.4.0"
+async-std = { version = "1.5.0", features = ["attributes"] }
+chrono = { version = "0.4.11", features = ["serde"] }
+crossbeam-channel = "0.4.2"
 env_logger = "0.7.1"
-heed = "0.6.0"
+futures = "0.3.4"
+heed = "0.7.0"
 http = "0.1.19"
-indexmap = { version = "1.3.0", features = ["serde-1"] }
-isahc = "0.7.6"
+indexmap = { version = "1.3.2", features = ["serde-1"] }
 log = "0.4.8"
 main_error = "0.1.0"
-meilisearch-core = { path = "../meilisearch-core", version = "0.8.1" }
-meilisearch-schema = { path = "../meilisearch-schema", version = "0.8.1" }
+meilisearch-core = { path = "../meilisearch-core", version = "0.10.0" }
+meilisearch-schema = { path = "../meilisearch-schema", version = "0.10.0" }
+meilisearch-tokenizer = {path = "../meilisearch-tokenizer", version = "0.10.0"}
+mime = "0.3.16"
 pretty-bytes = "0.2.2"
-rand = "0.7.2"
-rayon = "1.2.0"
-serde = { version = "1.0.101", features = ["derive"] }
-serde_json = { version = "1.0.41", features = ["preserve_order"] }
-serde_qs = "0.5.1"
-siphasher = "0.3.1"
-structopt = "0.3.3"
-sysinfo = "0.9.5"
-walkdir = "2.2.9"
-whoami = "0.6"
+rand = "0.7.3"
+rayon = "1.3.0"
+serde = { version = "1.0.105", features = ["derive"] }
+serde_json = { version = "1.0.50", features = ["preserve_order"] }
+serde_qs = "0.5.2"
+sha2 = "0.8.1"
+siphasher = "0.3.2"
+structopt = "0.3.12"
+sysinfo = "0.12.0"
+tide = "0.6.0"
+ureq = { version = "0.12.0", features = ["tls"], default-features = false }
+walkdir = "2.3.1"
+whoami = "0.8.1"
 
-[dependencies.async-compression]
-default-features = false
-features = ["stream", "gzip", "zlib", "brotli", "zstd"]
-version = "=0.1.0-alpha.7"
+[dev-dependencies]
+http-service = "0.4.0"
+http-service-mock = "0.4.0"
+tempdir = "0.3.7"
+once_cell = "1.3.1"
 
-[dependencies.tide]
-git = "https://github.com/rustasync/tide"
-rev = "e77709370bb24cf776fe6da902467c35131535b1"
-
-[dependencies.tide-log]
-git = "https://github.com/rustasync/tide"
-rev = "e77709370bb24cf776fe6da902467c35131535b1"
-
-[dependencies.tide-slog]
-git = "https://github.com/rustasync/tide"
-rev = "e77709370bb24cf776fe6da902467c35131535b1"
-
-[dependencies.tide-compression]
-git = "https://github.com/rustasync/tide"
-rev = "e77709370bb24cf776fe6da902467c35131535b1"
+[dev-dependencies.assert-json-diff]
+git = "https://github.com/qdequele/assert-json-diff"
+branch = "master"
 
 [build-dependencies]
-vergen = "3.0.4"
+vergen = "3.1.0"
 
 [target.'cfg(unix)'.dependencies]
 jemallocator = "0.3.2"

meilisearch-http/public/interface.html

@@ -0,0 +1,269 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link rel="stylesheet" href="/bulma.min.css">
+    <title>MeiliSearch</title>
+    <style>
+      em {
+        color: hsl(204, 86%, 25%);
+        font-style: inherit;
+        background-color: hsl(204, 86%, 88%);
+      }
+
+      #results {
+        max-width: 900px;
+        margin: 20px auto 0 auto;
+        padding: 0;
+      }
+
+      .notification {
+        display: flex;
+        justify-content: center;
+      }
+
+      .level-left {
+        margin-right: 50px;
+      }
+
+      .document {
+        padding: 20px 20px;
+        background-color: #f5f5f5;
+        border-radius: 4px;
+        margin-bottom: 20px;
+        display: flex;
+      }
+
+      .document ol {
+        flex: 0 0 75%;
+        max-width: 75%;
+        padding: 0;
+        margin: 0;
+      }
+
+      .document .image {
+        max-width: 25%;
+        flex: 0 0 25%;
+        padding-left: 30px;
+        box-sizing: border-box;
+      }
+
+      .document .image img {
+        width: 100%;
+      }
+
+      .field {
+        list-style-type: none;
+        display: flex;
+        flex-wrap: wrap;
+      }
+
+      .field:not(:last-child) {
+        margin-bottom: 7px;
+      }
+
+      .attribute {
+        flex: 0 0 25%;
+        max-width: 25%;
+        text-align: right;
+        padding-right: 10px;
+        box-sizing: border-box;
+        text-transform: uppercase;
+        color: rgba(0,0,0,.7);
+      }
+
+      .content {
+        max-width: 75%;
+        flex: 0 0 75%;
+        box-sizing: border-box;
+        padding-left: 10px;
+        color: rgba(0,0,0,.9);
+      }
+    </style>
+  </head>
+  <body>
+
+    <section class="hero is-light">
+      <div class="hero-body">
+        <div class="container">
+          <h1 class="title">
+            Welcome to MeiliSearch
+          </h1>
+          <h2 class="subtitle">
+            This dashboard will help you check the search results with ease.
+          </h2>
+        </div>
+      </div>
+    </section>
+
+    <section class="hero container">
+        <div class="notification" style="border-radius: 0 0 4px 4px;">
+
+          <nav class="level">
+            <!-- Left side -->
+            <div class="level-left">
+              <div class="level-item">
+                <div class="field has-addons has-addons-right">
+                  <p class="control">
+                    <span class="select">
+                      <select id="index">
+                        <!-- indexes names -->
+                      </select>
+                    </span>
+                  </p>
+                  <p class="control">
+                    <input id="search" class="input" type="text" autofocus placeholder="e.g. George Clooney">
+                  </p>
+                </div>
+              </div>
+            </div>
+
+            <!-- Right side -->
+            <nav class="level-right">
+              <div class="level-item has-text-centered">
+                <div>
+                  <p class="heading">Documents</p>
+                  <p id="count" class="title">25</p>
+                </div>
+              </div>
+              <div class="level-item has-text-centered">
+                <div>
+                  <p class="heading">Time Spent</p>
+                  <p id="time" class="title">4ms</p>
+                </div>
+              </div>
+            </nav>
+          </nav>
+
+        </div>
+    </section>
+
+    <section>
+      <ol id="results" class="content">
+        <!-- documents matching resquests -->
+      </ol>
+    </section>
+  </body>
+
+  <script>
+    function sanitizeHTMLEntities(str) {
+        if (str && typeof str === 'string') {
+            str = str.replace(/</g,"&lt;");
+            str = str.replace(/>/g,"&gt;");
+            str = str.replace(/&lt;em&gt;/g,"<em>");
+            str = str.replace(/&lt;\/em&gt;/g,"<\/em>");
+        }
+        return str;
+    }
+
+    function httpGet(theUrl) {
+        var xmlHttp = new XMLHttpRequest();
+        xmlHttp.open("GET", theUrl, false); // false for synchronous request
+        xmlHttp.send(null);
+        return xmlHttp.responseText;
+    }
+
+    let lastRequest = undefined;
+
+    function triggerSearch() {
+        var e = document.getElementById("index");
+        if (e.selectedIndex == -1) { return }
+        var index = e.options[e.selectedIndex].value;
+
+        let theUrl = `${baseUrl}/indexes/${index}/search?q=${search.value}&attributesToHighlight=*`;
+
+        if (lastRequest) { lastRequest.abort() }
+        lastRequest = new XMLHttpRequest();
+
+        lastRequest.open("GET", theUrl, true);
+        lastRequest.onload = function (e) {
+            if (lastRequest.readyState === 4 && lastRequest.status === 200) {
+                let sanitizedResponseText = sanitizeHTMLEntities(lastRequest.responseText);
+                let httpResults = JSON.parse(sanitizedResponseText);
+                results.innerHTML = '';
+
+                let processingTimeMs = httpResults.processingTimeMs;
+                let numberOfDocuments = httpResults.hits.length;
+                time.innerHTML = `${processingTimeMs}ms`;
+                count.innerHTML = `${numberOfDocuments}`;
+
+                for (result of httpResults.hits) {
+                    const element = {...result, ...result._formatted };
+                    delete element._formatted;
+
+                    const elem = document.createElement('li');
+                    elem.classList.add("document");
+
+                    const ol = document.createElement('ol');
+                    let image = undefined;
+
+                    for (const prop in element) {
+                        // Check if property is an image url link.
+                        if (typeof result[prop] === 'string') {
+                            if (image == undefined && result[prop].match(/^(https|http):\/\/.*(jpe?g|png|gif)(\?.*)?$/g)) {
+                                image = result[prop];
+                            }
+                        }
+
+                        const field = document.createElement('li');
+                        field.classList.add("field");
+
+                        const attribute = document.createElement('div');
+                        attribute.classList.add("attribute");
+                        attribute.innerHTML = prop;
+
+                        const content = document.createElement('div');
+                        content.classList.add("content");
+                        if (typeof (element[prop]) === "object") {
+                          content.innerHTML = JSON.stringify(element[prop]);
+                        } else {
+                          content.innerHTML = element[prop];
+                        }
+
+                        field.appendChild(attribute);
+                        field.appendChild(content);
+
+                        ol.appendChild(field);
+                    }
+
+                    elem.appendChild(ol);
+
+                    if (image != undefined) {
+                        const div = document.createElement('div');
+                        div.classList.add("image");
+
+                        const img = document.createElement('img');
+                        img.src = image;
+
+                        div.appendChild(img);
+                        elem.appendChild(div);
+                    }
+
+                    results.appendChild(elem)
+                }
+            } else {
+                console.error(lastRequest.statusText);
+            }
+        };
+        lastRequest.send(null);
+    }
+
+    let baseUrl = window.location.origin;
+    // TODO we must not block here
+    let result = JSON.parse(httpGet(`${baseUrl}/indexes`));
+
+    let select = document.getElementById("index");
+    for (index of result) {
+        const option = document.createElement('option');
+        option.value = index.uid;
+        option.innerHTML = index.name;
+        select.appendChild(option);
+    }
+
+    search.oninput = triggerSearch;
+    select.onchange = triggerSearch;
+
+    triggerSearch();
+  </script>
+</html>

meilisearch-http/src/analytics.rs

@@ -58,14 +58,10 @@ pub fn analytics_sender() {
         };
 
         let body = qs::to_string(&request).unwrap();
-        match isahc::post("https://api.amplitude.com/httpapi", body) {
-            Ok(response) => {
-                if !response.status().is_success() {
-                    let body = response.into_body().text().unwrap();
-                    error!("Unsuccessful call to Amplitude: {}", body);
-                }
-            }
-            Err(e) => error!("Error while sending a request to Amplitude: {}", e),
+        let response = ureq::post("https://api.amplitude.com/httpapi").send_string(&body);
+        if !response.ok() {
+            let body = response.into_string().unwrap();
+            error!("Unsuccessful call to Amplitude: {}", body);
         }
 
         thread::sleep(Duration::from_secs(86_400)) // one day

meilisearch-http/src/data.rs

@@ -5,7 +5,8 @@ use std::sync::Arc;
 use chrono::{DateTime, Utc};
 use heed::types::{SerdeBincode, Str};
 use log::error;
-use meilisearch_core::{Database, MainT, UpdateT, Error as MError, MResult};
+use meilisearch_core::{Database, Error as MError, MResult, MainT, UpdateT};
+use sha2::Digest;
 use sysinfo::Pid;
 
 use crate::option::Opt;
@@ -32,10 +33,34 @@ impl Deref for Data {
 pub struct DataInner {
     pub db: Arc<Database>,
     pub db_path: String,
-    pub api_key: Option<String>,
+    pub api_keys: ApiKeys,
     pub server_pid: Pid,
 }
 
+#[derive(Default, Clone)]
+pub struct ApiKeys {
+    pub public: Option<String>,
+    pub private: Option<String>,
+    pub master: Option<String>,
+}
+
+impl ApiKeys {
+    pub fn generate_missing_api_keys(&mut self) {
+        if let Some(master_key) = &self.master {
+            if self.private.is_none() {
+                let key = format!("{}-private", master_key);
+                let sha = sha2::Sha256::digest(key.as_bytes());
+                self.private = Some(format!("{:x}", sha));
+            }
+            if self.public.is_none() {
+                let key = format!("{}-public", master_key);
+                let sha = sha2::Sha256::digest(key.as_bytes());
+                self.public = Some(format!("{:x}", sha));
+            }
+        }
+    }
+}
+
 impl DataInner {
     pub fn is_indexing(&self, reader: &heed::RoTxn<UpdateT>, index: &str) -> MResult<Option<bool>> {
         match self.db.open_index(&index) {
@@ -84,13 +109,15 @@ impl DataInner {
         let mut fields_frequency = HashMap::<_, usize>::new();
         for result in all_documents_fields {
             let (_, attr, _) = result?;
-            *fields_frequency.entry(attr).or_default() += 1;
+            if let Some(field_id) = schema.indexed_pos_to_field_id(attr) {
+                *fields_frequency.entry(field_id).or_default() += 1;
+            }
         }
 
         // convert attributes to their names
         let frequency: HashMap<_, _> = fields_frequency
             .into_iter()
-            .map(|(a, c)| (schema.attribute_name(a).to_owned(), c))
+            .filter_map(|(a, c)| schema.name(a).map(|name| (name.to_string(), c)))
             .collect();
 
         index
@@ -103,15 +130,22 @@ impl DataInner {
 impl Data {
     pub fn new(opt: Opt) -> Data {
         let db_path = opt.db_path.clone();
-        let api_key = opt.api_key.clone();
         let server_pid = sysinfo::get_current_pid().unwrap();
 
-        let db = Arc::new(Database::open_or_create(opt.db_path.clone()).unwrap());
+        let db = Arc::new(Database::open_or_create(opt.db_path).unwrap());
+
+        let mut api_keys = ApiKeys {
+            master: opt.master_key.clone(),
+            private: None,
+            public: None,
+        };
+
+        api_keys.generate_missing_api_keys();
 
         let inner_data = DataInner {
             db: db.clone(),
             db_path,
-            api_key,
+            api_keys,
             server_pid,
         };
 

meilisearch-http/src/error.rs

@@ -2,10 +2,13 @@ use std::fmt::Display;
 
 use http::status::StatusCode;
 use log::{error, warn};
+use meilisearch_core::{FstError, HeedError};
 use serde::{Deserialize, Serialize};
-use tide::response::IntoResponse;
+use tide::IntoResponse;
 use tide::Response;
 
+use crate::helpers::meilisearch::Error as SearchError;
+
 pub type SResult<T> = Result<T, ResponseError>;
 
 pub enum ResponseError {
@@ -16,9 +19,11 @@ pub enum ResponseError {
     IndexNotFound(String),
     DocumentNotFound(String),
     MissingHeader(String),
+    FilterParsing(String),
     BadParameter(String, String),
     OpenIndex(String),
     CreateIndex(String),
+    InvalidIndexUid,
     Maintenance,
 }
 
@@ -69,17 +74,21 @@ impl IntoResponse for ResponseError {
         match self {
             ResponseError::Internal(err) => {
                 error!("internal server error: {}", err);
-                error(
-                    String::from("Internal server error"),
+                error("Internal server error".to_string(),
                     StatusCode::INTERNAL_SERVER_ERROR,
                 )
             }
+            ResponseError::FilterParsing(err) => {
+                warn!("error paring filter: {}", err);
+                error(format!("parsing error: {}", err),
+                StatusCode::BAD_REQUEST)
+            }
             ResponseError::BadRequest(err) => {
                 warn!("bad request: {}", err);
                 error(err, StatusCode::BAD_REQUEST)
             }
             ResponseError::InvalidToken(err) => {
-                error(format!("Invalid Token: {}", err), StatusCode::FORBIDDEN)
+                error(format!("Invalid API key: {}", err), StatusCode::FORBIDDEN)
             }
             ResponseError::NotFound(err) => error(err, StatusCode::NOT_FOUND),
             ResponseError::IndexNotFound(index) => {
@@ -105,6 +114,10 @@ impl IntoResponse for ResponseError {
                 format!("Impossible to open index; {}", err),
                 StatusCode::BAD_REQUEST,
             ),
+            ResponseError::InvalidIndexUid => error(
+                "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_).".to_string(),
+                StatusCode::BAD_REQUEST,
+            ),
             ResponseError::Maintenance => error(
                 String::from("Server is in maintenance, please try again later"),
                 StatusCode::SERVICE_UNAVAILABLE,
@@ -120,7 +133,59 @@ struct ErrorMessage {
 
 fn error(message: String, status: StatusCode) -> Response {
     let message = ErrorMessage { message };
-    tide::response::json(message)
-        .with_status(status)
-        .into_response()
+    tide::Response::new(status.as_u16())
+        .body_json(&message)
+        .unwrap()
+}
+
+impl From<serde_json::Error> for ResponseError {
+    fn from(err: serde_json::Error) -> ResponseError {
+        ResponseError::internal(err)
+    }
+}
+
+impl From<meilisearch_core::Error> for ResponseError {
+    fn from(err: meilisearch_core::Error) -> ResponseError {
+        ResponseError::internal(err)
+    }
+}
+
+impl From<HeedError> for ResponseError {
+    fn from(err: HeedError) -> ResponseError {
+        ResponseError::internal(err)
+    }
+}
+
+impl From<FstError> for ResponseError {
+    fn from(err: FstError) -> ResponseError {
+        ResponseError::internal(err)
+    }
+}
+
+impl From<SearchError> for ResponseError {
+    fn from(err: SearchError) -> ResponseError {
+        match err {
+            SearchError::FilterParsing(s) => ResponseError::FilterParsing(s),
+            _ => ResponseError::internal(err),
+        }
+    }
+}
+
+impl From<meilisearch_core::settings::RankingRuleConversionError> for ResponseError {
+    fn from(err: meilisearch_core::settings::RankingRuleConversionError) -> ResponseError {
+        ResponseError::internal(err)
+    }
+}
+
+pub trait IntoInternalError<T> {
+    fn into_internal_error(self) -> SResult<T>;
+}
+
+impl<T> IntoInternalError<T> for Option<T> {
+    fn into_internal_error(self) -> SResult<T> {
+        match self {
+            Some(value) => Ok(value),
+            None => Err(ResponseError::internal("Heed cannot find requested value")),
+        }
+    }
 }

meilisearch-http/src/helpers/meilisearch.rs

@@ -1,26 +1,30 @@
-use crate::routes::setting::{RankingOrdering, SettingBody};
-use indexmap::IndexMap;
-use log::error;
-use meilisearch_core::criterion::*;
-use meilisearch_core::Highlight;
-use meilisearch_core::{Index, RankedMap};
-use meilisearch_core::MainT;
-use meilisearch_schema::{Schema, SchemaAttr};
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
 use std::cmp::Ordering;
 use std::collections::{HashMap, HashSet};
 use std::convert::From;
 use std::error;
 use std::fmt;
+use std::hash::{Hash, Hasher};
 use std::time::{Duration, Instant};
 
+use indexmap::IndexMap;
+use log::error;
+use meilisearch_core::Filter;
+use meilisearch_core::criterion::*;
+use meilisearch_core::settings::RankingRule;
+use meilisearch_core::{Highlight, Index, MainT, RankedMap};
+use meilisearch_schema::{FieldId, Schema};
+use meilisearch_tokenizer::is_cjk;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use siphasher::sip::SipHasher;
+
 #[derive(Debug)]
 pub enum Error {
     SearchDocuments(String),
     RetrieveDocument(u64, String),
     DocumentNotFound(u64),
     CropFieldWrongType(String),
+    FilterParsing(String),
     AttributeNotFoundOnDocument(String),
     AttributeNotFoundOnSchema(String),
     MissingFilterValue,
@@ -54,12 +58,31 @@ impl fmt::Display for Error {
                 f.write_str("a filter is specifying an unknown schema attribute")
             }
             Internal(err) => write!(f, "internal error; {}", err),
+            FilterParsing(err) => write!(f, "filter parsing error: {}", err),
         }
     }
 }
 
 impl From<meilisearch_core::Error> for Error {
     fn from(error: meilisearch_core::Error) -> Self {
+        use meilisearch_core::pest_error::LineColLocation::*;
+        match error {
+            meilisearch_core::Error::FilterParseError(e) => {
+                let (line, column) = match e.line_col {
+                    Span((line, _), (column, _)) => (line, column),
+                    Pos((line, column)) => (line, column),
+                };
+                let message = format!("parsing error on line {} at column {}: {}", line, column, e.variant.message());
+
+                Error::FilterParsing(message) 
+            },
+            _ => Error::Internal(error.to_string()),
+        }
+    }
+}
+
+impl From<heed::Error> for Error {
+    fn from(error: heed::Error) -> Self {
         Error::Internal(error.to_string())
     }
 }
@@ -77,7 +100,6 @@ impl IndexSearchExt for Index {
             limit: 20,
             attributes_to_crop: None,
             attributes_to_retrieve: None,
-            attributes_to_search_in: None,
             attributes_to_highlight: None,
             filters: None,
             timeout: Duration::from_millis(30),
@@ -93,7 +115,6 @@ pub struct SearchBuilder<'a> {
     limit: usize,
     attributes_to_crop: Option<HashMap<String, usize>>,
     attributes_to_retrieve: Option<HashSet<String>>,
-    attributes_to_search_in: Option<HashSet<String>>,
     attributes_to_highlight: Option<HashSet<String>>,
     filters: Option<String>,
     timeout: Duration,
@@ -127,17 +148,6 @@ impl<'a> SearchBuilder<'a> {
         self
     }
 
-    pub fn attributes_to_search_in(&mut self, value: HashSet<String>) -> &SearchBuilder {
-        self.attributes_to_search_in = Some(value);
-        self
-    }
-
-    pub fn add_attribute_to_search_in(&mut self, value: String) -> &SearchBuilder {
-        let attributes_to_search_in = self.attributes_to_search_in.get_or_insert(HashSet::new());
-        attributes_to_search_in.insert(value);
-        self
-    }
-
     pub fn attributes_to_highlight(&mut self, value: HashSet<String>) -> &SearchBuilder {
         self.attributes_to_highlight = Some(value);
         self
@@ -170,80 +180,91 @@ impl<'a> SearchBuilder<'a> {
         let ranked_map = ranked_map.map_err(|e| Error::Internal(e.to_string()))?;
         let ranked_map = ranked_map.unwrap_or_default();
 
-        let start = Instant::now();
-
         // Change criteria
         let mut query_builder = match self.get_criteria(reader, &ranked_map, &schema)? {
             Some(criteria) => self.index.query_builder_with_criteria(criteria),
             None => self.index.query_builder(),
         };
 
-        // Filter searchable fields
-        if let Some(fields) = &self.attributes_to_search_in {
-            for attribute in fields.iter().filter_map(|f| schema.attribute(f)) {
-                query_builder.add_searchable_attribute(attribute.0);
-            }
-        }
-
-        if let Some(filters) = &self.filters {
-            let mut split = filters.split(':');
-            match (split.next(), split.next()) {
-                (Some(_), None) | (Some(_), Some("")) => return Err(Error::MissingFilterValue),
-                (Some(attr), Some(value)) => {
-                    let ref_reader = reader;
-                    let ref_index = &self.index;
-                    let value = value.trim().to_lowercase();
-
-                    let attr = match schema.attribute(attr) {
-                        Some(attr) => attr,
-                        None => return Err(Error::UnknownFilteredAttribute),
-                    };
-
-                    query_builder.with_filter(move |id| {
-                        let attr = attr;
-                        let index = ref_index;
-                        let reader = ref_reader;
-
-                        match index.document_attribute::<Value>(reader, id, attr) {
-                            Ok(Some(Value::String(s))) => s.to_lowercase() == value,
-                            Ok(Some(Value::Bool(b))) => {
-                                (value == "true" && b) || (value == "false" && !b)
-                            }
-                            Ok(Some(Value::Array(a))) => {
-                                a.into_iter().any(|s| s.as_str() == Some(&value))
-                            }
-                            _ => false,
-                        }
-                    });
+        if let Some(filter_expression) = &self.filters {
+            let filter = Filter::parse(filter_expression, &schema)?;
+            query_builder.with_filter(move |id| {
+                let index = &self.index;
+                let reader = &reader;
+                let filter = &filter;
+                match filter.test(reader, index, id) {
+                    Ok(res) => res,
+                    Err(e) => {
+                        log::warn!("unexpected error during filtering: {}", e);
+                        false
+                    }
                 }
-                (_, _) => (),
-            }
+            });
         }
 
         query_builder.with_fetch_timeout(self.timeout);
 
-        let docs =
+        if let Some(field) = self.index.main.distinct_attribute(reader)? {
+            if let Some(field_id) = schema.id(&field) {
+                query_builder.with_distinct(1, move |id| {
+                    match self.index.document_attribute_bytes(reader, id, field_id) {
+                        Ok(Some(bytes)) => {
+                            let mut s = SipHasher::new();
+                            bytes.hash(&mut s);
+                            Some(s.finish())
+                        }
+                        _ => None,
+                    }
+                });
+            }
+        }
+
+        let start = Instant::now();
+        let result =
             query_builder.query(reader, &self.query, self.offset..(self.offset + self.limit));
+        let (docs, nb_hits) = result.map_err(|e| Error::SearchDocuments(e.to_string()))?;
+        let time_ms = start.elapsed().as_millis() as usize;
+
+        let mut all_attributes: HashSet<&str> = HashSet::new();
+        let mut all_formatted: HashSet<&str> = HashSet::new();
+
+        match &self.attributes_to_retrieve {
+            Some(to_retrieve) => {
+                all_attributes.extend(to_retrieve.iter().map(String::as_str));
+
+                if let Some(to_highlight) = &self.attributes_to_highlight {
+                    all_formatted.extend(to_highlight.iter().map(String::as_str));
+                }
+
+                if let Some(to_crop) = &self.attributes_to_crop {
+                    all_formatted.extend(to_crop.keys().map(String::as_str));
+                }
+
+                all_attributes.extend(&all_formatted);
+            },
+            None => {
+                all_attributes.extend(schema.displayed_name());
+                // If we specified at least one attribute to highlight or crop then
+                // all available attributes will be returned in the _formatted field.
+                if self.attributes_to_highlight.is_some() || self.attributes_to_crop.is_some() {
+                    all_formatted.extend(all_attributes.iter().cloned());
+                }
+            },
+        }
 
         let mut hits = Vec::with_capacity(self.limit);
-        for doc in docs.map_err(|e| Error::SearchDocuments(e.to_string()))? {
-            // retrieve the content of document in kv store
-            let mut fields: Option<HashSet<&str>> = None;
-            if let Some(attributes_to_retrieve) = &self.attributes_to_retrieve {
-                let mut set = HashSet::new();
-                for field in attributes_to_retrieve {
-                    set.insert(field.as_str());
-                }
-                fields = Some(set);
-            }
-
-            let document: IndexMap<String, Value> = self
+        for doc in docs {
+            let mut document: IndexMap<String, Value> = self
                 .index
-                .document(reader, fields.as_ref(), doc.id)
+                .document(reader, Some(&all_attributes), doc.id)
                 .map_err(|e| Error::RetrieveDocument(doc.id.0, e.to_string()))?
                 .ok_or(Error::DocumentNotFound(doc.id.0))?;
 
-            let mut formatted = document.clone();
+            let mut formatted = document.iter()
+                .filter(|(key, _)| all_formatted.contains(key.as_str()))
+                .map(|(k, v)| (k.clone(), v.clone()))
+                .collect();
+
             let mut matches = doc.highlights.clone();
 
             // Crops fields if needed
@@ -252,15 +273,24 @@ impl<'a> SearchBuilder<'a> {
             }
 
             // Transform to readable matches
-            let matches = calculate_matches(matches, self.attributes_to_retrieve.clone(), &schema);
-
-            if !self.matches {
-                if let Some(attributes_to_highlight) = &self.attributes_to_highlight {
-                    formatted = calculate_highlights(&formatted, &matches, attributes_to_highlight);
-                }
+            if let Some(attributes_to_highlight) = &self.attributes_to_highlight {
+                let matches = calculate_matches(
+                    matches.clone(),
+                    self.attributes_to_highlight.clone(),
+                    &schema,
+                );
+                formatted = calculate_highlights(&formatted, &matches, attributes_to_highlight);
             }
 
-            let matches_info = if self.matches { Some(matches) } else { None };
+            let matches_info = if self.matches {
+                Some(calculate_matches(matches, self.attributes_to_retrieve.clone(), &schema))
+            } else {
+                None
+            };
+
+            if let Some(attributes_to_retrieve) = &self.attributes_to_retrieve {
+                document.retain(|key, _| attributes_to_retrieve.contains(&key.to_string()))
+            }
 
             let hit = SearchHit {
                 document,
@@ -271,12 +301,12 @@ impl<'a> SearchBuilder<'a> {
             hits.push(hit);
         }
 
-        let time_ms = start.elapsed().as_millis() as usize;
-
         let results = SearchResult {
             hits,
             offset: self.offset,
             limit: self.limit,
+            nb_hits,
+            exhaustive_nb_hits: false,
             processing_time_ms: time_ms,
             query: self.query.to_string(),
         };
@@ -290,69 +320,34 @@ impl<'a> SearchBuilder<'a> {
         ranked_map: &'a RankedMap,
         schema: &Schema,
     ) -> Result<Option<Criteria<'a>>, Error> {
-        let current_settings = match self.index.main.customs(reader).unwrap() {
-            Some(bytes) => bincode::deserialize(bytes).unwrap(),
-            None => SettingBody::default(),
-        };
-
-        let ranking_rules = &current_settings.ranking_rules;
-        let ranking_order = &current_settings.ranking_order;
+        let ranking_rules = self.index.main.ranking_rules(reader)?;
 
         if let Some(ranking_rules) = ranking_rules {
             let mut builder = CriteriaBuilder::with_capacity(7 + ranking_rules.len());
-            if let Some(ranking_rules_order) = ranking_order {
-                for rule in ranking_rules_order {
-                    match rule.as_str() {
-                        "_sum_of_typos" => builder.push(SumOfTypos),
-                        "_number_of_words" => builder.push(NumberOfWords),
-                        "_word_proximity" => builder.push(WordsProximity),
-                        "_sum_of_words_attribute" => builder.push(SumOfWordsAttribute),
-                        "_sum_of_words_position" => builder.push(SumOfWordsPosition),
-                        "_exact" => builder.push(Exact),
-                        _ => {
-                            let order = match ranking_rules.get(rule.as_str()) {
-                                Some(o) => o,
-                                None => continue,
-                            };
-
-                            let custom_ranking = match order {
-                                RankingOrdering::Asc => {
-                                    SortByAttr::lower_is_better(&ranked_map, &schema, &rule)
-                                        .unwrap()
-                                }
-                                RankingOrdering::Dsc => {
-                                    SortByAttr::higher_is_better(&ranked_map, &schema, &rule)
-                                        .unwrap()
-                                }
-                            };
-
-                            builder.push(custom_ranking);
+            for rule in ranking_rules {
+                match rule {
+                    RankingRule::Typo => builder.push(Typo),
+                    RankingRule::Words => builder.push(Words),
+                    RankingRule::Proximity => builder.push(Proximity),
+                    RankingRule::Attribute => builder.push(Attribute),
+                    RankingRule::WordsPosition => builder.push(WordsPosition),
+                    RankingRule::Exactness => builder.push(Exactness),
+                    RankingRule::Asc(field) => {
+                        match SortByAttr::lower_is_better(&ranked_map, &schema, &field) {
+                            Ok(rule) => builder.push(rule),
+                            Err(err) => error!("Error during criteria builder; {:?}", err),
+                        }
+                    }
+                    RankingRule::Desc(field) => {
+                        match SortByAttr::higher_is_better(&ranked_map, &schema, &field) {
+                            Ok(rule) => builder.push(rule),
+                            Err(err) => error!("Error during criteria builder; {:?}", err),
                         }
                     }
                 }
-                builder.push(DocumentId);
-                return Ok(Some(builder.build()));
-            } else {
-                builder.push(SumOfTypos);
-                builder.push(NumberOfWords);
-                builder.push(WordsProximity);
-                builder.push(SumOfWordsAttribute);
-                builder.push(SumOfWordsPosition);
-                builder.push(Exact);
-                for (rule, order) in ranking_rules.iter() {
-                    let custom_ranking = match order {
-                        RankingOrdering::Asc => {
-                            SortByAttr::lower_is_better(&ranked_map, &schema, &rule).unwrap()
-                        }
-                        RankingOrdering::Dsc => {
-                            SortByAttr::higher_is_better(&ranked_map, &schema, &rule).unwrap()
-                        }
-                    };
-                    builder.push(custom_ranking);
-                }
-                builder.push(DocumentId);
-                return Ok(Some(builder.build()));
             }
+            builder.push(DocumentId);
+            return Ok(Some(builder.build()));
         }
 
         Ok(None)
@@ -388,16 +383,40 @@ pub struct SearchHit {
     pub matches_info: Option<MatchesInfos>,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct SearchResult {
     pub hits: Vec<SearchHit>,
     pub offset: usize,
     pub limit: usize,
+    pub nb_hits: usize,
+    pub exhaustive_nb_hits: bool,
     pub processing_time_ms: usize,
     pub query: String,
-    // pub parsed_query: String,
-    // pub params: Option<String>,
+}
+
+/// returns the start index and the length on the crop.
+fn aligned_crop(text: &str, match_index: usize, context: usize) -> (usize, usize) {
+    let is_word_component = |c: &char| c.is_alphanumeric() && !is_cjk(*c);
+
+    let word_end_index = |mut index| {
+        if text.chars().nth(index - 1).map_or(false, |c| is_word_component(&c)) {
+            index += text.chars().skip(index).take_while(is_word_component).count();
+        }
+        index
+    };
+
+    if context == 0 {
+        // count need to be at least 1 for cjk queries to return something
+        return (match_index, 1 + text.chars().skip(match_index).take_while(is_word_component).count());
+    }
+    let start = match match_index.saturating_sub(context) {
+        n if n == 0 => n,
+        n => word_end_index(n)
+    };
+    let end = word_end_index(start + 2 * context);
+
+    (start, end - start)
 }
 
 fn crop_text(
@@ -408,9 +427,12 @@ fn crop_text(
     let mut matches = matches.into_iter().peekable();
 
     let char_index = matches.peek().map(|m| m.char_index as usize).unwrap_or(0);
-    let start = char_index.saturating_sub(context);
-    let text = text.chars().skip(start).take(context * 2).collect();
+    let (start, count) = aligned_crop(text, char_index, context);
 
+    //TODO do something about the double allocation
+    let text = text.chars().skip(start).take(count).collect::<String>().trim().to_string();
+
+    // update matches index to match the new cropped text
     let matches = matches
         .take_while(|m| (m.char_index as usize) + (m.char_length as usize) <= start + (context * 2))
         .map(|match_| Highlight {
@@ -431,14 +453,14 @@ fn crop_document(
     matches.sort_unstable_by_key(|m| (m.char_index, m.char_length));
 
     for (field, length) in fields {
-        let attribute = match schema.attribute(field) {
+        let attribute = match schema.id(field) {
             Some(attribute) => attribute,
             None => continue,
         };
 
         let selected_matches = matches
             .iter()
-            .filter(|m| SchemaAttr::new(m.attribute) == attribute)
+            .filter(|m| FieldId::new(m.attribute) == attribute)
             .cloned();
 
         if let Some(Value::String(ref mut original_text)) = document.get_mut(field) {
@@ -447,7 +469,7 @@ fn crop_document(
 
             *original_text = cropped_text;
 
-            matches.retain(|m| SchemaAttr::new(m.attribute) != attribute);
+            matches.retain(|m| FieldId::new(m.attribute) != attribute);
             matches.extend_from_slice(&cropped_matches);
         }
     }
@@ -460,26 +482,28 @@ fn calculate_matches(
 ) -> MatchesInfos {
     let mut matches_result: HashMap<String, Vec<MatchPosition>> = HashMap::new();
     for m in matches.iter() {
-        let attribute = schema
-            .attribute_name(SchemaAttr::new(m.attribute))
-            .to_string();
-        if let Some(attributes_to_retrieve) = attributes_to_retrieve.clone() {
-            if !attributes_to_retrieve.contains(attribute.as_str()) {
+        if let Some(attribute) = schema.name(FieldId::new(m.attribute)) {
+            if let Some(attributes_to_retrieve) = attributes_to_retrieve.clone() {
+                if !attributes_to_retrieve.contains(attribute) {
+                    continue;
+                }
+            }
+            if !schema.displayed_name().contains(attribute) {
                 continue;
             }
-        };
-        if let Some(pos) = matches_result.get_mut(&attribute) {
-            pos.push(MatchPosition {
-                start: m.char_index as usize,
-                length: m.char_length as usize,
-            });
-        } else {
-            let mut positions = Vec::new();
-            positions.push(MatchPosition {
-                start: m.char_index as usize,
-                length: m.char_length as usize,
-            });
-            matches_result.insert(attribute, positions);
+            if let Some(pos) = matches_result.get_mut(attribute) {
+                pos.push(MatchPosition {
+                    start: m.char_index as usize,
+                    length: m.char_length as usize,
+                });
+            } else {
+                let mut positions = Vec::new();
+                positions.push(MatchPosition {
+                    start: m.char_index as usize,
+                    length: m.char_length as usize,
+                });
+                matches_result.insert(attribute.to_string(), positions);
+            }
         }
     }
     for (_, val) in matches_result.iter_mut() {
@@ -494,7 +518,7 @@ fn calculate_highlights(
     matches: &MatchesInfos,
     attributes_to_highlight: &HashSet<String>,
 ) -> IndexMap<String, Value> {
-    let mut highlight_result = IndexMap::new();
+    let mut highlight_result = document.clone();
 
     for (attribute, matches) in matches.iter() {
         if attributes_to_highlight.contains(attribute) {
@@ -530,6 +554,39 @@ fn calculate_highlights(
 mod tests {
     use super::*;
 
+    #[test]
+    fn aligned_crops() {
+        let text = r#"En ce début de trentième millénaire, l'Empire n'a jamais été aussi puissant, aussi étendu à travers toute la galaxie. C'est dans sa capitale, Trantor, que l'éminent savant Hari Seldon invente la psychohistoire, une science toute nouvelle, à base de psychologie et de mathématiques, qui lui permet de prédire l'avenir... C'est-à-dire l'effondrement de l'Empire d'ici cinq siècles et au-delà, trente mille années de chaos et de ténèbres. Pour empêcher cette catastrophe et sauver la civilisation, Seldon crée la Fondation."#;
+
+        // simple test
+        let (start, length) = aligned_crop(&text, 6, 2);
+        let cropped =  text.chars().skip(start).take(length).collect::<String>().trim().to_string();
+        assert_eq!("début", cropped);
+
+        // first word test
+        let (start, length) = aligned_crop(&text, 0, 1);
+        let cropped =  text.chars().skip(start).take(length).collect::<String>().trim().to_string();
+        assert_eq!("En", cropped);
+        // last word test
+        let (start, length) = aligned_crop(&text, 510, 2);
+        let cropped =  text.chars().skip(start).take(length).collect::<String>().trim().to_string();
+        assert_eq!("Fondation", cropped);
+
+        // CJK tests
+        let text = "this isのス foo myタイリ test";
+
+        // mixed charset
+        let (start, length) = aligned_crop(&text, 5, 3);
+        let cropped =  text.chars().skip(start).take(length).collect::<String>().trim().to_string();
+        assert_eq!("isのス", cropped);
+
+        // split regular word / CJK word, no space
+        let (start, length) = aligned_crop(&text, 7, 1);
+        let cropped =  text.chars().skip(start).take(length).collect::<String>().trim().to_string();
+        assert_eq!("のス", cropped);
+
+    }
+
     #[test]
     fn calculate_highlights() {
         let data = r#"{

meilisearch-http/src/helpers/tide.rs

@@ -1,97 +1,69 @@
 use crate::error::{ResponseError, SResult};
-use crate::models::token::*;
 use crate::Data;
-use chrono::Utc;
-use heed::types::{SerdeBincode, Str};
 use meilisearch_core::Index;
-use serde_json::Value;
-use tide::Context;
+use tide::Request;
 
-pub trait ContextExt {
-    fn is_allowed(&self, acl: ACL) -> SResult<()>;
-    fn header(&self, name: &str) -> Result<String, ResponseError>;
-    fn url_param(&self, name: &str) -> Result<String, ResponseError>;
-    fn index(&self) -> Result<Index, ResponseError>;
-    fn identifier(&self) -> Result<String, ResponseError>;
+pub enum ACL {
+    Admin,
+    Private,
+    Public,
 }
 
-impl ContextExt for Context<Data> {
+pub trait RequestExt {
+    fn is_allowed(&self, acl: ACL) -> SResult<()>;
+    fn url_param(&self, name: &str) -> SResult<String>;
+    fn index(&self) -> SResult<Index>;
+    fn document_id(&self) -> SResult<String>;
+}
+
+impl RequestExt for Request<Data> {
     fn is_allowed(&self, acl: ACL) -> SResult<()> {
-        let api_key = match &self.state().api_key {
-            Some(api_key) => api_key,
-            None => return Ok(()),
-        };
+        let user_api_key = self.header("X-Meili-API-Key");
 
-        let user_api_key = self.header("X-Meili-API-Key")?;
-        if user_api_key == *api_key {
-            return Ok(());
-        }
-        let request_index: Option<String> = None; //self.param::<String>("index").ok();
-
-        let db = &self.state().db;
-        let reader = db.main_read_txn().map_err(ResponseError::internal)?;
-
-        let token_key = format!("{}{}", TOKEN_PREFIX_KEY, user_api_key);
-
-        let token_config = db
-            .common_store()
-            .get::<_, Str, SerdeBincode<Token>>(&reader, &token_key)
-            .map_err(ResponseError::internal)?
-            .ok_or(ResponseError::invalid_token(format!(
-                "Api key does not exist: {}",
-                user_api_key
-            )))?;
-
-        if token_config.revoked {
-            return Err(ResponseError::invalid_token("token revoked"));
+        if self.state().api_keys.master.is_none() {
+            return Ok(())
         }
 
-        if let Some(index) = request_index {
-            if !token_config
-                .indexes
-                .iter()
-                .any(|r| match_wildcard(&r, &index))
-            {
-                return Err(ResponseError::invalid_token(
-                    "token is not allowed to access to this index",
-                ));
+        match acl {
+            ACL::Admin => {
+                if user_api_key == self.state().api_keys.master.as_deref() {
+                    return Ok(());
+                }
+            }
+            ACL::Private => {
+                if user_api_key == self.state().api_keys.master.as_deref() {
+                    return Ok(());
+                }
+                if user_api_key == self.state().api_keys.private.as_deref() {
+                    return Ok(());
+                }
+            }
+            ACL::Public => {
+                if user_api_key == self.state().api_keys.master.as_deref() {
+                    return Ok(());
+                }
+                if user_api_key == self.state().api_keys.private.as_deref() {
+                    return Ok(());
+                }
+                if user_api_key == self.state().api_keys.public.as_deref() {
+                    return Ok(());
+                }
             }
         }
 
-        if token_config.expires_at < Utc::now() {
-            return Err(ResponseError::invalid_token("token expired"));
-        }
-
-        if token_config.acl.contains(&ACL::All) {
-            return Ok(());
-        }
-
-        if !token_config.acl.contains(&acl) {
-            return Err(ResponseError::invalid_token("token do not have this ACL"));
-        }
-
-        Ok(())
+        Err(ResponseError::InvalidToken(
+            user_api_key.unwrap_or("Need a token").to_owned(),
+        ))
     }
 
-    fn header(&self, name: &str) -> Result<String, ResponseError> {
-        let header = self
-            .headers()
-            .get(name)
-            .ok_or(ResponseError::missing_header(name))?
-            .to_str()
-            .map_err(|_| ResponseError::missing_header("X-Meili-API-Key"))?
-            .to_string();
-        Ok(header)
-    }
-
-    fn url_param(&self, name: &str) -> Result<String, ResponseError> {
+    fn url_param(&self, name: &str) -> SResult<String> {
         let param = self
             .param::<String>(name)
             .map_err(|e| ResponseError::bad_parameter(name, e))?;
         Ok(param)
     }
 
-    fn index(&self) -> Result<Index, ResponseError> {
+    fn index(&self) -> SResult<Index> {
         let index_uid = self.url_param("index")?;
         let index = self
             .state()
@@ -101,16 +73,10 @@ impl ContextExt for Context<Data> {
         Ok(index)
     }
 
-    fn identifier(&self) -> Result<String, ResponseError> {
+    fn document_id(&self) -> SResult<String> {
         let name = self
-            .param::<Value>("identifier")
-            .as_ref()
-            .map(meilisearch_core::serde::value_to_string)
-            .map_err(|e| ResponseError::bad_parameter("identifier", e))?
-            .ok_or(ResponseError::bad_parameter(
-                "identifier",
-                "missing parameter",
-            ))?;
+            .param::<String>("document_id")
+            .map_err(|_| ResponseError::bad_parameter("documentId", "primaryKey"))?;
 
         Ok(name)
     }

meilisearch-http/src/lib.rs

@@ -1,3 +1,5 @@
+#![allow(clippy::or_fun_call)]
+
 pub mod data;
 pub mod error;
 pub mod helpers;

meilisearch-http/src/main.rs

@@ -1,12 +1,11 @@
-use std::env::VarError::NotPresent;
 use std::{env, thread};
 
-use http::header::HeaderValue;
+use async_std::task;
 use log::info;
 use main_error::MainError;
 use structopt::StructOpt;
-use tide::middleware::{CorsMiddleware, CorsOrigin};
-use tide_log::RequestLogger;
+use tide::middleware::{Cors, RequestLogger, Origin};
+use http::header::HeaderValue;
 
 use meilisearch_http::data::Data;
 use meilisearch_http::option::Opt;
@@ -20,35 +19,94 @@ mod analytics;
 static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
 
 pub fn main() -> Result<(), MainError> {
-    env_logger::init();
-
     let opt = Opt::from_args();
-    let data = Data::new(opt.clone());
 
-    if env::var("MEILI_NO_ANALYTICS") == Err(NotPresent) {
-        thread::spawn(|| analytics::analytics_sender());
+    match opt.env.as_ref() {
+        "production" => {
+            if opt.master_key.is_none() {
+                return Err(
+                    "In production mode, the environment variable MEILI_MASTER_KEY is mandatory"
+                        .into(),
+                );
+            }
+            env_logger::init();
+        }
+        "development" => {
+            env_logger::from_env(env_logger::Env::default().default_filter_or("info")).init();
+        }
+        _ => unreachable!(),
     }
 
+    if !opt.no_analytics {
+        thread::spawn(analytics::analytics_sender);
+    }
+
+    let data = Data::new(opt.clone());
+
     let data_cloned = data.clone();
     data.db.set_update_callback(Box::new(move |name, status| {
         index_update_callback(name, &data_cloned, status);
     }));
 
-    let mut app = tide::App::with_state(data);
+    print_launch_resume(&opt, &data);
 
-    app.middleware(
-        CorsMiddleware::new()
-            .allow_origin(CorsOrigin::from("*"))
-            .allow_methods(HeaderValue::from_static("GET, POST, OPTIONS")),
-    );
+    let mut app = tide::with_state(data);
+
+    app.middleware(Cors::new()
+        .allow_methods(HeaderValue::from_static("GET, POST, PUT, DELETE, OPTIONS"))
+        .allow_headers(HeaderValue::from_static("X-Meili-API-Key"))
+        .allow_origin(Origin::from("*")));
     app.middleware(RequestLogger::new());
-    app.middleware(tide_compression::Compression::new());
-    app.middleware(tide_compression::Decompression::new());
 
     routes::load_routes(&mut app);
 
-    info!("Server HTTP enabled");
-    app.run(opt.http_addr)?;
-
+    task::block_on(app.listen(opt.http_addr))?;
     Ok(())
 }
+
+pub fn print_launch_resume(opt: &Opt, data: &Data) {
+    let ascii_name = r#"
+888b     d888          d8b 888 d8b  .d8888b.                                    888
+8888b   d8888          Y8P 888 Y8P d88P  Y88b                                   888
+88888b.d88888              888     Y88b.                                        888
+888Y88888P888  .d88b.  888 888 888  "Y888b.    .d88b.   8888b.  888d888 .d8888b 88888b.
+888 Y888P 888 d8P  Y8b 888 888 888     "Y88b. d8P  Y8b     "88b 888P"  d88P"    888 "88b
+888  Y8P  888 88888888 888 888 888       "888 88888888 .d888888 888    888      888  888
+888   "   888 Y8b.     888 888 888 Y88b  d88P Y8b.     888  888 888    Y88b.    888  888
+888       888  "Y8888  888 888 888  "Y8888P"   "Y8888  "Y888888 888     "Y8888P 888  888
+"#;
+
+    println!("{}", ascii_name);
+
+    info!("Database path: {:?}", opt.db_path);
+    info!("Start server on: {:?}", opt.http_addr);
+    info!("Environment: {:?}", opt.env);
+    info!("Commit SHA: {:?}", env!("VERGEN_SHA").to_string());
+    info!(
+        "Build date: {:?}",
+        env!("VERGEN_BUILD_TIMESTAMP").to_string()
+    );
+    info!(
+        "Package version: {:?}",
+        env!("CARGO_PKG_VERSION").to_string()
+    );
+
+    if let Some(master_key) = &data.api_keys.master {
+        info!("Master Key: {:?}", master_key);
+
+        if let Some(private_key) = &data.api_keys.private {
+            info!("Private Key: {:?}", private_key);
+        }
+
+        if let Some(public_key) = &data.api_keys.public {
+            info!("Public Key: {:?}", public_key);
+        }
+    } else {
+        info!("No master key found; The server will have no securities.\
+            If you need some protection in development mode, please export a key. export MEILI_MASTER_KEY=xxx");
+    }
+
+    info!("If you need extra information; Please refer to the documentation: http://docs.meilisearch.com");
+    info!("If you want to support us or help us; Please consult our Github repo: http://github.com/meilisearch/meilisearch");
+    info!("If you want to contact us; Please chat with us on http://meilisearch.com or by email to bonjour@meilisearch.com");
+}

meilisearch-http/src/models/mod.rs

@@ -1,3 +1 @@
-pub mod schema;
-pub mod token;
 pub mod update_operation;

meilisearch-http/src/models/schema.rs

@@ -1,118 +0,0 @@
-use std::collections::HashSet;
-
-use indexmap::IndexMap;
-use meilisearch_schema::{Schema, SchemaBuilder, SchemaProps};
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub enum FieldProperties {
-    Identifier,
-    Indexed,
-    Displayed,
-    Ranked,
-}
-
-#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
-pub struct SchemaBody(IndexMap<String, HashSet<FieldProperties>>);
-
-impl From<Schema> for SchemaBody {
-    fn from(value: Schema) -> SchemaBody {
-        let mut map = IndexMap::new();
-        for (name, _attr, props) in value.iter() {
-            let old_properties = map.entry(name.to_owned()).or_insert(HashSet::new());
-            if props.is_indexed() {
-                old_properties.insert(FieldProperties::Indexed);
-            }
-            if props.is_displayed() {
-                old_properties.insert(FieldProperties::Displayed);
-            }
-            if props.is_ranked() {
-                old_properties.insert(FieldProperties::Ranked);
-            }
-        }
-        let old_properties = map
-            .entry(value.identifier_name().to_string())
-            .or_insert(HashSet::new());
-        old_properties.insert(FieldProperties::Identifier);
-        old_properties.insert(FieldProperties::Displayed);
-        SchemaBody(map)
-    }
-}
-
-impl Into<Schema> for SchemaBody {
-    fn into(self) -> Schema {
-        let mut identifier = "documentId".to_string();
-        let mut attributes = IndexMap::new();
-        for (field, properties) in self.0 {
-            let mut indexed = false;
-            let mut displayed = false;
-            let mut ranked = false;
-            for property in properties {
-                match property {
-                    FieldProperties::Indexed => indexed = true,
-                    FieldProperties::Displayed => displayed = true,
-                    FieldProperties::Ranked => ranked = true,
-                    FieldProperties::Identifier => identifier = field.clone(),
-                }
-            }
-            attributes.insert(
-                field,
-                SchemaProps {
-                    indexed,
-                    displayed,
-                    ranked,
-                },
-            );
-        }
-
-        let mut builder = SchemaBuilder::with_identifier(identifier);
-        for (field, props) in attributes {
-            builder.new_attribute(field, props);
-        }
-        builder.build()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_schema_body_conversion() {
-        let schema_body = r#"
-        {
-            "id": ["identifier", "indexed", "displayed"],
-            "title": ["indexed", "displayed"],
-            "date": ["displayed"]
-        }
-        "#;
-
-        let schema_builder = r#"
-        {
-            "identifier": "id",
-            "attributes": {
-                "id": {
-                    "indexed": true,
-                    "displayed": true
-                },
-                "title": {
-                    "indexed": true,
-                    "displayed": true
-                },
-                "date": {
-                    "displayed": true
-                }
-            }
-        }
-        "#;
-
-        let schema_body: SchemaBody = serde_json::from_str(schema_body).unwrap();
-        let schema_builder: SchemaBuilder = serde_json::from_str(schema_builder).unwrap();
-
-        let schema_from_body: Schema = schema_body.into();
-        let schema_from_builder: Schema = schema_builder.build();
-
-        assert_eq!(schema_from_body, schema_from_builder);
-    }
-}

meilisearch-http/src/models/token.rs

@@ -1,72 +0,0 @@
-use chrono::{DateTime, Utc};
-use serde::{Deserialize, Serialize};
-
-pub const TOKEN_PREFIX_KEY: &str = "_token_";
-
-#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub enum ACL {
-    IndexesRead,
-    IndexesWrite,
-    DocumentsRead,
-    DocumentsWrite,
-    SettingsRead,
-    SettingsWrite,
-    Admin,
-    #[serde(rename = "*")]
-    All,
-}
-
-pub type Wildcard = String;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct Token {
-    pub key: String,
-    pub description: String,
-    pub acl: Vec<ACL>,
-    pub indexes: Vec<Wildcard>,
-    pub created_at: DateTime<Utc>,
-    pub updated_at: DateTime<Utc>,
-    pub expires_at: DateTime<Utc>,
-    pub revoked: bool,
-}
-
-fn cleanup_wildcard(input: &str) -> (bool, &str, bool) {
-    let first = input.chars().next().filter(|&c| c == '*').is_some();
-    let last = input.chars().last().filter(|&c| c == '*').is_some();
-    let bound_last = std::cmp::max(input.len().saturating_sub(last as usize), first as usize);
-    let output = input.get(first as usize..bound_last).unwrap();
-    (first, output, last)
-}
-
-pub fn match_wildcard(pattern: &str, input: &str) -> bool {
-    let (first, pattern, last) = cleanup_wildcard(pattern);
-
-    match (first, last) {
-        (false, false) => pattern == input,
-        (true, false) => input.ends_with(pattern),
-        (false, true) => input.starts_with(pattern),
-        (true, true) => input.contains(pattern),
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_match_wildcard() {
-        assert!(match_wildcard("*", "qqq"));
-        assert!(match_wildcard("*", ""));
-        assert!(match_wildcard("*ab", "qqqab"));
-        assert!(match_wildcard("*ab*", "qqqabqq"));
-        assert!(match_wildcard("ab*", "abqqq"));
-        assert!(match_wildcard("**", "ab"));
-        assert!(match_wildcard("ab", "ab"));
-        assert!(match_wildcard("ab*", "ab"));
-        assert!(match_wildcard("*ab", "ab"));
-        assert!(match_wildcard("*ab*", "ab"));
-        assert!(match_wildcard("*😆*", "ab😆dsa"));
-    }
-}

meilisearch-http/src/models/update_operation.rs

@@ -6,7 +6,7 @@ pub enum UpdateOperation {
     ClearAllDocuments,
     DocumentsAddition,
     DocumentsDeletion,
-    SynonymsAddition,
+    SynonymsUpdate,
     SynonymsDeletion,
     StopWordsAddition,
     StopWordsDeletion,
@@ -22,7 +22,7 @@ impl fmt::Display for UpdateOperation {
             ClearAllDocuments => write!(f, "ClearAllDocuments"),
             DocumentsAddition => write!(f, "DocumentsAddition"),
             DocumentsDeletion => write!(f, "DocumentsDeletion"),
-            SynonymsAddition => write!(f, "SynonymsAddition"),
+            SynonymsUpdate => write!(f, "SynonymsUpdate"),
             SynonymsDeletion => write!(f, "SynonymsDelettion"),
             StopWordsAddition => write!(f, "StopWordsAddition"),
             StopWordsDeletion => write!(f, "StopWordsDeletion"),

meilisearch-http/src/option.rs

@@ -1,18 +1,27 @@
 use structopt::StructOpt;
 
+const POSSIBLE_ENV: [&str; 2] = ["development", "production"];
+
 #[derive(Debug, Clone, StructOpt)]
 pub struct Opt {
     /// The destination where the database must be created.
-    #[structopt(long, env = "MEILI_DB_PATH", default_value = "/tmp/meilisearch")]
+    #[structopt(long, env = "MEILI_DB_PATH", default_value = "./data.ms")]
     pub db_path: String,
 
     /// The address on which the http server will listen.
-    #[structopt(long, env = "MEILI_HTTP_ADDR", default_value = "127.0.0.1:8080")]
+    #[structopt(long, env = "MEILI_HTTP_ADDR", default_value = "127.0.0.1:7700")]
     pub http_addr: String,
 
     /// The master key allowing you to do everything on the server.
-    #[structopt(long, env = "MEILI_API_KEY")]
-    pub api_key: Option<String>,
+    #[structopt(long, env = "MEILI_MASTER_KEY")]
+    pub master_key: Option<String>,
+
+    /// This environment variable must be set to `production` if your are running in production.
+    /// If the server is running in development mode more logs will be displayed,
+    /// and the master key can be avoided which implies that there is no security on the updates routes.
+    /// This is useful to debug when integrating the engine with another service.
+    #[structopt(long, env = "MEILI_ENV", default_value = "development", possible_values = &POSSIBLE_ENV)]
+    pub env: String,
 
     /// Do not send analytics to Meili.
     #[structopt(long, env = "MEILI_NO_ANALYTICS")]

meilisearch-http/src/routes/document.rs

@@ -1,39 +1,35 @@
 use std::collections::{BTreeSet, HashSet};
 
-use http::StatusCode;
 use indexmap::IndexMap;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use tide::querystring::ContextExt as QSContextExt;
-use tide::response::IntoResponse;
-use tide::{Context, Response};
+use tide::{Request, Response};
 
 use crate::error::{ResponseError, SResult};
-use crate::helpers::tide::ContextExt;
-use crate::models::token::ACL::*;
+use crate::helpers::tide::RequestExt;
+use crate::helpers::tide::ACL::*;
 use crate::Data;
 
-pub async fn get_document(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(DocumentsRead)?;
+pub async fn get_document(ctx: Request<Data>) -> SResult<Response> {
+    ctx.is_allowed(Public)?;
 
     let index = ctx.index()?;
 
-    let identifier = ctx.identifier()?;
-    let document_id = meilisearch_core::serde::compute_document_id(identifier.clone());
+    let original_document_id = ctx.document_id()?;
+    let document_id = meilisearch_core::serde::compute_document_id(original_document_id.clone());
 
     let db = &ctx.state().db;
-    let reader = db.main_read_txn().map_err(ResponseError::internal)?;
+    let reader = db.main_read_txn()?;
 
     let response = index
-        .document::<IndexMap<String, Value>>(&reader, None, document_id)
-        .map_err(ResponseError::internal)?
-        .ok_or(ResponseError::document_not_found(&identifier))?;
+        .document::<IndexMap<String, Value>>(&reader, None, document_id)?
+        .ok_or(ResponseError::document_not_found(&original_document_id))?;
 
     if response.is_empty() {
-        return Err(ResponseError::document_not_found(identifier));
+        return Err(ResponseError::document_not_found(&original_document_id));
     }
 
-    Ok(tide::response::json(response))
+    Ok(tide::Response::new(200).body_json(&response)?)
 }
 
 #[derive(Default, Serialize)]
@@ -42,28 +38,22 @@ pub struct IndexUpdateResponse {
     pub update_id: u64,
 }
 
-pub async fn delete_document(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(DocumentsWrite)?;
+pub async fn delete_document(ctx: Request<Data>) -> SResult<Response> {
+    ctx.is_allowed(Private)?;
 
     let index = ctx.index()?;
-    let identifier = ctx.identifier()?;
-    let document_id = meilisearch_core::serde::compute_document_id(identifier.clone());
-
+    let document_id = ctx.document_id()?;
+    let document_id = meilisearch_core::serde::compute_document_id(document_id);
     let db = &ctx.state().db;
-    let mut update_writer = db.update_write_txn().map_err(ResponseError::internal)?;
-
+    let mut update_writer = db.update_write_txn()?;
     let mut documents_deletion = index.documents_deletion();
     documents_deletion.delete_document_by_id(document_id);
-    let update_id = documents_deletion
-        .finalize(&mut update_writer)
-        .map_err(ResponseError::internal)?;
+    let update_id = documents_deletion.finalize(&mut update_writer)?;
 
-    update_writer.commit().map_err(ResponseError::internal)?;
+    update_writer.commit()?;
 
     let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
+    Ok(tide::Response::new(202).body_json(&response_body)?)
 }
 
 #[derive(Default, Deserialize)]
@@ -74,23 +64,24 @@ struct BrowseQuery {
     attributes_to_retrieve: Option<String>,
 }
 
-pub async fn get_all_documents(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(DocumentsRead)?;
+pub async fn get_all_documents(ctx: Request<Data>) -> SResult<Response> {
+    ctx.is_allowed(Private)?;
 
     let index = ctx.index()?;
-    let query: BrowseQuery = ctx.url_query().unwrap_or(BrowseQuery::default());
+    let query: BrowseQuery = ctx.query().unwrap_or_default();
 
     let offset = query.offset.unwrap_or(0);
     let limit = query.limit.unwrap_or(20);
 
     let db = &ctx.state().db;
-    let reader = db.main_read_txn().map_err(ResponseError::internal)?;
+    let reader = db.main_read_txn()?;
 
-    let documents_ids: Result<BTreeSet<_>, _> =
-        match index.documents_fields_counts.documents_ids(&reader) {
-            Ok(documents_ids) => documents_ids.skip(offset).take(limit).collect(),
-            Err(e) => return Err(ResponseError::internal(e)),
-        };
+    let documents_ids: Result<BTreeSet<_>, _> = index
+        .documents_fields_counts
+        .documents_ids(&reader)?
+        .skip(offset)
+        .take(limit)
+        .collect();
 
     let documents_ids = match documents_ids {
         Ok(documents_ids) => documents_ids,
@@ -114,55 +105,54 @@ pub async fn get_all_documents(ctx: Context<Data>) -> SResult<Response> {
         }
     }
 
-    Ok(tide::response::json(response_body))
+    Ok(tide::Response::new(200).body_json(&response_body)?)
 }
 
-fn infered_schema(document: &IndexMap<String, Value>) -> Option<meilisearch_schema::Schema> {
-    use meilisearch_schema::{SchemaBuilder, DISPLAYED, INDEXED};
-
-    let mut identifier = None;
+fn find_primary_key(document: &IndexMap<String, Value>) -> Option<String> {
     for key in document.keys() {
-        if identifier.is_none() && key.to_lowercase().contains("id") {
-            identifier = Some(key);
+        if key.to_lowercase().contains("id") {
+            return Some(key.to_string());
         }
     }
-
-    match identifier {
-        Some(identifier) => {
-            let mut builder = SchemaBuilder::with_identifier(identifier);
-            for key in document.keys() {
-                builder.new_attribute(key, DISPLAYED | INDEXED);
-            }
-            Some(builder.build())
-        }
-        None => None,
-    }
+    None
 }
 
-async fn update_multiple_documents(mut ctx: Context<Data>, is_partial: bool) -> SResult<Response> {
-    ctx.is_allowed(DocumentsWrite)?;
+#[derive(Default, Deserialize)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+struct UpdateDocumentsQuery {
+    primary_key: Option<String>,
+}
+
+async fn update_multiple_documents(mut ctx: Request<Data>, is_partial: bool) -> SResult<Response> {
+    ctx.is_allowed(Private)?;
+
+    let index = ctx.index()?;
 
     let data: Vec<IndexMap<String, Value>> =
         ctx.body_json().await.map_err(ResponseError::bad_request)?;
-    let index = ctx.index()?;
+    let query: UpdateDocumentsQuery = ctx.query().unwrap_or_default();
 
     let db = &ctx.state().db;
-    let reader = db.main_read_txn().map_err(ResponseError::internal)?;
-    let mut update_writer = db.update_write_txn().map_err(ResponseError::internal)?;
 
-    let current_schema = index
+    let reader = db.main_read_txn()?;
+    let mut schema = index
         .main
-        .schema(&reader)
-        .map_err(ResponseError::internal)?;
-    if current_schema.is_none() {
-        match data.first().and_then(infered_schema) {
-            Some(schema) => {
-                index
-                    .schema_update(&mut update_writer, schema)
-                    .map_err(ResponseError::internal)?;
-            }
-            None => return Err(ResponseError::bad_request("Could not infer a schema")),
-        }
+        .schema(&reader)?
+        .ok_or(ResponseError::internal("schema not found"))?;
+
+    if schema.primary_key().is_none() {
+        let id = match query.primary_key {
+            Some(id) => id,
+            None => match data.first().and_then(|docs| find_primary_key(docs)) {
+                Some(id) => id,
+                None => return Err(ResponseError::bad_request("Could not infer a primary key")),
+            },
+        };
+
+        let mut writer = db.main_write_txn()?;
+        schema.set_primary_key(&id).map_err(ResponseError::bad_request)?;
+        index.main.put_schema(&mut writer, &schema)?;
+        writer.commit()?;
     }
 
     let mut document_addition = if is_partial {
@@ -175,71 +165,59 @@ async fn update_multiple_documents(mut ctx: Context<Data>, is_partial: bool) ->
         document_addition.update_document(document);
     }
 
-    let update_id = document_addition
-        .finalize(&mut update_writer)
-        .map_err(ResponseError::internal)?;
-
-    update_writer.commit().map_err(ResponseError::internal)?;
+    let mut update_writer = db.update_write_txn()?;
+    let update_id = document_addition.finalize(&mut update_writer)?;
+    update_writer.commit()?;
 
     let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
+    Ok(tide::Response::new(202).body_json(&response_body)?)
 }
 
-pub async fn add_or_replace_multiple_documents(ctx: Context<Data>) -> SResult<Response> {
+pub async fn add_or_replace_multiple_documents(ctx: Request<Data>) -> SResult<Response> {
     update_multiple_documents(ctx, false).await
 }
 
-pub async fn add_or_update_multiple_documents(ctx: Context<Data>) -> SResult<Response> {
+pub async fn add_or_update_multiple_documents(ctx: Request<Data>) -> SResult<Response> {
     update_multiple_documents(ctx, true).await
 }
 
-pub async fn delete_multiple_documents(mut ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(DocumentsWrite)?;
+pub async fn delete_multiple_documents(mut ctx: Request<Data>) -> SResult<Response> {
+    ctx.is_allowed(Private)?;
 
     let data: Vec<Value> = ctx.body_json().await.map_err(ResponseError::bad_request)?;
     let index = ctx.index()?;
 
     let db = &ctx.state().db;
-    let mut writer = db.update_write_txn().map_err(ResponseError::internal)?;
+    let mut writer = db.update_write_txn()?;
 
     let mut documents_deletion = index.documents_deletion();
 
-    for identifier in data {
-        if let Some(identifier) = meilisearch_core::serde::value_to_string(&identifier) {
+    for document_id in data {
+        if let Some(document_id) = meilisearch_core::serde::value_to_string(&document_id) {
             documents_deletion
-                .delete_document_by_id(meilisearch_core::serde::compute_document_id(identifier));
+                .delete_document_by_id(meilisearch_core::serde::compute_document_id(document_id));
         }
     }
 
-    let update_id = documents_deletion
-        .finalize(&mut writer)
-        .map_err(ResponseError::internal)?;
+    let update_id = documents_deletion.finalize(&mut writer)?;
 
-    writer.commit().map_err(ResponseError::internal)?;
+    writer.commit()?;
 
     let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
+    Ok(tide::Response::new(202).body_json(&response_body)?)
 }
 
-pub async fn clear_all_documents(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(DocumentsWrite)?;
+pub async fn clear_all_documents(ctx: Request<Data>) -> SResult<Response> {
+    ctx.is_allowed(Private)?;
 
     let index = ctx.index()?;
 
     let db = &ctx.state().db;
-    let mut writer = db.update_write_txn().map_err(ResponseError::internal)?;
+    let mut writer = db.update_write_txn()?;
 
-    let update_id = index
-        .clear_all(&mut writer)
-        .map_err(ResponseError::internal)?;
-    writer.commit().map_err(ResponseError::internal)?;
+    let update_id = index.clear_all(&mut writer)?;
+    writer.commit()?;
 
     let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
+    Ok(tide::Response::new(202).body_json(&response_body)?)
 }

meilisearch-http/src/routes/health.rs

@@ -1,17 +1,17 @@
 use crate::error::{ResponseError, SResult};
-use crate::helpers::tide::ContextExt;
-use crate::models::token::ACL::*;
+use crate::helpers::tide::RequestExt;
+use crate::helpers::tide::ACL::*;
 use crate::Data;
 
 use heed::types::{Str, Unit};
 use serde::Deserialize;
-use tide::Context;
+use tide::{Request, Response};
 
 const UNHEALTHY_KEY: &str = "_is_unhealthy";
 
-pub async fn get_health(ctx: Context<Data>) -> SResult<()> {
+pub async fn get_health(ctx: Request<Data>) -> SResult<Response> {
     let db = &ctx.state().db;
-    let reader = db.main_read_txn().map_err(ResponseError::internal)?;
+    let reader = db.main_read_txn()?;
 
     let common_store = ctx.state().db.common_store();
 
@@ -19,45 +19,29 @@ pub async fn get_health(ctx: Context<Data>) -> SResult<()> {
         return Err(ResponseError::Maintenance);
     }
 
-    Ok(())
+    Ok(tide::Response::new(200))
 }
 
-pub async fn set_healthy(ctx: Context<Data>) -> SResult<()> {
+pub async fn set_healthy(ctx: Request<Data>) -> SResult<Response> {
     ctx.is_allowed(Admin)?;
-
     let db = &ctx.state().db;
-    let mut writer = db.main_write_txn().map_err(ResponseError::internal)?;
-
+    let mut writer = db.main_write_txn()?;
     let common_store = ctx.state().db.common_store();
-    match common_store.delete::<_, Str>(&mut writer, UNHEALTHY_KEY) {
-        Ok(_) => (),
-        Err(e) => return Err(ResponseError::internal(e)),
-    }
+    common_store.delete::<_, Str>(&mut writer, UNHEALTHY_KEY)?;
+    writer.commit()?;
 
-    if let Err(e) = writer.commit() {
-        return Err(ResponseError::internal(e));
-    }
-
-    Ok(())
+    Ok(tide::Response::new(200))
 }
 
-pub async fn set_unhealthy(ctx: Context<Data>) -> SResult<()> {
+pub async fn set_unhealthy(ctx: Request<Data>) -> SResult<Response> {
     ctx.is_allowed(Admin)?;
-
     let db = &ctx.state().db;
-    let mut writer = db.main_write_txn().map_err(ResponseError::internal)?;
-
+    let mut writer = db.main_write_txn()?;
     let common_store = ctx.state().db.common_store();
+    common_store.put::<_, Str, Unit>(&mut writer, UNHEALTHY_KEY, &())?;
+    writer.commit()?;
 
-    if let Err(e) = common_store.put::<_, Str, Unit>(&mut writer, UNHEALTHY_KEY, &()) {
-        return Err(ResponseError::internal(e));
-    }
-
-    if let Err(e) = writer.commit() {
-        return Err(ResponseError::internal(e));
-    }
-
-    Ok(())
+    Ok(tide::Response::new(200))
 }
 
 #[derive(Deserialize, Clone)]
@@ -65,7 +49,7 @@ struct HealtBody {
     health: bool,
 }
 
-pub async fn change_healthyness(mut ctx: Context<Data>) -> SResult<()> {
+pub async fn change_healthyness(mut ctx: Request<Data>) -> SResult<Response> {
     let body: HealtBody = ctx.body_json().await.map_err(ResponseError::bad_request)?;
 
     if body.health {

meilisearch-http/src/routes/index.rs

@@ -1,20 +1,14 @@
 use chrono::{DateTime, Utc};
-use http::StatusCode;
 use log::error;
 use meilisearch_core::ProcessedUpdateResult;
-use meilisearch_schema::{Schema, SchemaBuilder};
 use rand::seq::SliceRandom;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
-use tide::querystring::ContextExt as QSContextExt;
-use tide::response::IntoResponse;
-use tide::{Context, Response};
+use tide::{Request, Response};
 
-use crate::error::{ResponseError, SResult};
-use crate::helpers::tide::ContextExt;
-use crate::models::schema::SchemaBody;
-use crate::models::token::ACL::*;
-use crate::routes::document::IndexUpdateResponse;
+use crate::error::{IntoInternalError, ResponseError, SResult};
+use crate::helpers::tide::RequestExt;
+use crate::helpers::tide::ACL::*;
 use crate::Data;
 
 fn generate_uid() -> String {
@@ -26,13 +20,13 @@ fn generate_uid() -> String {
         .collect()
 }
 
-pub async fn list_indexes(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(IndexesRead)?;
+pub async fn list_indexes(ctx: Request<Data>) -> SResult<Response> {
+    ctx.is_allowed(Private)?;
 
     let indexes_uids = ctx.state().db.indexes_uids();
 
     let db = &ctx.state().db;
-    let reader = db.main_read_txn().map_err(ResponseError::internal)?;
+    let reader = db.main_read_txn()?;
 
     let mut response_body = Vec::new();
 
@@ -41,29 +35,26 @@ pub async fn list_indexes(ctx: Context<Data>) -> SResult<Response> {
 
         match index {
             Some(index) => {
-                let name = index
-                    .main
-                    .name(&reader)
-                    .map_err(ResponseError::internal)?
-                    .ok_or(ResponseError::internal("'name' not found"))?;
-                let created_at = index
-                    .main
-                    .created_at(&reader)
-                    .map_err(ResponseError::internal)?
-                    .ok_or(ResponseError::internal("'created_at' date not found"))?;
-                let updated_at = index
-                    .main
-                    .updated_at(&reader)
-                    .map_err(ResponseError::internal)?
-                    .ok_or(ResponseError::internal("'updated_at' date not found"))?;
+                let name = index.main.name(&reader)?.into_internal_error()?;
+                let created_at = index.main.created_at(&reader)?.into_internal_error()?;
+                let updated_at = index.main.updated_at(&reader)?.into_internal_error()?;
 
-                let index_reponse = IndexResponse {
+                let primary_key = match index.main.schema(&reader) {
+                    Ok(Some(schema)) => match schema.primary_key() {
+                        Some(primary_key) => Some(primary_key.to_owned()),
+                        None => None,
+                    },
+                    _ => None,
+                };
+
+                let index_response = IndexResponse {
                     name,
                     uid: index_uid,
                     created_at,
                     updated_at,
+                    primary_key,
                 };
-                response_body.push(index_reponse);
+                response_body.push(index_response);
             }
             None => error!(
                 "Index {} is referenced in the indexes list but cannot be found",
@@ -72,7 +63,7 @@ pub async fn list_indexes(ctx: Context<Data>) -> SResult<Response> {
         }
     }
 
-    Ok(tide::response::json(response_body))
+    Ok(tide::Response::new(200).body_json(&response_body)?)
 }
 
 #[derive(Debug, Serialize)]
@@ -82,49 +73,47 @@ struct IndexResponse {
     uid: String,
     created_at: DateTime<Utc>,
     updated_at: DateTime<Utc>,
+    primary_key: Option<String>,
 }
 
-pub async fn get_index(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(IndexesRead)?;
+pub async fn get_index(ctx: Request<Data>) -> SResult<Response> {
+    ctx.is_allowed(Private)?;
 
     let index = ctx.index()?;
 
     let db = &ctx.state().db;
-    let reader = db.main_read_txn().map_err(ResponseError::internal)?;
+    let reader = db.main_read_txn()?;
 
     let uid = ctx.url_param("index")?;
-    let name = index
-        .main
-        .name(&reader)
-        .map_err(ResponseError::internal)?
-        .ok_or(ResponseError::internal("'name' not found"))?;
-    let created_at = index
-        .main
-        .created_at(&reader)
-        .map_err(ResponseError::internal)?
-        .ok_or(ResponseError::internal("'created_at' date not found"))?;
-    let updated_at = index
-        .main
-        .updated_at(&reader)
-        .map_err(ResponseError::internal)?
-        .ok_or(ResponseError::internal("'updated_at' date not found"))?;
+    let name = index.main.name(&reader)?.into_internal_error()?;
+    let created_at = index.main.created_at(&reader)?.into_internal_error()?;
+    let updated_at = index.main.updated_at(&reader)?.into_internal_error()?;
+
+    let primary_key = match index.main.schema(&reader) {
+        Ok(Some(schema)) => match schema.primary_key() {
+            Some(primary_key) => Some(primary_key.to_owned()),
+            None => None,
+        },
+        _ => None,
+    };
 
     let response_body = IndexResponse {
         name,
         uid,
         created_at,
         updated_at,
+        primary_key,
     };
 
-    Ok(tide::response::json(response_body))
+    Ok(tide::Response::new(200).body_json(&response_body)?)
 }
 
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 struct IndexCreateRequest {
-    name: String,
+    name: Option<String>,
     uid: Option<String>,
-    schema: Option<SchemaBody>,
+    primary_key: Option<String>,
 }
 
 #[derive(Debug, Serialize)]
@@ -132,25 +121,38 @@ struct IndexCreateRequest {
 struct IndexCreateResponse {
     name: String,
     uid: String,
-    schema: Option<SchemaBody>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    update_id: Option<u64>,
     created_at: DateTime<Utc>,
     updated_at: DateTime<Utc>,
+    primary_key: Option<String>,
 }
 
-pub async fn create_index(mut ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(IndexesWrite)?;
+pub async fn create_index(mut ctx: Request<Data>) -> SResult<Response> {
+    ctx.is_allowed(Private)?;
 
     let body = ctx
         .body_json::<IndexCreateRequest>()
         .await
         .map_err(ResponseError::bad_request)?;
 
+    if let (None, None) = (body.name.clone(), body.uid.clone()) {
+        return Err(ResponseError::bad_request(
+            "Index creation must have an uid",
+        ));
+    }
+
     let db = &ctx.state().db;
 
     let uid = match body.uid {
-        Some(uid) => uid,
+        Some(uid) => {
+            if uid
+                .chars()
+                .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_')
+            {
+                uid
+            } else {
+                return Err(ResponseError::InvalidIndexUid);
+            }
+        }
         None => loop {
             let uid = generate_uid();
             if db.open_index(&uid).is_none() {
@@ -164,52 +166,43 @@ pub async fn create_index(mut ctx: Context<Data>) -> SResult<Response> {
         Err(e) => return Err(ResponseError::create_index(e)),
     };
 
-    let mut writer = db.main_write_txn().map_err(ResponseError::internal)?;
-    let mut update_writer = db.update_write_txn().map_err(ResponseError::internal)?;
+    let mut writer = db.main_write_txn()?;
+    let name = body.name.unwrap_or(uid.clone());
+    created_index.main.put_name(&mut writer, &name)?;
+    let created_at = created_index
+        .main
+        .created_at(&writer)?
+        .into_internal_error()?;
+    let updated_at = created_index
+        .main
+        .updated_at(&writer)?
+        .into_internal_error()?;
 
-    created_index
-        .main
-        .put_name(&mut writer, &body.name)
-        .map_err(ResponseError::internal)?;
-    created_index
-        .main
-        .put_created_at(&mut writer)
-        .map_err(ResponseError::internal)?;
-    created_index
-        .main
-        .put_updated_at(&mut writer)
-        .map_err(ResponseError::internal)?;
-
-    let schema: Option<Schema> = body.schema.clone().map(Into::into);
-    let mut response_update_id = None;
-    if let Some(schema) = schema {
-        let update_id = created_index
-            .schema_update(&mut update_writer, schema)
-            .map_err(ResponseError::internal)?;
-        response_update_id = Some(update_id)
+    if let Some(id) = body.primary_key.clone() {
+        if let Some(mut schema) = created_index.main.schema(&mut writer)? {
+            schema.set_primary_key(&id).map_err(ResponseError::bad_request)?;
+            created_index.main.put_schema(&mut writer, &schema)?;
+        }
     }
 
-    writer.commit().map_err(ResponseError::internal)?;
-    update_writer.commit().map_err(ResponseError::internal)?;
+    writer.commit()?;
 
     let response_body = IndexCreateResponse {
-        name: body.name,
+        name,
         uid,
-        schema: body.schema,
-        update_id: response_update_id,
-        created_at: Utc::now(),
-        updated_at: Utc::now(),
+        created_at,
+        updated_at,
+        primary_key: body.primary_key,
     };
 
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::CREATED)
-        .into_response())
+    Ok(tide::Response::new(201).body_json(&response_body)?)
 }
 
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 struct UpdateIndexRequest {
-    name: String,
+    name: Option<String>,
+    primary_key: Option<String>,
 }
 
 #[derive(Debug, Serialize)]
@@ -219,10 +212,11 @@ struct UpdateIndexResponse {
     uid: String,
     created_at: DateTime<Utc>,
     updated_at: DateTime<Utc>,
+    primary_key: Option<String>,
 }
 
-pub async fn update_index(mut ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(IndexesWrite)?;
+pub async fn update_index(mut ctx: Request<Data>) -> SResult<Response> {
+    ctx.is_allowed(Private)?;
 
     let body = ctx
         .body_json::<UpdateIndexRequest>()
@@ -233,176 +227,95 @@ pub async fn update_index(mut ctx: Context<Data>) -> SResult<Response> {
     let index = ctx.index()?;
 
     let db = &ctx.state().db;
-    let mut writer = db.main_write_txn().map_err(ResponseError::internal)?;
+    let mut writer = db.main_write_txn()?;
 
-    index
-        .main
-        .put_name(&mut writer, &body.name)
-        .map_err(ResponseError::internal)?;
+    if let Some(name) = body.name {
+        index.main.put_name(&mut writer, &name)?;
+    }
 
-    index
-        .main
-        .put_updated_at(&mut writer)
-        .map_err(ResponseError::internal)?;
+    if let Some(id) = body.primary_key.clone() {
+        if let Some(mut schema) = index.main.schema(&mut writer)? {
+            match schema.primary_key() {
+                Some(_) => {
+                    return Err(ResponseError::bad_request(
+                        "The primary key cannot be updated",
+                    ));
+                }
+                None => {
+                    schema
+                        .set_primary_key(&id)
+                        .map_err(ResponseError::bad_request)?;
+                    index.main.put_schema(&mut writer, &schema)?;
+                }
+            }
+        }
+    }
 
-    writer.commit().map_err(ResponseError::internal)?;
-    let reader = db.main_read_txn().map_err(ResponseError::internal)?;
+    index.main.put_updated_at(&mut writer)?;
+    writer.commit()?;
 
-    let created_at = index
-        .main
-        .created_at(&reader)
-        .map_err(ResponseError::internal)?
-        .ok_or(ResponseError::internal("'created_at' date not found"))?;
-    let updated_at = index
-        .main
-        .updated_at(&reader)
-        .map_err(ResponseError::internal)?
-        .ok_or(ResponseError::internal("'updated_at' date not found"))?;
+    let reader = db.main_read_txn()?;
+    let name = index.main.name(&reader)?.into_internal_error()?;
+    let created_at = index.main.created_at(&reader)?.into_internal_error()?;
+    let updated_at = index.main.updated_at(&reader)?.into_internal_error()?;
+
+    let primary_key = match index.main.schema(&reader) {
+        Ok(Some(schema)) => match schema.primary_key() {
+            Some(primary_key) => Some(primary_key.to_owned()),
+            None => None,
+        },
+        _ => None,
+    };
 
     let response_body = UpdateIndexResponse {
-        name: body.name,
+        name,
         uid: index_uid,
         created_at,
         updated_at,
+        primary_key,
     };
 
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
+    Ok(tide::Response::new(200).body_json(&response_body)?)
 }
 
-#[derive(Default, Deserialize)]
-#[serde(rename_all = "camelCase", deny_unknown_fields)]
-struct SchemaParams {
-    raw: bool,
-}
-
-pub async fn get_index_schema(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(IndexesRead)?;
-
-    let index = ctx.index()?;
-
-    // Tide doesn't support "no query param"
-    let params: SchemaParams = ctx.url_query().unwrap_or_default();
+pub async fn get_update_status(ctx: Request<Data>) -> SResult<Response> {
+    ctx.is_allowed(Private)?;
 
     let db = &ctx.state().db;
-    let reader = db.main_read_txn().map_err(ResponseError::internal)?;
-
-    let schema = index
-        .main
-        .schema(&reader)
-        .map_err(ResponseError::open_index)?;
-
-    match schema {
-        Some(schema) => {
-            if params.raw {
-                Ok(tide::response::json(schema))
-            } else {
-                Ok(tide::response::json(SchemaBody::from(schema)))
-            }
-        }
-        None => Err(ResponseError::not_found("missing index schema")),
-    }
-}
-
-pub async fn update_schema(mut ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(IndexesWrite)?;
-
-    let index_uid = ctx.url_param("index")?;
-
-    let params: SchemaParams = ctx.url_query().unwrap_or_default();
-
-    let schema = if params.raw {
-        ctx.body_json::<SchemaBuilder>()
-            .await
-            .map_err(ResponseError::bad_request)?
-            .build()
-    } else {
-        ctx.body_json::<SchemaBody>()
-            .await
-            .map_err(ResponseError::bad_request)?
-            .into()
-    };
-
-    let db = &ctx.state().db;
-    let mut writer = db.update_write_txn().map_err(ResponseError::internal)?;
-
-    let index = db
-        .open_index(&index_uid)
-        .ok_or(ResponseError::index_not_found(index_uid))?;
-
-    let update_id = index
-        .schema_update(&mut writer, schema.clone())
-        .map_err(ResponseError::internal)?;
-
-    writer.commit().map_err(ResponseError::internal)?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
-}
-
-pub async fn get_update_status(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(IndexesRead)?;
-
-    let db = &ctx.state().db;
-    let reader = db.update_read_txn().map_err(ResponseError::internal)?;
+    let reader = db.update_read_txn()?;
 
     let update_id = ctx
         .param::<u64>("update_id")
         .map_err(|e| ResponseError::bad_parameter("update_id", e))?;
 
     let index = ctx.index()?;
-    let status = index
-        .update_status(&reader, update_id)
-        .map_err(ResponseError::internal)?;
+    let status = index.update_status(&reader, update_id)?;
 
     let response = match status {
-        Some(status) => tide::response::json(status)
-            .with_status(StatusCode::OK)
-            .into_response(),
-        None => tide::response::json(json!({ "message": "unknown update id" }))
-            .with_status(StatusCode::NOT_FOUND)
-            .into_response(),
+        Some(status) => tide::Response::new(200).body_json(&status).unwrap(),
+        None => tide::Response::new(404)
+            .body_json(&json!({ "message": "unknown update id" }))
+            .unwrap(),
     };
 
     Ok(response)
 }
 
-pub async fn get_all_updates_status(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(IndexesRead)?;
-
+pub async fn get_all_updates_status(ctx: Request<Data>) -> SResult<Response> {
+    ctx.is_allowed(Private)?;
     let db = &ctx.state().db;
-    let reader = db.update_read_txn().map_err(ResponseError::internal)?;
-
+    let reader = db.update_read_txn()?;
     let index = ctx.index()?;
-    let all_status = index
-        .all_updates_status(&reader)
-        .map_err(ResponseError::internal)?;
-
-    let response = tide::response::json(all_status)
-        .with_status(StatusCode::OK)
-        .into_response();
-
-    Ok(response)
+    let response = index.all_updates_status(&reader)?;
+    Ok(tide::Response::new(200).body_json(&response).unwrap())
 }
 
-pub async fn delete_index(ctx: Context<Data>) -> SResult<StatusCode> {
-    ctx.is_allowed(IndexesWrite)?;
+pub async fn delete_index(ctx: Request<Data>) -> SResult<Response> {
+    ctx.is_allowed(Private)?;
+    let _ = ctx.index()?;
     let index_uid = ctx.url_param("index")?;
-
-    let found = ctx
-        .state()
-        .db
-        .delete_index(&index_uid)
-        .map_err(ResponseError::internal)?;
-
-    if found {
-        Ok(StatusCode::NO_CONTENT)
-    } else {
-        Ok(StatusCode::NOT_FOUND)
-    }
+    ctx.state().db.delete_index(&index_uid)?;
+    Ok(tide::Response::new(204))
 }
 
 pub fn index_update_callback(index_uid: &str, data: &Data, status: ProcessedUpdateResult) {