Merge pull request #775 from meilisearch/bump-version

Bump Meilisearch to v0.11.0

.dockerignore

@@ -0,0 +1,5 @@
+target
+Dockerfile
+.dockerignore
+.git
+.gitignore

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/is-latest-release.sh

@@ -0,0 +1,132 @@
+#!/bin/sh
+
+# Checks if the current tag should be the latest (in terms of semver and not of release date).
+# Ex: previous tag -> v0.10.1
+#     new tag -> v0.8.12
+#     The new tag should not be the latest
+#     So it returns "false", the CI should not run for the release v0.8.2
+
+# Used in GHA in publish-docker-latest.yml
+# Returns "true" or "false" (as a string) to be used in the `if` in GHA
+
+# GLOBAL
+GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number]
+
+# FUNCTIONS
+
+# semverParseInto and semverLT from https://github.com/cloudflare/semver_bash/blob/master/semver.sh
+
+# usage: semverParseInto version major minor patch special
+# version: the string version
+# major, minor, patch, special: will be assigned by the function
+semverParseInto() {
+    local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)'
+    #MAJOR
+    eval $2=`echo $1 | sed -e "s#$RE#\1#"`
+    #MINOR
+    eval $3=`echo $1 | sed -e "s#$RE#\2#"`
+    #MINOR
+    eval $4=`echo $1 | sed -e "s#$RE#\3#"`
+    #SPECIAL
+    eval $5=`echo $1 | sed -e "s#$RE#\4#"`
+}
+
+# usage: semverLT version1 version2
+semverLT() {
+    local MAJOR_A=0
+    local MINOR_A=0
+    local PATCH_A=0
+    local SPECIAL_A=0
+
+    local MAJOR_B=0
+    local MINOR_B=0
+    local PATCH_B=0
+    local SPECIAL_B=0
+
+    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
+    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
+
+    if [ $MAJOR_A -lt $MAJOR_B ]; then
+        return 0
+    fi
+    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -lt $MINOR_B ]; then
+        return 0
+    fi
+    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then
+        return 0
+    fi
+    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+        return 1
+    fi
+    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  != "_" ] ; then
+        return 1
+    fi
+    if [ "_$SPECIAL_A"  != "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+        return 0
+    fi
+    if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then
+        return 0
+    fi
+
+    return 1
+}
+
+# Returns the tag of the latest stable release (in terms of semver and not of release date)
+get_latest() {
+    temp_file='temp_file' # temp_file needed because the grep would start before the download is over
+    curl -s 'https://api.github.com/repos/meilisearch/MeiliSearch/releases' > "$temp_file"
+    releases=$(cat "$temp_file" | \
+        grep -E "tag_name|draft|prerelease" \
+        | tr -d ',"' | cut -d ':' -f2 | tr -d ' ')
+        # Returns a list of [tag_name draft_boolean prerelease_boolean ...]
+        # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false...
+
+    i=0
+    latest=""
+    current_tag=""
+    for release_info in $releases; do
+        if [ $i -eq 0 ]; then # Cheking tag_name
+            if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release
+                current_tag=$release_info
+            else
+                current_tag=""
+            fi
+            i=1
+        elif [ $i -eq 1 ]; then # Checking draft boolean
+            if [ "$release_info" == "true" ]; then
+                current_tag=""
+            fi
+            i=2
+        elif [ $i -eq 2 ]; then # Checking prerelease boolean
+            if [ "$release_info" == "true" ]; then
+                current_tag=""
+            fi
+            i=0
+            if [ "$current_tag" != "" ]; then # If the current_tag is valid
+                if [ "$latest" = "" ]; then # If there is no latest yet
+                    latest="$current_tag"
+                else
+                    semverLT $current_tag $latest # Comparing latest and the current tag
+                    if [ $? -eq 1 ]; then
+                        latest="$current_tag"
+                    fi
+                fi
+            fi
+        fi
+    done
+
+    rm -f "$temp_file"
+    echo $latest
+}
+
+# MAIN
+current_tag="$(echo $GITHUB_REF | tr -d 'refs/tags/')"
+latest="$(get_latest)"
+
+if [ "$current_tag" != "$latest" ]; then
+    # The current release tag is not the latest
+    echo "false"
+else
+    # The current release tag is the latest
+    echo "true"
+then

.github/workflows/README.md

@@ -0,0 +1,20 @@
+# GitHub Actions Workflow for MeiliSearch
+
+> **Note:**
+
+> - We do not use [cache](https://github.com/actions/cache) yet but we could use it to speed up CI
+
+## Workflow
+
+- On each pull request, we trigger `cargo test`.
+- On each tag, we build:
+    - the tagged Docker image and publish it to Docker Hub
+    - the binaries for MacOS, Ubuntu, and Windows
+    - the Debian package
+- On each stable release (`v*.*.*` tag):
+    - we build the `latest` Docker image and publish it to Docker Hub
+    - we publish the binary to Hombrew and Gemfury
+
+## Problems
+
+- We do not test on Windows because we are unable to make it work, there is a disk space problem.

.github/workflows/check-updated-changelog.yml

@@ -0,0 +1,16 @@
+name: Check if the CHANGELOG.md has been updated
+
+on: [pull_request]
+
+jobs:
+  check:
+    name: Test on ${{ matrix.os }}
+    if: ${{ !contains(github.event.pull_request.labels.*.name, 'ignore-changelog') }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Checking the CHANGELOG.md has been updated in this PR
+      run: |
+        set -e
+        git fetch origin ${{ github.base_ref }}
+        git diff --name-only origin/${{ github.base_ref }} | grep -q CHANGELOG.md

.github/workflows/publish-binaries.yml

@@ -0,0 +1,87 @@
+name: Publish binaries to GitHub release
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  publish:
+    name: Publish for ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        include:
+          - os: ubuntu-latest
+            artifact_name: meilisearch
+            asset_name: meilisearch-linux-amd64
+          - os: macos-latest
+            artifact_name: meilisearch
+            asset_name: meilisearch-macos-amd64
+          - os: windows-latest
+            artifact_name: meilisearch.exe
+            asset_name: meilisearch-windows-amd64.exe
+
+    steps:
+    - uses: hecrj/setup-rust-action@master
+      with:
+        rust-version: stable
+    - uses: actions/checkout@v1
+    - name: Build
+      run: cargo build --release --locked
+    - name: Upload binaries to release
+      uses: svenstaro/upload-release-action@v1-release
+      with:
+        repo_token: ${{ secrets.PUBLISH_TOKEN }}
+        file: target/release/${{ matrix.artifact_name }}
+        asset_name: ${{ matrix.asset_name }}
+        tag: ${{ github.ref }}
+
+  publish-armv7:
+    name: Publish for ARMv7
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v1.0.0
+      - uses: uraimo/run-on-arch-action@v1.0.7
+        id: runcmd
+        with:
+          architecture: armv7
+          distribution: ubuntu18.04
+          run: |
+            apt update
+            apt install -y curl gcc make
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
+            source $HOME/.cargo/env
+            cargo build --release --locked
+      - name: Upload the binary to release
+        uses: svenstaro/upload-release-action@v1-release
+        with:
+          repo_token: ${{ secrets.PUBLISH_TOKEN }}
+          file: target/release/meilisearch
+          asset_name: meilisearch-linux-armv7
+          tag: ${{ github.ref }}
+
+  publish-armv8:
+    name: Publish for ARMv8
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v1.0.0
+      - uses: uraimo/run-on-arch-action@v1.0.7
+        id: runcmd
+        with:
+          architecture: aarch64 # aka ARMv8
+          distribution: ubuntu18.04
+          run: |
+            apt update
+            apt install -y curl gcc make
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
+            source $HOME/.cargo/env
+            cargo build --release --locked
+      - name: Upload the binary to release
+        uses: svenstaro/upload-release-action@v1-release
+        with:
+          repo_token: ${{ secrets.PUBLISH_TOKEN }}
+          file: target/release/meilisearch
+          asset_name: meilisearch-linux-armv8
+          tag: ${{ github.ref }}

.github/workflows/publish-deb-brew-pkg.yml

@@ -0,0 +1,39 @@
+name: Publish deb pkg to GitHub release & APT repository & Homebrew
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  debian:
+    name: Publish debian packagge
+    runs-on: ubuntu-latest
+    steps:
+    - uses: hecrj/setup-rust-action@master
+      with:
+        rust-version: stable
+    - name: Install cargo-deb
+      run: cargo install cargo-deb
+    - uses: actions/checkout@v1
+    - name: Build deb package
+      run: cargo deb -p meilisearch-http -o target/debian/meilisearch.deb
+    - name: Upload debian pkg to release
+      uses: svenstaro/upload-release-action@v1-release
+      with:
+        repo_token: ${{ secrets.GITHUB_TOKEN }}
+        file: target/debian/meilisearch.deb
+        asset_name: meilisearch.deb
+        tag: ${{ github.ref }}
+    - name: Upload debian pkg to apt repository
+      run: curl -F package=@target/debian/meilisearch.deb https://${{ secrets.GEMFURY_PUSH_TOKEN }}@push.fury.io/meilisearch/
+
+  homebrew:
+    name: Bump Homebrew formula
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create PR to Homebrew
+        uses: mislav/bump-homebrew-formula-action@v1
+        with:
+          formula-name: meilisearch
+        env:
+          COMMITTER_TOKEN: ${{ secrets.HOMEBREW_COMMITTER_TOKEN }}

.github/workflows/publish-docker-latest.yml

@@ -0,0 +1,22 @@
+---
+on:
+  release:
+    types: [published]
+
+name: Publish latest image to Docker Hub
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Check if current release is latest
+        run: echo "##[set-output name=is_latest;]$(sh .github/is-latest-release.sh)"
+        id: release
+      - name: Publish to Registry
+        if: steps.release.outputs.is_latest == 'true'
+        uses: elgohr/Publish-Docker-Github-Action@master
+        with:
+          name: getmeili/meilisearch
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/publish-docker-tag.yml

@@ -0,0 +1,20 @@
+---
+on:
+  push:
+    tags:
+      - '*'
+
+name: Publish tagged image to Docker Hub
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Publish to Registry
+        uses: elgohr/Publish-Docker-Github-Action@master
+        with:
+          name: getmeili/meilisearch
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          tag_names: true

.github/workflows/test.yml

@@ -0,0 +1,32 @@
+---
+on: [pull_request]
+
+name: Test binaries with cargo test
+
+jobs:
+  check:
+    name: Test on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+
+    steps:
+    - uses: actions/checkout@v1
+    - uses: actions-rs/toolchain@v1
+      with:
+        profile: minimal
+        toolchain: stable
+        override: true
+    - name: Run cargo test
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --locked --release
+  build-image:
+    name: Test the build of Docker image
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - run: docker build . --file Dockerfile -t meilisearch
+      name: Docker build

.gitignore

@@ -1,7 +1,8 @@
 /target
-Cargo.lock
+meilisearch-core/target
 **/*.csv
 **/*.json_lines
 **/*.rs.bk
 /*.mdb
 /query-history.txt
+/data.ms

CHANGELOG.md

@@ -0,0 +1,43 @@
+## v0.11.0
+
+  - Change the HTTP framework, moving from tide to actix-web (#601)
+  - Bump sentry version to 0.18.1 (#690)
+  - Enable max payload size override (#684)
+  - Disable sentry in debug (#681)
+  - Better terminal greeting (#680)
+  - Fix highlight misalignment (#679)
+  - Add support for facet count (#676)
+  - Add support for faceted search (#631)
+  - Add support for configuring the lmdb map size (#646, #647)
+  - Add exposed port for Dockerfile (#654)
+  - Add sentry probe (#664)
+  - Fix url trailing slash and double slash issues (#659)
+  - Fix accept all Content-Type by default (#653)
+  - Return the error message from Serde when a deserialization error is encountered (#661)
+  - Fix NormalizePath middleware to make the dashboard accessible (#695)
+  - Update sentry features to remove openssl (#702)
+  - Add SSL support (#669)
+  - Rename fieldsFrequency into fieldsDistribution in stats (#719)
+  - Add support for error code reporting (#703)
+  - Allow the dashboard to query private servers (#732)
+  - Add telemetry (#720)
+
+## v0.10.1
+
+  - Add support for floating points in filters (#640)
+  - Add '@' character as tokenizer separator (#607)
+  - Add support for filtering on arrays of strings (#611)
+
+## v0.10.0
+
+  - Refined filtering (#592)
+  - Add the number of hits in search result (#541)
+  - Add support for aligned crop in search result (#543)
+  - Sanitize the content displayed in the web interface (#539)
+  - Add support of nested null, boolean and seq values (#571 and #568, #574)
+  - Fixed the core benchmark (#576)
+  - Publish an ARMv7 and ARMv8 binaries on releases (#540 and #581)
+  - Fixed a bug where the result of the update status after the first update was empty (#542)
+  - Fixed a bug where stop words were not handled correctly (#594)
+  - Fix CORS issues (#602)
+  - Support wildcard on attributes to retrieve, highlight, and crop (#549, #565, and #598)

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at bonjour@meilisearch.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

Cargo.toml

@@ -1,9 +1,10 @@
 [workspace]
 members = [
-    "meilidb-core",
-    "meilidb-http",
-    "meilidb-schema",
-    "meilidb-tokenizer",
+    "meilisearch-core",
+    "meilisearch-http",
+    "meilisearch-schema",
+    "meilisearch-tokenizer",
+    "meilisearch-types",
 ]
 
 [profile.release]

Dockerfile

@@ -0,0 +1,29 @@
+# Compile
+FROM    alpine:3.10 AS compiler
+
+RUN     apk update --quiet
+RUN     apk add curl
+RUN     apk add build-base
+
+RUN     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+
+WORKDIR /meilisearch
+
+COPY    . .
+
+ENV     RUSTFLAGS="-C target-feature=-crt-static"
+
+RUN     $HOME/.cargo/bin/cargo build --release
+
+# Run
+FROM    alpine:3.10
+
+RUN     apk add -q --no-cache libgcc tini
+
+COPY    --from=compiler /meilisearch/target/release/meilisearch .
+
+ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
+EXPOSE  7700/tcp
+
+ENTRYPOINT ["tini", "--"]
+CMD     ./meilisearch

LICENSE

@@ -1,13 +1,21 @@
-“Commons Clause” License Condition v1.0
+MIT License
 
-The Software is provided to you by the Licensor under the License, as defined below, subject to the following condition.
+Copyright (c) 2019-2020 Meili SAS
 
-Without limiting other conditions in the License, the grant of rights under the License will not include, and the License does not grant to you, the right to Sell the Software.
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
 
-For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you under the License to provide to third parties, for a fee or other consideration (including without limitation fees for hosting or consulting/ support services related to the Software), a product or service whose value derives, entirely or substantially, from the functionality of the Software. Any license notice or attribution required by the License must also include this Commons Clause License Condition notice.
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
 
-Software: MeiliDB
-
-License: MIT
-
-Licensor: MEILI SAS
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,79 +1,193 @@
-# MeiliDB
+<p align="center">
+  <img src="assets/logo.svg" alt="MeiliSearch" width="200" height="200" />
+</p>
 
-[![Build Status](https://dev.azure.com/thomas0884/thomas/_apis/build/status/meilisearch.MeiliDB?branchName=master)](https://dev.azure.com/thomas0884/thomas/_build/latest?definitionId=1&branchName=master)
-[![dependency status](https://deps.rs/repo/github/meilisearch/MeiliDB/status.svg)](https://deps.rs/repo/github/meilisearch/MeiliDB)
-[![License](https://img.shields.io/badge/license-commons%20clause-lightgrey)](https://commonsclause.com/)
 
-A _full-text search database_ based on the fast [LMDB key-value store](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database).
+<h1 align="center">MeiliSearch</h1>
+
+<h4 align="center">
+  <a href="https://www.meilisearch.com">Website</a> |
+  <a href="https://blog.meilisearch.com">Blog</a> |
+  <a href="https://fr.linkedin.com/company/meilisearch">LinkedIn</a> |
+  <a href="https://twitter.com/meilisearch">Twitter</a> |
+  <a href="https://docs.meilisearch.com">Documentation</a> |
+  <a href="https://docs.meilisearch.com/faq/">FAQ</a>
+</h4>
+
+<p align="center">
+  <a href="https://github.com/meilisearch/MeiliSearch/actions"><img src="https://github.com/meilisearch/MeiliSearch/workflows/Cargo%20test/badge.svg" alt="Build Status"></a>
+  <a href="https://deps.rs/repo/github/meilisearch/MeiliSearch"><img src="https://deps.rs/repo/github/meilisearch/MeiliSearch/status.svg" alt="Dependency status"></a>
+  <a href="https://github.com/meilisearch/MeiliSearch/blob/master/LICENSE"><img src="https://img.shields.io/badge/license-MIT-informational" alt="License"></a>
+  <a href="https://slack.meilisearch.com"><img src="https://img.shields.io/badge/slack-MeiliSearch-blue.svg?logo=slack" alt="Slack"></a>
+  <a href="https://github.com/meilisearch/MeiliSearch/discussions" alt="Discussions"><img src="https://img.shields.io/badge/github-discussions-red" /></a>
+</p>
+
+<p align="center">⚡ Lightning Fast, Ultra Relevant, and Typo-Tolerant Search Engine 🔍</p>
+
+**MeiliSearch** is a powerful, fast, open-source, easy to use and deploy search engine. Both searching and indexing are highly customizable. Features such as typo-tolerance, filters, and synonyms are provided out-of-the-box.
+For more information about features go to [our documentation](https://docs.meilisearch.com/).
+
+<p align="center">
+  <a href="https://crates.meilisearch.com"><img src="assets/crates-io-demo.gif" alt="crates.io demo gif" /></a>
+</p>
+
+> MeiliSearch helps the Rust community find crates on [crates.meilisearch.com](https://crates.meilisearch.com)
 
 ## Features
+* Search as-you-type experience (answers < 50 milliseconds)
+* Full-text search
+* Typo tolerant (understands typos and miss-spelling)
+* Supports Kanji characters
+* Supports Synonym
+* Easy to install, deploy, and maintain
+* Whole documents are returned
+* Highly customizable
+* RESTful API
 
-- Provides [6 default ranking criteria](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-core/src/criterion/mod.rs#L107-L113) used to [bucket sort](https://en.wikipedia.org/wiki/Bucket_sort) documents
-- Accepts [custom criteria](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-core/src/criterion/mod.rs#L24-L33) and can apply them in any custom order
-- Support [ranged queries](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-core/src/query_builder.rs#L283), useful for paginating results
-- Can [distinct](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-core/src/query_builder.rs#L265-L270) and [filter](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-core/src/query_builder.rs#L246-L259) returned documents based on context defined rules
-- Searches for [concatenated](https://github.com/meilisearch/MeiliDB/pull/164) and [splitted query words](https://github.com/meilisearch/MeiliDB/pull/232) to improve the search quality.
-- Can store complete documents or only [user schema specified fields](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-schema/src/lib.rs#L265-L279)
-- The [default tokenizer](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-tokenizer/src/lib.rs) can index latin and kanji based languages
-- Returns [the matching text areas](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-core/src/lib.rs#L66-L88), useful to highlight matched words in results
-- Accepts query time search config like the [searchable attributes](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-core/src/query_builder.rs#L272-L275)
-- Supports [runtime incremental indexing](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-core/src/store/mod.rs#L143-L173)
+## Get started
 
+### Deploy the Server
 
+#### Run it using Digital Ocean
 
-It uses [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database) as the internal key-value store. The key-value store allows us to handle updates and queries with small memory and CPU overheads. The whole ranking system is [data oriented](https://github.com/meilisearch/MeiliDB/issues/82) and provides great performances.
+[![DigitalOcean Marketplace](assets/do-btn-blue.svg)](https://marketplace.digitalocean.com/apps/meilisearch?action=deploy&refcode=7c67bd97e101)
 
-You can [read the deep dive](deep-dive.md) if you want more information on the engine, it describes the whole process of generating updates and handling queries or you can take a look at the [typos and ranking rules](typos-ranking-rules.md) if you want to know the default rules used to sort the documents.
-
-We will be proud if you submit issues and pull requests. You can help to grow this project and start contributing by checking [issues tagged "good-first-issue"](https://github.com/meilisearch/MeiliDB/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22). It is a good start!
-
-The project is only a library yet. It means that there is no binary provided yet. To get started, you can check the examples wich are made to work with the data located in the `datasets/` folder.
-
-MeiliDB will be a binary in a near future so you will be able to use it as a database out-of-the-box. We should be able to query it using HTTP. This is our current goal, [see the milestones](https://github.com/meilisearch/MeiliDB/milestones). In the end, the binary will be a bunch of network protocols and wrappers around the library - which will also be published on [crates.io](https://crates.io). Both the binary and the library will follow the same update cycle.
-
-
-
-## Performances
-
-With a database composed of _100 353_ documents with _352_ attributes each and _3_ of them indexed.
-So more than _300 000_ fields indexed for _35 million_ stored we can handle more than _2.8k req/sec_ with an average response time of _9 ms_ on an Intel i7-7700 (8) @ 4.2GHz.
-
-Requests are made using [wrk](https://github.com/wg/wrk) and scripted to simulate real users queries.
-
-```
-Running 10s test @ http://localhost:2230
-  2 threads and 25 connections
-  Thread Stats   Avg      Stdev     Max   +/- Stdev
-    Latency     9.52ms    7.61ms  99.25ms   84.58%
-    Req/Sec     1.41k   119.11     1.78k    64.50%
-  28080 requests in 10.01s, 7.42MB read
-Requests/sec:   2806.46
-Transfer/sec:    759.17KB
-```
-
-### Notes
-
-With Rust 1.32 the allocator has been [changed to use the system allocator](https://blog.rust-lang.org/2019/01/17/Rust-1.32.0.html#jemalloc-is-removed-by-default).
-We have seen much better performances when [using jemalloc as the global allocator](https://github.com/alexcrichton/jemallocator#documentation).
-
-## Usage and examples
-
-Currently MeiliDB do not provide an http server but you can run the example binary.
-
-The _index_ subcommand has been made to create an index and inject documents into it. Using the command line below, the index will be named _movies_ and the _19 700_ movies of the `datasets/` will be injected in MeiliDB.
+#### Run it using Docker
 
 ```bash
-cargo run --release --example from_file -- \
-    index example.mdb datasets/movies/data.csv \
-    --schema datasets/movies/schema.toml
+docker run -p 7700:7700 -v $(pwd)/data.ms:/data.ms getmeili/meilisearch
 ```
 
-Once the first command is done, you can query the freshly created _movies_ index using the _search_ subcomand. In this example we filtered the dataset to only show _non-adult_ movies using the non-definitive `!adult` syntax filter.
+#### Installing with Homebrew
 
 ```bash
-cargo run --release --example from_file -- \
-    search example.mdb
-    --number 4 \
-    --filter '!adult' \
-    id popularity adult original_title
+brew update && brew install meilisearch
+meilisearch
 ```
+
+#### Installing with APT
+
+```bash
+echo "deb [trusted=yes] https://apt.fury.io/meilisearch/ /" > /etc/apt/sources.list.d/fury.list
+apt update && apt install meilisearch-http
+meilisearch
+```
+
+#### Download the binary
+
+```bash
+curl -L https://install.meilisearch.com | sh
+./meilisearch
+```
+
+#### Compile and run it from sources
+
+If you have the Rust toolchain already installed on your local system, clone the repository and change it to your working directory.
+
+```bash
+git clone https://github.com/meilisearch/MeiliSearch.git
+cd MeiliSearch
+```
+
+In the cloned repository, compile MeiliSearch.
+
+```bash
+cargo run --release
+```
+
+### Create an Index and Upload Some Documents
+
+Let's create an index! If you need a sample dataset, use [this movie database](https://www.notion.so/meilisearch/A-movies-dataset-to-test-Meili-1cbf7c9cfa4247249c40edfa22d7ca87#b5ae399b81834705ba5420ac70358a65). You can also find it in the `datasets/` directory.
+
+```bash
+curl -L 'https://bit.ly/2PAcw9l' -o movies.json
+```
+
+MeiliSearch can serve multiple indexes, with different kinds of documents.
+It is required to create an index before sending documents to it.
+
+```bash
+curl -i -X POST 'http://127.0.0.1:7700/indexes' --data '{ "name": "Movies", "uid": "movies" }'
+```
+
+Now that the server knows about your brand new index, you're ready to send it some data.
+
+```bash
+curl -i -X POST 'http://127.0.0.1:7700/indexes/movies/documents' \
+  --header 'content-type: application/json' \
+  --data-binary @movies.json
+```
+
+### Search for Documents
+
+#### In command line
+
+The search engine is now aware of your documents and can serve those via an HTTP server.
+
+The [`jq` command-line tool](https://stedolan.github.io/jq/) can greatly help you read the server responses.
+
+```bash
+curl 'http://127.0.0.1:7700/indexes/movies/search?q=botman+robin&limit=2' | jq
+```
+
+```json
+{
+  "hits": [
+    {
+      "id": "415",
+      "title": "Batman & Robin",
+      "poster": "https://image.tmdb.org/t/p/w1280/79AYCcxw3kSKbhGpx1LiqaCAbwo.jpg",
+      "overview": "Along with crime-fighting partner Robin and new recruit Batgirl...",
+      "release_date": "1997-06-20",
+    },
+    {
+      "id": "411736",
+      "title": "Batman: Return of the Caped Crusaders",
+      "poster": "https://image.tmdb.org/t/p/w1280/GW3IyMW5Xgl0cgCN8wu96IlNpD.jpg",
+      "overview": "Adam West and Burt Ward returns to their iconic roles of Batman and Robin...",
+      "release_date": "2016-10-08",
+    }
+  ],
+  "offset": 0,
+  "limit": 2,
+  "processingTimeMs": 1,
+  "query": "botman robin"
+}
+```
+
+#### Use the Web Interface
+
+We also deliver an **out-of-the-box web interface** in which you can test MeiliSearch interactively.
+
+You can access the web interface in your web browser at the root of the server. The default URL is [http://127.0.0.1:7700](http://127.0.0.1:7700). All you need to do is open your web browser and enter MeiliSearch’s address to visit it. This will lead you to a web page with a search bar that will allow you to search in the selected index.
+
+<p align="center">
+  <img src="assets/movies-web-demo.gif" alt="Web interface gif" />
+</p>
+
+### Documentation
+
+Now that your MeiliSearch server is up and running, you can learn more about how to tune your search engine in [the documentation](https://docs.meilisearch.com).
+
+
+## Contributing
+
+Hey! We're glad you're thinking about contributing to MeiliSearch! If you think something is missing or could be improved, please open issues and pull requests. If you'd like to help this project grow, we'd love to have you! To start contributing, checking [issues tagged as "good-first-issue"](https://github.com/meilisearch/MeiliSearch/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) is a good start!
+
+### Analytic Events
+
+Every hour, events are being sent to our Amplitude instance so we can know how many people are using MeiliSearch.<br/>
+To see what information we're retrieving, please see the complete list [on the dedicated issue](https://github.com/meilisearch/MeiliSearch/issues/720).<br/>
+We also use Sentry to make us crash and error reports. If you want to know more about what Sentry collects, please visit their [privacy policy website](https://sentry.io/privacy/).<br/>
+If this doesn't suit you, you can disable these analytics by using the `MEILI_NO_ANALYTICS` env variable.
+
+## Contact
+
+Feel free to contact us about any questions you may have:
+* At [bonjour@meilisearch.com](mailto:bonjour@meilisearch.com): English or French is welcome! 🇬🇧 🇫🇷
+* Via the chat box available on every page of [our documentation](https://docs.meilisearch.com/) and on [our landing page](https://www.meilisearch.com/).
+* 🆕 Join our [GitHub Discussions forum](https://github.com/meilisearch/MeiliSearch/discussions) (BETA hype!)
+* Join our [Slack community](https://slack.meilisearch.com/).
+* By opening an issue.
+
+Any suggestion or feedback is highly appreciated. Thank you for your support!

assets/do-btn-blue.svg

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="200px" height="42px" viewBox="0 0 200 42" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 52.5 (67469) - http://www.bohemiancoding.com/sketch -->
+    <title>do-btn-blue</title>
+    <desc>Created with Sketch.</desc>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="Partner-welcome-kit-Copy-3" transform="translate(-651.000000, -762.000000)">
+            <g id="do-btn-blue" transform="translate(651.000000, 763.000000)">
+                <rect id="Rectangle-Copy" fill="#0069FF" x="0" y="0" width="200" height="40" rx="6"></rect>
+                <path d="M45,0 L45,40" id="Line-2" stroke="#FFFFFF" stroke-linecap="square"></path>
+                <g id="DO_Logo_horizontal_blue-Copy" transform="translate(13.000000, 11.000000)" fill="#FFFFFF">
+                    <path d="M10.0098493,20 L10.0098493,16.1262429 C14.12457,16.1262429 17.2897398,12.0548452 15.7269372,7.74627862 C15.1334679,6.14538921 13.8674,4.86072487 12.2650328,4.28756693 C7.952489,2.72620566 3.87733294,5.88845634 3.87733294,9.99938223 C3.87733294,9.99938223 3.87733294,9.99938223 3.87733294,9.99938223 L0,9.99938223 C0,3.45747613 6.3303395,-1.64165309 13.1948014,0.492866119 C16.2017127,1.42177726 18.57559,3.81322933 19.5053586,6.79760341 C21.6418482,13.6754986 16.5577943,20 10.0098493,20 Z" id="XMLID_49_"></path>
+                    <polygon id="XMLID_47_" points="9.56521739 15.6521739 6.08695652 15.6521739 6.08695652 12.173913 6.08695652 12.173913 9.56521739 12.173913 9.56521739 12.173913"></polygon>
+                    <polygon id="XMLID_46_" points="6.08695652 19.1304348 3.47826087 19.1304348 3.47826087 19.1304348 3.47826087 16.5217391 6.08695652 16.5217391"></polygon>
+                    <polygon id="XMLID_45_" points="3.47826087 16.5217391 0.869565217 16.5217391 0.869565217 16.5217391 0.869565217 13.9130435 0.869565217 13.9130435 3.47826087 13.9130435 3.47826087 13.9130435"></polygon>
+                </g>
+                <text id="Create-a-Droplet-Copy" font-family="Sailec-Medium, Sailec" font-size="16" font-weight="400" fill="#FFFFFF">
+                    <tspan x="58" y="26">Create a Droplet</tspan>
+                </text>
+            </g>
+        </g>
+    </g>
+</svg>

assets/logo.svg

@@ -0,0 +1,17 @@
+<svg width="360" height="360" viewBox="0 0 360 360" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g id="logo_main">
+<rect id="Rectangle" x="107.333" y="0.150146" width="274.315" height="274.315" rx="98.8334" transform="rotate(23 107.333 0.150146)" fill="url(#paint0_linear)"/>
+<path id="Rectangle_2" fill-rule="evenodd" clip-rule="evenodd" d="M61.3296 230.199C46.2224 194.608 38.6688 176.813 38.208 160.329C37.5286 136.025 47.0175 112.539 64.3891 95.5282C76.1718 83.9904 93.9669 76.4368 129.557 61.3296C165.147 46.2224 182.943 38.6688 199.427 38.208C223.731 37.5286 247.217 47.0175 264.228 64.3891C275.766 76.1718 283.319 93.9669 298.426 129.557C313.534 165.147 321.087 182.943 321.548 199.427C322.227 223.731 312.738 247.217 295.367 264.228C283.584 275.766 265.789 283.319 230.199 298.426C194.608 313.534 176.813 321.087 160.329 321.548C136.025 322.227 112.539 312.738 95.5282 295.367C83.9903 283.584 76.4368 265.789 61.3296 230.199Z" fill="url(#paint1_linear)"/>
+<path id="m" fill-rule="evenodd" clip-rule="evenodd" d="M219.568 130.748C242.363 130.748 259.263 147.451 259.263 174.569V229.001H227.232V179.678C227.232 166.119 220.747 159.634 210.136 159.634C205.223 159.634 200.311 161.796 195.595 167.494C195.791 169.852 195.988 172.21 195.988 174.569V229.001H164.154V179.678C164.154 166.119 157.472 159.634 147.057 159.634C142.145 159.634 137.429 161.992 132.712 168.084V229.001H100.878V133.695H132.712V139.394C139.197 133.892 145.878 130.748 156.49 130.748C168.477 130.748 178.695 135.267 185.769 143.52C195.791 134.678 205.42 130.748 219.568 130.748Z" fill="white"/>
+</g>
+<defs>
+<linearGradient id="paint0_linear" x1="-13.6248" y1="129.208" x2="244.49" y2="403.522" gradientUnits="userSpaceOnUse">
+<stop stop-color="#E41359"/>
+<stop offset="1" stop-color="#F23C79"/>
+</linearGradient>
+<linearGradient id="paint1_linear" x1="11.0088" y1="111.65" x2="111.65" y2="348.747" gradientUnits="userSpaceOnUse">
+<stop stop-color="#24222F"/>
+<stop offset="1" stop-color="#2B2937"/>
+</linearGradient>
+</defs>
+</svg>

azure-pipelines.yml

@@ -1,52 +0,0 @@
----
-trigger:
-  branches:
-    include: [ master ]
-
-pr: [ master ]
-
-jobs:
-  - job: test
-    pool:
-      vmImage: 'Ubuntu 16.04'
-    container: tpayet/chiquitita:latest
-    steps:
-      - script: |
-          curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain nightly
-          $HOME/.cargo/bin/rustup component add rustfmt
-        displayName: 'Install rustc and components'
-      - script: |
-          $HOME/.cargo/bin/cargo check
-        displayName: 'Check MeiliDB'
-      - script: |
-          $HOME/.cargo/bin/cargo test
-        displayName: 'Test MeiliDB'
-      - script: |
-          $HOME/.cargo/bin/cargo fmt --all -- --check
-        displayName: 'Fmt MeiliDB'
-
-  - job: build
-    dependsOn:
-      - test
-    condition: succeeded()
-    pool:
-      vmImage: 'Ubuntu 16.04'
-    container: tpayet/chiquitita:latest
-    steps:
-      - script: |
-          curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain nightly
-          $HOME/.cargo/bin/rustup component add rustfmt
-        displayName: 'Install rustc and components'
-      - script: |
-          $HOME/.cargo/bin/cargo build --release
-        displayName: 'Build MeiliDB'
-      - task: CopyFiles@2
-        inputs:
-          contents: '$(System.DefaultWorkingDirectory)/target/release/libmeilidb.rlib'
-          targetFolder: $(Build.ArtifactStagingDirectory)
-        displayName: 'Copy build'
-      - task: PublishBuildArtifacts@1
-        inputs:
-          artifactName: libmeilidb.rlib
-        displayName: 'Upload artifacts'
-

datasets/movies/README.md

@@ -1 +1 @@
-_datas in movies.csv are from https://www.themoviedb.org/_
+_datas in movies.csv are from https://www.themoviedb.org/_

datasets/movies/schema-movies.toml

@@ -1,21 +0,0 @@
-# This schema has been generated ...
-# The order in which the attributes are declared is important,
-# it specify the attribute xxx...
-identifier = "id"
-
-[attributes.id]
-displayed = true
-
-[attributes.title]
-displayed = true
-indexed = true
-
-[attributes.overview]
-displayed = true
-indexed = true
-
-[attributes.release_date]
-displayed = true
-
-[attributes.poster]
-displayed = true

datasets/movies/settings.json

@@ -0,0 +1,10 @@
+{
+    "searchableAttributes": ["title", "overview"],
+    "displayedAttributes": [
+        "id",
+        "title",
+        "overview",
+        "release_date",
+        "poster"
+    ]
+}

deep-dive.md

@@ -1,95 +0,0 @@
-# A deep dive in MeiliDB
-
-On the 15 of May 2019.
-
-MeiliDB is a full text search engine based on a final state transducer named [fst](https://github.com/BurntSushi/fst) and a key-value store named [sled](https://github.com/spacejam/sled). The goal of a search engine is to store data and to respond to queries as accurate and fast as possible. To achieve this it must save the matching words in an [inverted index](https://en.wikipedia.org/wiki/Inverted_index).
-
-<!-- MarkdownTOC autolink="true" -->
-
-- [Where is the data stored?](#where-is-the-data-stored)
-- [What does the key-value store contains?](#what-does-the-key-value-store-contains)
-    - [The inverted word index](#the-inverted-word-index)
-        - [A final state transducer](#a-final-state-transducer)
-        - [Document indexes](#document-indexes)
-    - [The schema](#the-schema)
-    - [Document attributes](#document-attributes)
-- [How is a request processed?](#how-is-a-request-processed)
-    - [Query lexemes](#query-lexemes)
-    - [Automatons and query index](#automatons-and-query-index)
-    - [Sort by criteria](#sort-by-criteria)
-
-<!-- /MarkdownTOC -->
-
-## Where is the data stored?
-
-MeiliDB is entirely backed by a key-value store like any good database (i.e. Postgres, MySQL). This brings a great flexibility in the way documents can be stored and updates handled along time.
-
-[sled will brings some](https://github.com/spacejam/sled/tree/434533332a3f485e6d2e467023be0a0b55d3a1af#plans) of the [A.C.I.D. properties](https://en.wikipedia.org/wiki/ACID_(computer_science)) to help us be sure the saved data is consistent.
-
-
-
-## What does the key-value store contains?
-
-It contain the inverted word index, the schema and the documents fields.
-
-### The inverted word index
-
-[The inverted word index](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-data/src/database/words_index.rs) is a sled Tree dedicated to store and give access to all documents that contains a specific word. The information stored under the word is simply a big ordered array of where in the document the word has been found. In other word, a big list of [`DocIndex`](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-core/src/lib.rs#L35-L51).
-
-#### A final state transducer
-
-_...also abbreviated fst_
-
-This is the first entry point of the engine, you can read more about how it work with the beautiful blog post of @BurntSushi, [Index 1,600,000,000 Keys with Automata and Rust](https://blog.burntsushi.net/transducers/).
-
-To make it short it is a powerful way to store all the words that are present in the indexed documents. You construct it by giving it all the words you want to index. When you want to search in it you can provide any automaton you want, in MeiliDB [a custom levenshtein automaton](https://github.com/tantivy-search/levenshtein-automata/) is used.
-
-#### Document indexes
-
-The `fst` will only return the words that match with the search automaton but the goal of the search engine is to retrieve all matches in all the documents when a query is made. You want it to return some sort of position in an attribute in a document, an information about where the given word matched.
-
-To make it possible we retrieve all of the `DocIndex` corresponding to all the matching words in the fst, we use the [`WordsIndex`](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-data/src/database/words_index.rs#L11-L21) Tree to get the `DocIndexes` corresponding the words.
-
-### The schema
-
-The schema is a data structure that represents which documents attributes should be stored and which should be indexed. It is stored under a the [`MainIndex`](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-data/src/database/main_index.rs#L12) Tree and given to MeiliDB only at the creation of an index.
-
-Each document attribute is associated to a unique 16 bit number named [`SchemaAttr`](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-data/src/schema.rs#L186).
-
-In the future, this schema type could be given along with updates, the database could be able to handled a new schema and reindex the database according to the new one.
-
-### Document attributes
-
-When the engine handle a query the result that the requester want is a document, not only the [`Matches`](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-core/src/lib.rs#L62-L88) associated to it, fields of the original document must be returned too.
-
-So MeiliDB again uses the power of the underlying key-value store and save the documents attributes marked as _STORE_ in the schema. The dedicated Tree for this information is the [`DocumentsIndex`](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-data/src/database/documents_index.rs#L11).
-
-When a document field is saved in the key-value store its value is binary encoded using [message pack](https://github.com/3Hren/msgpack-rust), so a document must be serializable using serde.
-
-
-
-## How is a request processed?
-
-Now that we have our inverted index we are able to return results based on a query. In the MeiliDB universe a query is a simple string containing words.
-
-### Query lexemes
-
-The first step to be able to call the underlying structures is to split the query in words, for that we use a [custom tokenizer](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-tokenizer/src/lib.rs#L82-L84). Note that a tokenizer is specialized for a human language, this is the hard part.
-
-### Automatons and query index
-
-So to query the fst we need an automaton, in MeiliDB we use a [levenshtein automaton](https://en.wikipedia.org/wiki/Levenshtein_automaton), this automaton is constructed using a string and a maximum distance. According to the [Algolia's blog post](https://blog.algolia.com/inside-the-algolia-engine-part-3-query-processing/#algolia%e2%80%99s-way-of-searching-for-alternatives) we [created the DFAs](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-core/src/automaton.rs#L59-L78) with different settings.
-
-Thanks to the power of the fst library [it is possible to union multiple automatons](https://docs.rs/fst/0.3.2/fst/map/struct.OpBuilder.html#method.union) on the same fst set. The `Stream` is able to return all the matching words. We use these words to find the whole list of `DocIndexes` associated.
-
-With all these informations it is possible [to reconstruct a list of all the `DocIndexes` associated](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-core/src/query_builder.rs#L103-L130) with the words queried.
-
-### Sort by criteria
-
-Now that we are able to get a big list of [DocIndexes](https://github.com/Kerollmops/MeiliDB/blob/550dc1e99224e386516877450320f694947332d4/src/lib.rs#L21-L36) it is not enough to sort them by criteria, we need more informations like the levenshtein distance or the fact that a query word match exactly the word stored in the fst. So [we stuff it a little bit](https://github.com/Kerollmops/MeiliDB/blob/550dc1e99224e386516877450320f694947332d4/src/rank/query_builder.rs#L86-L93), and aggregate all these [Matches](https://github.com/Kerollmops/MeiliDB/blob/550dc1e99224e386516877450320f694947332d4/src/lib.rs#L47-L74) for each document. This way it will be easy to sort a simple vector of document using a bunch of functions.
-
-With this big list of documents and associated matches [we are able to sort only the part of the slice that we want](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-core/src/query_builder.rs#L160-L188) using bucket sorting. [Each criterion](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-core/src/criterion/mod.rs#L95-L101) is evaluated on each subslice without copy, thanks to [GroupByMut](https://docs.rs/slice-group-by/0.2.4/slice_group_by/) which, I hope [will soon be merged](https://github.com/rust-lang/rfcs/pull/2477).
-
-Note that it is possible to customize the criteria used by using the `QueryBuilder::with_criteria` constructor, this way you can implement some custom ranking based on the document attributes using the appropriate structure and the [`document` method](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-data/src/database/index.rs#L86).
-
-At this point, MeiliDB work is over 🎉

download-latest.sh

@@ -0,0 +1,187 @@
+#!/bin/sh
+
+# COLORS
+RED='\033[31m'
+GREEN='\033[32m'
+DEFAULT='\033[0m'
+
+# GLOBALS
+BINARY_NAME='meilisearch'
+GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number]
+
+# FUNCTIONS
+
+# semverParseInto and semverLT from https://github.com/cloudflare/semver_bash/blob/master/semver.sh
+
+# usage: semverParseInto version major minor patch special
+# version: the string version
+# major, minor, patch, special: will be assigned by the function
+semverParseInto() {
+    local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)'
+    #MAJOR
+    eval $2=`echo $1 | sed -e "s#$RE#\1#"`
+    #MINOR
+    eval $3=`echo $1 | sed -e "s#$RE#\2#"`
+    #MINOR
+    eval $4=`echo $1 | sed -e "s#$RE#\3#"`
+    #SPECIAL
+    eval $5=`echo $1 | sed -e "s#$RE#\4#"`
+}
+
+# usage: semverLT version1 version2
+semverLT() {
+    local MAJOR_A=0
+    local MINOR_A=0
+    local PATCH_A=0
+    local SPECIAL_A=0
+
+    local MAJOR_B=0
+    local MINOR_B=0
+    local PATCH_B=0
+    local SPECIAL_B=0
+
+    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
+    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
+
+    if [ $MAJOR_A -lt $MAJOR_B ]; then
+        return 0
+    fi
+    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -lt $MINOR_B ]; then
+        return 0
+    fi
+    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then
+        return 0
+    fi
+    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+        return 1
+    fi
+    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  != "_" ] ; then
+        return 1
+    fi
+    if [ "_$SPECIAL_A"  != "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+        return 0
+    fi
+    if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then
+        return 0
+    fi
+
+    return 1
+}
+
+# Returns the tag of the latest stable release (in terms of semver and not of release date)
+get_latest() {
+    temp_file='temp_file' # temp_file needed because the grep would start before the download is over
+    curl -s 'https://api.github.com/repos/meilisearch/MeiliSearch/releases' > "$temp_file"
+    releases=$(cat "$temp_file" | \
+        grep -E "tag_name|draft|prerelease" \
+        | tr -d ',"' | cut -d ':' -f2 | tr -d ' ')
+        # Returns a list of [tag_name draft_boolean prerelease_boolean ...]
+        # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false...
+
+    i=0
+    latest=""
+    current_tag=""
+    for release_info in $releases; do
+        if [ $i -eq 0 ]; then # Cheking tag_name
+            if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release
+                current_tag=$release_info
+            else
+                current_tag=""
+            fi
+            i=1
+        elif [ $i -eq 1 ]; then # Checking draft boolean
+            if [ "$release_info" = "true" ]; then
+                current_tag=""
+            fi
+            i=2
+        elif [ $i -eq 2 ]; then # Checking prerelease boolean
+            if [ "$release_info" = "true" ]; then
+                current_tag=""
+            fi
+            i=0
+            if [ "$current_tag" != "" ]; then # If the current_tag is valid
+                if [ "$latest" = "" ]; then # If there is no latest yet
+                    latest="$current_tag"
+                else
+                    semverLT $current_tag $latest # Comparing latest and the current tag
+                    if [ $? -eq 1 ]; then
+                        latest="$current_tag"
+                    fi
+                fi
+            fi
+        fi
+    done
+
+    rm -f "$temp_file"
+    echo $latest
+}
+
+# Gets the OS by setting the $os variable
+# Returns 0 in case of success, 1 otherwise.
+get_os() {
+    os_name=$(uname -s)
+    case "$os_name" in
+    'Darwin')
+        os='macos'
+        ;;
+    'Linux')
+        os='linux'
+        ;;
+    *)
+        return 1
+    esac
+    return 0
+}
+
+# Gets the architecture by setting the $archi variable
+# Returns 0 in case of success, 1 otherwise.
+get_archi() {
+    architecture=$(uname -m)
+    case "$architecture" in
+    'x86_64' | 'amd64')
+        archi='amd64'
+        ;;
+    'aarch64')
+        archi='armv8'
+        ;;
+    *)
+        return 1
+    esac
+    return 0
+}
+
+success_usage() {
+    printf "$GREEN%s\n$DEFAULT" "MeiliSearch binary successfully downloaded as '$BINARY_NAME' file."
+    echo ''
+    echo 'Run it:'
+    echo '    $ ./meilisearch'
+    echo 'Usage:'
+    echo '    $ ./meilisearch --help'
+}
+
+failure_usage() {
+    printf "$RED%s\n$DEFAULT" 'ERROR: MeiliSearch binary is not available for your OS distribution or your architecture yet.'
+    echo ''
+    echo 'However, you can easily compile the binary from the source files.'
+    echo 'Follow the steps at the page ("Source" tab): https://docs.meilisearch.com/guides/advanced_guides/installation.html'
+}
+
+# MAIN
+latest="$(get_latest)"
+get_os
+if [ "$?" -eq 1 ]; then
+    failure_usage
+    exit 1
+fi
+get_archi
+if [ "$?" -eq 1 ]; then
+    failure_usage
+    exit 1
+fi
+echo "Downloading MeiliSearch binary $latest for $os, architecture $archi..."
+release_file="meilisearch-$os-$archi"
+link="https://github.com/meilisearch/MeiliSearch/releases/download/$latest/$release_file"
+curl -OL "$link"
+mv "$release_file" "$BINARY_NAME"
+chmod 744 "$BINARY_NAME"
+success_usage

meilidb-core/Cargo.toml

@@ -1,45 +0,0 @@
-[package]
-name = "meilidb-core"
-version = "0.6.2"
-authors = ["Kerollmops <clement@meilisearch.com>"]
-edition = "2018"
-
-[dependencies]
-arc-swap = "0.4.3"
-bincode = "1.1.4"
-byteorder = "1.3.2"
-crossbeam-channel = "0.3.9"
-deunicode = "1.0.0"
-env_logger = "0.7.0"
-hashbrown = { version = "0.6.0", features = ["serde"] }
-heed = "0.5.0"
-log = "0.4.8"
-meilidb-schema = { path = "../meilidb-schema", version = "0.6.0" }
-meilidb-tokenizer = { path = "../meilidb-tokenizer", version = "0.6.0" }
-once_cell = "1.2.0"
-ordered-float = { version = "1.0.2", features = ["serde"] }
-sdset = "0.3.3"
-serde = { version = "1.0.101", features = ["derive"] }
-serde_json = "1.0.41"
-siphasher = "0.3.0"
-slice-group-by = "0.2.6"
-zerocopy = "0.2.8"
-
-[dependencies.levenshtein_automata]
-git = "https://github.com/Kerollmops/levenshtein-automata.git"
-branch = "arc-byte-slice"
-features = ["fst_automaton"]
-
-[dependencies.fst]
-git = "https://github.com/Kerollmops/fst.git"
-branch = "arc-byte-slice"
-
-[dev-dependencies]
-assert_matches = "1.3"
-csv = "1.0.7"
-indexmap = { version = "1.2.0", features = ["serde-1"] }
-rustyline = { version = "5.0.0", default-features = false }
-structopt = "0.3.2"
-tempfile = "3.1.0"
-termcolor = "1.0.4"
-toml = "0.5.3"

meilidb-core/src/automaton/mod.rs

@@ -1,295 +0,0 @@
-mod dfa;
-mod query_enhancer;
-
-use std::cmp::Reverse;
-use std::{cmp, vec};
-
-use fst::{IntoStreamer, Streamer};
-use levenshtein_automata::DFA;
-use meilidb_tokenizer::{is_cjk, split_query_string};
-
-use crate::error::MResult;
-use crate::store;
-
-use self::dfa::{build_dfa, build_prefix_dfa};
-pub use self::query_enhancer::QueryEnhancer;
-use self::query_enhancer::QueryEnhancerBuilder;
-
-const NGRAMS: usize = 3;
-
-pub struct AutomatonProducer {
-    automatons: Vec<AutomatonGroup>,
-}
-
-impl AutomatonProducer {
-    pub fn new(
-        reader: &heed::RoTxn,
-        query: &str,
-        main_store: store::Main,
-        postings_list_store: store::PostingsLists,
-        synonyms_store: store::Synonyms,
-    ) -> MResult<(AutomatonProducer, QueryEnhancer)> {
-        let (automatons, query_enhancer) = generate_automatons(
-            reader,
-            query,
-            main_store,
-            postings_list_store,
-            synonyms_store,
-        )?;
-
-        Ok((AutomatonProducer { automatons }, query_enhancer))
-    }
-
-    pub fn into_iter(self) -> vec::IntoIter<AutomatonGroup> {
-        self.automatons.into_iter()
-    }
-}
-
-#[derive(Debug)]
-pub struct AutomatonGroup {
-    pub is_phrase_query: bool,
-    pub automatons: Vec<Automaton>,
-}
-
-impl AutomatonGroup {
-    fn normal(automatons: Vec<Automaton>) -> AutomatonGroup {
-        AutomatonGroup {
-            is_phrase_query: false,
-            automatons,
-        }
-    }
-
-    fn phrase_query(automatons: Vec<Automaton>) -> AutomatonGroup {
-        AutomatonGroup {
-            is_phrase_query: true,
-            automatons,
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct Automaton {
-    pub index: usize,
-    pub ngram: usize,
-    pub query_len: usize,
-    pub is_exact: bool,
-    pub is_prefix: bool,
-    pub query: String,
-}
-
-impl Automaton {
-    pub fn dfa(&self) -> DFA {
-        if self.is_prefix {
-            build_prefix_dfa(&self.query)
-        } else {
-            build_dfa(&self.query)
-        }
-    }
-
-    fn exact(index: usize, ngram: usize, query: &str) -> Automaton {
-        Automaton {
-            index,
-            ngram,
-            query_len: query.len(),
-            is_exact: true,
-            is_prefix: false,
-            query: query.to_string(),
-        }
-    }
-
-    fn prefix_exact(index: usize, ngram: usize, query: &str) -> Automaton {
-        Automaton {
-            index,
-            ngram,
-            query_len: query.len(),
-            is_exact: true,
-            is_prefix: true,
-            query: query.to_string(),
-        }
-    }
-
-    fn non_exact(index: usize, ngram: usize, query: &str) -> Automaton {
-        Automaton {
-            index,
-            ngram,
-            query_len: query.len(),
-            is_exact: false,
-            is_prefix: false,
-            query: query.to_string(),
-        }
-    }
-}
-
-pub fn normalize_str(string: &str) -> String {
-    let mut string = string.to_lowercase();
-
-    if !string.contains(is_cjk) {
-        string = deunicode::deunicode_with_tofu(&string, "");
-    }
-
-    string
-}
-
-fn split_best_frequency<'a>(
-    reader: &heed::RoTxn,
-    word: &'a str,
-    postings_lists_store: store::PostingsLists,
-) -> MResult<Option<(&'a str, &'a str)>> {
-    let chars = word.char_indices().skip(1);
-    let mut best = None;
-
-    for (i, _) in chars {
-        let (left, right) = word.split_at(i);
-
-        let left_freq = postings_lists_store
-            .postings_list(reader, left.as_ref())?
-            .map_or(0, |i| i.len());
-
-        let right_freq = postings_lists_store
-            .postings_list(reader, right.as_ref())?
-            .map_or(0, |i| i.len());
-
-        let min_freq = cmp::min(left_freq, right_freq);
-        if min_freq != 0 && best.map_or(true, |(old, _, _)| min_freq > old) {
-            best = Some((min_freq, left, right));
-        }
-    }
-
-    Ok(best.map(|(_, l, r)| (l, r)))
-}
-
-fn generate_automatons(
-    reader: &heed::RoTxn,
-    query: &str,
-    main_store: store::Main,
-    postings_lists_store: store::PostingsLists,
-    synonym_store: store::Synonyms,
-) -> MResult<(Vec<AutomatonGroup>, QueryEnhancer)> {
-    let has_end_whitespace = query.chars().last().map_or(false, char::is_whitespace);
-    let query_words: Vec<_> = split_query_string(query).map(str::to_lowercase).collect();
-    let synonyms = match main_store.synonyms_fst(reader)? {
-        Some(synonym) => synonym,
-        None => fst::Set::default(),
-    };
-
-    let mut automaton_index = 0;
-    let mut automatons = Vec::new();
-    let mut enhancer_builder = QueryEnhancerBuilder::new(&query_words);
-
-    // We must not declare the original words to the query enhancer
-    // *but* we need to push them in the automatons list first
-    let mut original_automatons = Vec::new();
-    let mut original_words = query_words.iter().peekable();
-    while let Some(word) = original_words.next() {
-        let has_following_word = original_words.peek().is_some();
-        let not_prefix_dfa = has_following_word || has_end_whitespace || word.chars().all(is_cjk);
-
-        let automaton = if not_prefix_dfa {
-            Automaton::exact(automaton_index, 1, word)
-        } else {
-            Automaton::prefix_exact(automaton_index, 1, word)
-        };
-        automaton_index += 1;
-        original_automatons.push(automaton);
-    }
-
-    automatons.push(AutomatonGroup::normal(original_automatons));
-
-    for n in 1..=NGRAMS {
-        let mut ngrams = query_words.windows(n).enumerate().peekable();
-        while let Some((query_index, ngram_slice)) = ngrams.next() {
-            let query_range = query_index..query_index + n;
-            let ngram_nb_words = ngram_slice.len();
-            let ngram = ngram_slice.join(" ");
-
-            let has_following_word = ngrams.peek().is_some();
-            let not_prefix_dfa =
-                has_following_word || has_end_whitespace || ngram.chars().all(is_cjk);
-
-            // automaton of synonyms of the ngrams
-            let normalized = normalize_str(&ngram);
-            let lev = if not_prefix_dfa {
-                build_dfa(&normalized)
-            } else {
-                build_prefix_dfa(&normalized)
-            };
-
-            let mut stream = synonyms.search(&lev).into_stream();
-            while let Some(base) = stream.next() {
-                // only trigger alternatives when the last word has been typed
-                // i.e. "new " do not but "new yo" triggers alternatives to "new york"
-                let base = std::str::from_utf8(base).unwrap();
-                let base_nb_words = split_query_string(base).count();
-                if ngram_nb_words != base_nb_words {
-                    continue;
-                }
-
-                if let Some(synonyms) = synonym_store.synonyms(reader, base.as_bytes())? {
-                    let mut stream = synonyms.into_stream();
-                    while let Some(synonyms) = stream.next() {
-                        let synonyms = std::str::from_utf8(synonyms).unwrap();
-                        let synonyms_words: Vec<_> = split_query_string(synonyms).collect();
-                        let nb_synonym_words = synonyms_words.len();
-
-                        let real_query_index = automaton_index;
-                        enhancer_builder.declare(
-                            query_range.clone(),
-                            real_query_index,
-                            &synonyms_words,
-                        );
-
-                        for synonym in synonyms_words {
-                            let automaton = if nb_synonym_words == 1 {
-                                Automaton::exact(automaton_index, n, synonym)
-                            } else {
-                                Automaton::non_exact(automaton_index, n, synonym)
-                            };
-                            automaton_index += 1;
-                            automatons.push(AutomatonGroup::normal(vec![automaton]));
-                        }
-                    }
-                }
-            }
-
-            if n == 1 {
-                if let Some((left, right)) =
-                    split_best_frequency(reader, &normalized, postings_lists_store)?
-                {
-                    let a = Automaton::exact(automaton_index, 1, left);
-                    enhancer_builder.declare(query_range.clone(), automaton_index, &[left]);
-                    automaton_index += 1;
-
-                    let b = Automaton::exact(automaton_index, 1, right);
-                    enhancer_builder.declare(query_range.clone(), automaton_index, &[left]);
-                    automaton_index += 1;
-
-                    automatons.push(AutomatonGroup::phrase_query(vec![a, b]));
-                }
-            } else {
-                // automaton of concatenation of query words
-                let concat = ngram_slice.concat();
-                let normalized = normalize_str(&concat);
-
-                let real_query_index = automaton_index;
-                enhancer_builder.declare(query_range.clone(), real_query_index, &[&normalized]);
-
-                let automaton = Automaton::exact(automaton_index, n, &normalized);
-                automaton_index += 1;
-                automatons.push(AutomatonGroup::normal(vec![automaton]));
-            }
-        }
-    }
-
-    // order automatons, the most important first,
-    // we keep the original automatons at the front.
-    automatons[1..].sort_by_key(|group| {
-        let a = group.automatons.first().unwrap();
-        (
-            Reverse(a.is_exact),
-            a.ngram,
-            Reverse(group.automatons.len()),
-        )
-    });
-
-    Ok((automatons, enhancer_builder.build()))
-}

meilidb-core/src/automaton/query_enhancer.rs

@@ -1,423 +0,0 @@
-use std::cmp::Ordering::{Equal, Greater, Less};
-use std::ops::Range;
-
-/// Return `true` if the specified range can accept the given replacements words.
-/// Returns `false` if the replacements words are already present in the original query
-/// or if there is fewer replacement words than the range to replace.
-//
-//
-// ## Ignored because already present in original
-//
-//     new york city subway
-//     -------- ^^^^
-//   /          \
-//  [new york city]
-//
-//
-// ## Ignored because smaller than the original
-//
-//   new york city subway
-//   -------------
-//   \          /
-//    [new york]
-//
-//
-// ## Accepted because bigger than the original
-//
-//        NYC subway
-//        ---
-//       /   \
-//      /     \
-//     /       \
-//    /         \
-//   /           \
-//  [new york city]
-//
-fn rewrite_range_with<S, T>(query: &[S], range: Range<usize>, words: &[T]) -> bool
-where
-    S: AsRef<str>,
-    T: AsRef<str>,
-{
-    if words.len() <= range.len() {
-        // there is fewer or equal replacement words
-        // than there is already in the replaced range
-        return false;
-    }
-
-    // retrieve the part to rewrite but with the length
-    // of the replacement part
-    let original = query.iter().skip(range.start).take(words.len());
-
-    // check if the original query doesn't already contain
-    // the replacement words
-    !original
-        .map(AsRef::as_ref)
-        .eq(words.iter().map(AsRef::as_ref))
-}
-
-type Origin = usize;
-type RealLength = usize;
-
-struct FakeIntervalTree {
-    intervals: Vec<(Range<usize>, (Origin, RealLength))>,
-}
-
-impl FakeIntervalTree {
-    fn new(mut intervals: Vec<(Range<usize>, (Origin, RealLength))>) -> FakeIntervalTree {
-        intervals.sort_unstable_by_key(|(r, _)| (r.start, r.end));
-        FakeIntervalTree { intervals }
-    }
-
-    fn query(&self, point: usize) -> Option<(Range<usize>, (Origin, RealLength))> {
-        let element = self.intervals.binary_search_by(|(r, _)| {
-            if point >= r.start {
-                if point < r.end {
-                    Equal
-                } else {
-                    Less
-                }
-            } else {
-                Greater
-            }
-        });
-
-        let n = match element {
-            Ok(n) => n,
-            Err(n) => n,
-        };
-
-        match self.intervals.get(n) {
-            Some((range, value)) if range.contains(&point) => Some((range.clone(), *value)),
-            _otherwise => None,
-        }
-    }
-}
-
-pub struct QueryEnhancerBuilder<'a, S> {
-    query: &'a [S],
-    origins: Vec<usize>,
-    real_to_origin: Vec<(Range<usize>, (Origin, RealLength))>,
-}
-
-impl<S: AsRef<str>> QueryEnhancerBuilder<'_, S> {
-    pub fn new(query: &[S]) -> QueryEnhancerBuilder<S> {
-        // we initialize origins query indices based on their positions
-        let origins: Vec<_> = (0..=query.len()).collect();
-        let real_to_origin = origins.iter().map(|&o| (o..o + 1, (o, 1))).collect();
-
-        QueryEnhancerBuilder {
-            query,
-            origins,
-            real_to_origin,
-        }
-    }
-
-    /// Update the final real to origin query indices mapping.
-    ///
-    /// `range` is the original words range that this `replacement` words replace
-    /// and `real` is the first real query index of these replacement words.
-    pub fn declare<T>(&mut self, range: Range<usize>, real: usize, replacement: &[T])
-    where
-        T: AsRef<str>,
-    {
-        // check if the range of original words
-        // can be rewritten with the replacement words
-        if rewrite_range_with(self.query, range.clone(), replacement) {
-            // this range can be replaced so we need to
-            // modify the origins accordingly
-            let offset = replacement.len() - range.len();
-
-            let previous_padding = self.origins[range.end - 1];
-            let current_offset = (self.origins[range.end] - 1) - previous_padding;
-            let diff = offset.saturating_sub(current_offset);
-            self.origins[range.end] += diff;
-
-            for r in &mut self.origins[range.end + 1..] {
-                *r += diff;
-            }
-        }
-
-        // we need to store the real number and origins relations
-        // this way it will be possible to know by how many
-        // we need to pad real query indices
-        let real_range = real..real + replacement.len().max(range.len());
-        let real_length = replacement.len();
-        self.real_to_origin
-            .push((real_range, (range.start, real_length)));
-    }
-
-    pub fn build(self) -> QueryEnhancer {
-        QueryEnhancer {
-            origins: self.origins,
-            real_to_origin: FakeIntervalTree::new(self.real_to_origin),
-        }
-    }
-}
-
-pub struct QueryEnhancer {
-    origins: Vec<usize>,
-    real_to_origin: FakeIntervalTree,
-}
-
-impl QueryEnhancer {
-    /// Returns the query indices to use to replace this real query index.
-    pub fn replacement(&self, real: u32) -> Range<u32> {
-        let real = real as usize;
-
-        // query the fake interval tree with the real query index
-        let (range, (origin, real_length)) = self
-            .real_to_origin
-            .query(real)
-            .expect("real has never been declared");
-
-        // if `real` is the end bound of the range
-        if (range.start + real_length - 1) == real {
-            let mut count = range.len();
-            let mut new_origin = origin;
-            for (i, slice) in self.origins[new_origin..].windows(2).enumerate() {
-                let len = slice[1] - slice[0];
-                count = count.saturating_sub(len);
-                if count == 0 {
-                    new_origin = origin + i;
-                    break;
-                }
-            }
-
-            let n = real - range.start;
-            let start = self.origins[origin];
-            let end = self.origins[new_origin + 1];
-            let remaining = (end - start) - n;
-
-            Range {
-                start: (start + n) as u32,
-                end: (start + n + remaining) as u32,
-            }
-        } else {
-            // just return the origin along with
-            // the real position of the word
-            let n = real as usize - range.start;
-            let origin = self.origins[origin];
-
-            Range {
-                start: (origin + n) as u32,
-                end: (origin + n + 1) as u32,
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn original_unmodified() {
-        let query = ["new", "york", "city", "subway"];
-        //             0       1       2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // new york = new york city
-        builder.declare(0..2, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // new
-        assert_eq!(enhancer.replacement(1), 1..2); // york
-        assert_eq!(enhancer.replacement(2), 2..3); // city
-        assert_eq!(enhancer.replacement(3), 3..4); // subway
-        assert_eq!(enhancer.replacement(4), 0..1); // new
-        assert_eq!(enhancer.replacement(5), 1..2); // york
-        assert_eq!(enhancer.replacement(6), 2..3); // city
-    }
-
-    #[test]
-    fn simple_growing() {
-        let query = ["new", "york", "subway"];
-        //             0       1        2
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // new york = new york city
-        builder.declare(0..2, 3, &["new", "york", "city"]);
-        //                    ^      3       4       5
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // new
-        assert_eq!(enhancer.replacement(1), 1..3); // york
-        assert_eq!(enhancer.replacement(2), 3..4); // subway
-        assert_eq!(enhancer.replacement(3), 0..1); // new
-        assert_eq!(enhancer.replacement(4), 1..2); // york
-        assert_eq!(enhancer.replacement(5), 2..3); // city
-    }
-
-    #[test]
-    fn same_place_growings() {
-        let query = ["NY", "subway"];
-        //             0       1
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NY = new york
-        builder.declare(0..1, 2, &["new", "york"]);
-        //                    ^      2       3
-
-        // NY = new york city
-        builder.declare(0..1, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // NY = NYC
-        builder.declare(0..1, 7, &["NYC"]);
-        //                    ^      7
-
-        // NY = new york city
-        builder.declare(0..1, 8, &["new", "york", "city"]);
-        //                    ^      8       9      10
-
-        // subway = underground train
-        builder.declare(1..2, 11, &["underground", "train"]);
-        //                    ^          11          12
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..3); // NY
-        assert_eq!(enhancer.replacement(1), 3..5); // subway
-        assert_eq!(enhancer.replacement(2), 0..1); // new
-        assert_eq!(enhancer.replacement(3), 1..3); // york
-        assert_eq!(enhancer.replacement(4), 0..1); // new
-        assert_eq!(enhancer.replacement(5), 1..2); // york
-        assert_eq!(enhancer.replacement(6), 2..3); // city
-        assert_eq!(enhancer.replacement(7), 0..3); // NYC
-        assert_eq!(enhancer.replacement(8), 0..1); // new
-        assert_eq!(enhancer.replacement(9), 1..2); // york
-        assert_eq!(enhancer.replacement(10), 2..3); // city
-        assert_eq!(enhancer.replacement(11), 3..4); // underground
-        assert_eq!(enhancer.replacement(12), 4..5); // train
-    }
-
-    #[test]
-    fn bigger_growing() {
-        let query = ["NYC", "subway"];
-        //             0        1
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(0..1, 2, &["new", "york", "city"]);
-        //                    ^      2       3       4
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..3); // NYC
-        assert_eq!(enhancer.replacement(1), 3..4); // subway
-        assert_eq!(enhancer.replacement(2), 0..1); // new
-        assert_eq!(enhancer.replacement(3), 1..2); // york
-        assert_eq!(enhancer.replacement(4), 2..3); // city
-    }
-
-    #[test]
-    fn middle_query_growing() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // great
-        assert_eq!(enhancer.replacement(1), 1..2); // awesome
-        assert_eq!(enhancer.replacement(2), 2..5); // NYC
-        assert_eq!(enhancer.replacement(3), 5..6); // subway
-        assert_eq!(enhancer.replacement(4), 2..3); // new
-        assert_eq!(enhancer.replacement(5), 3..4); // york
-        assert_eq!(enhancer.replacement(6), 4..5); // city
-    }
-
-    #[test]
-    fn end_query_growing() {
-        let query = ["NYC", "subway"];
-        //             0        1
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(1..2, 2, &["underground", "train"]);
-        //                    ^         2            3
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // NYC
-        assert_eq!(enhancer.replacement(1), 1..3); // subway
-        assert_eq!(enhancer.replacement(2), 1..2); // underground
-        assert_eq!(enhancer.replacement(3), 2..3); // train
-    }
-
-    #[test]
-    fn multiple_growings() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // subway = underground train
-        builder.declare(3..4, 7, &["underground", "train"]);
-        //                    ^          7           8
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // great
-        assert_eq!(enhancer.replacement(1), 1..2); // awesome
-        assert_eq!(enhancer.replacement(2), 2..5); // NYC
-        assert_eq!(enhancer.replacement(3), 5..7); // subway
-        assert_eq!(enhancer.replacement(4), 2..3); // new
-        assert_eq!(enhancer.replacement(5), 3..4); // york
-        assert_eq!(enhancer.replacement(6), 4..5); // city
-        assert_eq!(enhancer.replacement(7), 5..6); // underground
-        assert_eq!(enhancer.replacement(8), 6..7); // train
-    }
-
-    #[test]
-    fn multiple_probable_growings() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // subway = underground train
-        builder.declare(3..4, 7, &["underground", "train"]);
-        //                    ^          7           8
-
-        // great awesome = good
-        builder.declare(0..2, 9, &["good"]);
-        //                    ^       9
-
-        // awesome NYC = NY
-        builder.declare(1..3, 10, &["NY"]);
-        //                    ^^     10
-
-        // NYC subway = metro
-        builder.declare(2..4, 11, &["metro"]);
-        //                    ^^      11
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // great
-        assert_eq!(enhancer.replacement(1), 1..2); // awesome
-        assert_eq!(enhancer.replacement(2), 2..5); // NYC
-        assert_eq!(enhancer.replacement(3), 5..7); // subway
-        assert_eq!(enhancer.replacement(4), 2..3); // new
-        assert_eq!(enhancer.replacement(5), 3..4); // york
-        assert_eq!(enhancer.replacement(6), 4..5); // city
-        assert_eq!(enhancer.replacement(7), 5..6); // underground
-        assert_eq!(enhancer.replacement(8), 6..7); // train
-        assert_eq!(enhancer.replacement(9), 0..2); // good
-        assert_eq!(enhancer.replacement(10), 1..5); // NY
-        assert_eq!(enhancer.replacement(11), 2..5); // metro
-    }
-}

meilidb-core/src/criterion/document_id.rs

@@ -1,16 +0,0 @@
-use crate::criterion::Criterion;
-use crate::RawDocument;
-use std::cmp::Ordering;
-
-#[derive(Debug, Clone, Copy)]
-pub struct DocumentId;
-
-impl Criterion for DocumentId {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        lhs.id.cmp(&rhs.id)
-    }
-
-    fn name(&self) -> &str {
-        "DocumentId"
-    }
-}

meilidb-core/src/criterion/exact.rs

@@ -1,132 +0,0 @@
-use std::cmp::Ordering;
-
-use meilidb_schema::SchemaAttr;
-use sdset::Set;
-use slice_group_by::GroupBy;
-
-use crate::criterion::Criterion;
-use crate::RawDocument;
-
-#[inline]
-fn number_exact_matches(
-    query_index: &[u32],
-    attribute: &[u16],
-    is_exact: &[bool],
-    fields_counts: &Set<(SchemaAttr, u64)>,
-) -> usize {
-    let mut count = 0;
-    let mut index = 0;
-
-    for group in query_index.linear_group() {
-        let len = group.len();
-
-        let mut found_exact = false;
-        for (pos, is_exact) in is_exact[index..index + len].iter().enumerate() {
-            if *is_exact {
-                found_exact = true;
-                let attr = &attribute[index + pos];
-                if let Ok(pos) = fields_counts.binary_search_by_key(attr, |(a, _)| a.0) {
-                    let (_, count) = fields_counts[pos];
-                    if count == 1 {
-                        return usize::max_value();
-                    }
-                }
-            }
-        }
-
-        count += found_exact as usize;
-        index += len;
-    }
-
-    count
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct Exact;
-
-impl Criterion for Exact {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = {
-            let query_index = lhs.query_index();
-            let is_exact = lhs.is_exact();
-            let attribute = lhs.attribute();
-            let fields_counts = &lhs.fields_counts;
-
-            number_exact_matches(query_index, attribute, is_exact, fields_counts)
-        };
-
-        let rhs = {
-            let query_index = rhs.query_index();
-            let is_exact = rhs.is_exact();
-            let attribute = rhs.attribute();
-            let fields_counts = &rhs.fields_counts;
-
-            number_exact_matches(query_index, attribute, is_exact, fields_counts)
-        };
-
-        lhs.cmp(&rhs).reverse()
-    }
-
-    fn name(&self) -> &str {
-        "Exact"
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    // typing: "soulier"
-    //
-    // doc0: "Soulier bleu"
-    // doc1: "souliereres rouge"
-    #[test]
-    fn easy_case() {
-        let doc0 = {
-            let query_index = &[0];
-            let attribute = &[0];
-            let is_exact = &[true];
-            let fields_counts = Set::new(&[(SchemaAttr(0), 2)]).unwrap();
-
-            number_exact_matches(query_index, attribute, is_exact, fields_counts)
-        };
-
-        let doc1 = {
-            let query_index = &[0];
-            let attribute = &[0];
-            let is_exact = &[false];
-            let fields_counts = Set::new(&[(SchemaAttr(0), 2)]).unwrap();
-
-            number_exact_matches(query_index, attribute, is_exact, fields_counts)
-        };
-
-        assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
-    }
-
-    // typing: "soulier"
-    //
-    // doc0: { 0. "soulier" }
-    // doc1: { 0. "soulier bleu et blanc" }
-    #[test]
-    fn basic() {
-        let doc0 = {
-            let query_index = &[0];
-            let attribute = &[0];
-            let is_exact = &[true];
-            let fields_counts = Set::new(&[(SchemaAttr(0), 1)]).unwrap();
-
-            number_exact_matches(query_index, attribute, is_exact, fields_counts)
-        };
-
-        let doc1 = {
-            let query_index = &[0];
-            let attribute = &[0];
-            let is_exact = &[true];
-            let fields_counts = Set::new(&[(SchemaAttr(0), 4)]).unwrap();
-
-            number_exact_matches(query_index, attribute, is_exact, fields_counts)
-        };
-
-        assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
-    }
-}

meilidb-core/src/criterion/mod.rs

@@ -1,121 +0,0 @@
-mod document_id;
-mod exact;
-mod number_of_words;
-mod sort_by_attr;
-mod sum_of_typos;
-mod sum_of_words_attribute;
-mod sum_of_words_position;
-mod words_proximity;
-
-use crate::RawDocument;
-use std::cmp::Ordering;
-
-pub use self::{
-    document_id::DocumentId, exact::Exact, number_of_words::NumberOfWords,
-    sort_by_attr::SortByAttr, sum_of_typos::SumOfTypos,
-    sum_of_words_attribute::SumOfWordsAttribute, sum_of_words_position::SumOfWordsPosition,
-    words_proximity::WordsProximity,
-};
-
-pub trait Criterion: Send + Sync {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering;
-
-    fn name(&self) -> &str;
-
-    #[inline]
-    fn eq(&self, lhs: &RawDocument, rhs: &RawDocument) -> bool {
-        self.evaluate(lhs, rhs) == Ordering::Equal
-    }
-}
-
-impl<'a, T: Criterion + ?Sized + Send + Sync> Criterion for &'a T {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        (**self).evaluate(lhs, rhs)
-    }
-
-    fn name(&self) -> &str {
-        (**self).name()
-    }
-
-    fn eq(&self, lhs: &RawDocument, rhs: &RawDocument) -> bool {
-        (**self).eq(lhs, rhs)
-    }
-}
-
-impl<T: Criterion + ?Sized> Criterion for Box<T> {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        (**self).evaluate(lhs, rhs)
-    }
-
-    fn name(&self) -> &str {
-        (**self).name()
-    }
-
-    fn eq(&self, lhs: &RawDocument, rhs: &RawDocument) -> bool {
-        (**self).eq(lhs, rhs)
-    }
-}
-
-#[derive(Default)]
-pub struct CriteriaBuilder<'a> {
-    inner: Vec<Box<dyn Criterion + 'a>>,
-}
-
-impl<'a> CriteriaBuilder<'a> {
-    pub fn new() -> CriteriaBuilder<'a> {
-        CriteriaBuilder { inner: Vec::new() }
-    }
-
-    pub fn with_capacity(capacity: usize) -> CriteriaBuilder<'a> {
-        CriteriaBuilder {
-            inner: Vec::with_capacity(capacity),
-        }
-    }
-
-    pub fn reserve(&mut self, additional: usize) {
-        self.inner.reserve(additional)
-    }
-
-    pub fn add<C: 'a>(mut self, criterion: C) -> CriteriaBuilder<'a>
-    where
-        C: Criterion,
-    {
-        self.push(criterion);
-        self
-    }
-
-    pub fn push<C: 'a>(&mut self, criterion: C)
-    where
-        C: Criterion,
-    {
-        self.inner.push(Box::new(criterion));
-    }
-
-    pub fn build(self) -> Criteria<'a> {
-        Criteria { inner: self.inner }
-    }
-}
-
-pub struct Criteria<'a> {
-    inner: Vec<Box<dyn Criterion + 'a>>,
-}
-
-impl<'a> Default for Criteria<'a> {
-    fn default() -> Self {
-        CriteriaBuilder::with_capacity(7)
-            .add(SumOfTypos)
-            .add(NumberOfWords)
-            .add(WordsProximity)
-            .add(SumOfWordsAttribute)
-            .add(SumOfWordsPosition)
-            .add(Exact)
-            .add(DocumentId)
-            .build()
-    }
-}
-
-impl<'a> AsRef<[Box<dyn Criterion + 'a>]> for Criteria<'a> {
-    fn as_ref(&self) -> &[Box<dyn Criterion + 'a>] {
-        &self.inner
-    }
-}

meilidb-core/src/criterion/number_of_words.rs

@@ -1,31 +0,0 @@
-use crate::criterion::Criterion;
-use crate::RawDocument;
-use slice_group_by::GroupBy;
-use std::cmp::Ordering;
-
-#[inline]
-fn number_of_query_words(query_index: &[u32]) -> usize {
-    query_index.linear_group().count()
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct NumberOfWords;
-
-impl Criterion for NumberOfWords {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = {
-            let query_index = lhs.query_index();
-            number_of_query_words(query_index)
-        };
-        let rhs = {
-            let query_index = rhs.query_index();
-            number_of_query_words(query_index)
-        };
-
-        lhs.cmp(&rhs).reverse()
-    }
-
-    fn name(&self) -> &str {
-        "NumberOfWords"
-    }
-}

meilidb-core/src/criterion/sum_of_typos.rs

@@ -1,116 +0,0 @@
-use std::cmp::Ordering;
-
-use slice_group_by::GroupBy;
-
-use crate::criterion::Criterion;
-use crate::RawDocument;
-
-// This function is a wrong logarithmic 10 function.
-// It is safe to panic on input number higher than 3,
-// the number of typos is never bigger than that.
-#[inline]
-fn custom_log10(n: u8) -> f32 {
-    match n {
-        0 => 0.0,     // log(1)
-        1 => 0.30102, // log(2)
-        2 => 0.47712, // log(3)
-        3 => 0.60205, // log(4)
-        _ => panic!("invalid number"),
-    }
-}
-
-#[inline]
-fn sum_matches_typos(query_index: &[u32], distance: &[u8]) -> usize {
-    let mut number_words: usize = 0;
-    let mut sum_typos = 0.0;
-    let mut index = 0;
-
-    for group in query_index.linear_group() {
-        sum_typos += custom_log10(distance[index]);
-        number_words += 1;
-        index += group.len();
-    }
-
-    (number_words as f32 / (sum_typos + 1.0) * 1000.0) as usize
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct SumOfTypos;
-
-impl Criterion for SumOfTypos {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = {
-            let query_index = lhs.query_index();
-            let distance = lhs.distance();
-            sum_matches_typos(query_index, distance)
-        };
-
-        let rhs = {
-            let query_index = rhs.query_index();
-            let distance = rhs.distance();
-            sum_matches_typos(query_index, distance)
-        };
-
-        lhs.cmp(&rhs).reverse()
-    }
-
-    fn name(&self) -> &str {
-        "SumOfTypos"
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    // typing: "Geox CEO"
-    //
-    // doc0: "Geox SpA: CEO and Executive"
-    // doc1: "Mt. Gox CEO Resigns From Bitcoin Foundation"
-    #[test]
-    fn one_typo_reference() {
-        let query_index0 = &[0, 1];
-        let distance0 = &[0, 0];
-
-        let query_index1 = &[0, 1];
-        let distance1 = &[1, 0];
-
-        let doc0 = sum_matches_typos(query_index0, distance0);
-        let doc1 = sum_matches_typos(query_index1, distance1);
-        assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
-    }
-
-    // typing: "bouton manchette"
-    //
-    // doc0: "bouton manchette"
-    // doc1: "bouton"
-    #[test]
-    fn no_typo() {
-        let query_index0 = &[0, 1];
-        let distance0 = &[0, 0];
-
-        let query_index1 = &[0];
-        let distance1 = &[0];
-
-        let doc0 = sum_matches_typos(query_index0, distance0);
-        let doc1 = sum_matches_typos(query_index1, distance1);
-        assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
-    }
-
-    // typing: "bouton manchztte"
-    //
-    // doc0: "bouton manchette"
-    // doc1: "bouton"
-    #[test]
-    fn one_typo() {
-        let query_index0 = &[0, 1];
-        let distance0 = &[0, 1];
-
-        let query_index1 = &[0];
-        let distance1 = &[0];
-
-        let doc0 = sum_matches_typos(query_index0, distance0);
-        let doc1 = sum_matches_typos(query_index1, distance1);
-        assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
-    }
-}

meilidb-core/src/criterion/sum_of_words_attribute.rs

@@ -1,64 +0,0 @@
-use crate::criterion::Criterion;
-use crate::RawDocument;
-use slice_group_by::GroupBy;
-use std::cmp::Ordering;
-
-#[inline]
-fn sum_matches_attributes(query_index: &[u32], attribute: &[u16]) -> usize {
-    let mut sum_attributes = 0;
-    let mut index = 0;
-
-    for group in query_index.linear_group() {
-        sum_attributes += attribute[index] as usize;
-        index += group.len();
-    }
-
-    sum_attributes
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct SumOfWordsAttribute;
-
-impl Criterion for SumOfWordsAttribute {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = {
-            let query_index = lhs.query_index();
-            let attribute = lhs.attribute();
-            sum_matches_attributes(query_index, attribute)
-        };
-
-        let rhs = {
-            let query_index = rhs.query_index();
-            let attribute = rhs.attribute();
-            sum_matches_attributes(query_index, attribute)
-        };
-
-        lhs.cmp(&rhs)
-    }
-
-    fn name(&self) -> &str {
-        "SumOfWordsAttribute"
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    // typing: "soulier"
-    //
-    // doc0: { 0. "Soulier bleu", 1. "bla bla bla" }
-    // doc1: { 0. "Botte rouge", 1. "Soulier en cuir" }
-    #[test]
-    fn title_vs_description() {
-        let query_index0 = &[0];
-        let attribute0 = &[0];
-
-        let query_index1 = &[0];
-        let attribute1 = &[1];
-
-        let doc0 = sum_matches_attributes(query_index0, attribute0);
-        let doc1 = sum_matches_attributes(query_index1, attribute1);
-        assert_eq!(doc0.cmp(&doc1), Ordering::Less);
-    }
-}

meilidb-core/src/criterion/sum_of_words_position.rs

@@ -1,64 +0,0 @@
-use crate::criterion::Criterion;
-use crate::RawDocument;
-use slice_group_by::GroupBy;
-use std::cmp::Ordering;
-
-#[inline]
-fn sum_matches_attribute_index(query_index: &[u32], word_index: &[u16]) -> usize {
-    let mut sum_word_index = 0;
-    let mut index = 0;
-
-    for group in query_index.linear_group() {
-        sum_word_index += word_index[index] as usize;
-        index += group.len();
-    }
-
-    sum_word_index
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct SumOfWordsPosition;
-
-impl Criterion for SumOfWordsPosition {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = {
-            let query_index = lhs.query_index();
-            let word_index = lhs.word_index();
-            sum_matches_attribute_index(query_index, word_index)
-        };
-
-        let rhs = {
-            let query_index = rhs.query_index();
-            let word_index = rhs.word_index();
-            sum_matches_attribute_index(query_index, word_index)
-        };
-
-        lhs.cmp(&rhs)
-    }
-
-    fn name(&self) -> &str {
-        "SumOfWordsPosition"
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    // typing: "soulier"
-    //
-    // doc0: "Soulier bleu"
-    // doc1: "Botte rouge et soulier noir"
-    #[test]
-    fn easy_case() {
-        let query_index0 = &[0];
-        let word_index0 = &[0];
-
-        let query_index1 = &[0];
-        let word_index1 = &[3];
-
-        let doc0 = sum_matches_attribute_index(query_index0, word_index0);
-        let doc1 = sum_matches_attribute_index(query_index1, word_index1);
-        assert_eq!(doc0.cmp(&doc1), Ordering::Less);
-    }
-}

meilidb-core/src/criterion/words_proximity.rs

@@ -1,164 +0,0 @@
-use crate::criterion::Criterion;
-use crate::RawDocument;
-use slice_group_by::GroupBy;
-use std::cmp::{self, Ordering};
-
-const MAX_DISTANCE: u16 = 8;
-
-#[inline]
-fn clone_tuple<T: Clone, U: Clone>((a, b): (&T, &U)) -> (T, U) {
-    (a.clone(), b.clone())
-}
-
-fn index_proximity(lhs: u16, rhs: u16) -> u16 {
-    if lhs < rhs {
-        cmp::min(rhs - lhs, MAX_DISTANCE)
-    } else {
-        cmp::min(lhs - rhs, MAX_DISTANCE) + 1
-    }
-}
-
-fn attribute_proximity((lattr, lwi): (u16, u16), (rattr, rwi): (u16, u16)) -> u16 {
-    if lattr != rattr {
-        return MAX_DISTANCE;
-    }
-    index_proximity(lwi, rwi)
-}
-
-fn min_proximity((lattr, lwi): (&[u16], &[u16]), (rattr, rwi): (&[u16], &[u16])) -> u16 {
-    let mut min_prox = u16::max_value();
-
-    for a in lattr.iter().zip(lwi) {
-        for b in rattr.iter().zip(rwi) {
-            let a = clone_tuple(a);
-            let b = clone_tuple(b);
-            min_prox = cmp::min(min_prox, attribute_proximity(a, b));
-        }
-    }
-
-    min_prox
-}
-
-fn matches_proximity(
-    query_index: &[u32],
-    distance: &[u8],
-    attribute: &[u16],
-    word_index: &[u16],
-) -> u16 {
-    let mut query_index_groups = query_index.linear_group();
-    let mut proximity = 0;
-    let mut index = 0;
-
-    let get_attr_wi = |index: usize, group_len: usize| {
-        // retrieve the first distance group (with the lowest values)
-        let len = distance[index..index + group_len]
-            .linear_group()
-            .next()
-            .unwrap()
-            .len();
-
-        let rattr = &attribute[index..index + len];
-        let rwi = &word_index[index..index + len];
-
-        (rattr, rwi)
-    };
-
-    let mut last = query_index_groups.next().map(|group| {
-        let attr_wi = get_attr_wi(index, group.len());
-        index += group.len();
-        attr_wi
-    });
-
-    // iter by windows of size 2
-    while let (Some(lhs), Some(rhs)) = (last, query_index_groups.next()) {
-        let attr_wi = get_attr_wi(index, rhs.len());
-        proximity += min_proximity(lhs, attr_wi);
-        last = Some(attr_wi);
-        index += rhs.len();
-    }
-
-    proximity
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct WordsProximity;
-
-impl Criterion for WordsProximity {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = {
-            let query_index = lhs.query_index();
-            let distance = lhs.distance();
-            let attribute = lhs.attribute();
-            let word_index = lhs.word_index();
-            matches_proximity(query_index, distance, attribute, word_index)
-        };
-
-        let rhs = {
-            let query_index = rhs.query_index();
-            let distance = rhs.distance();
-            let attribute = rhs.attribute();
-            let word_index = rhs.word_index();
-            matches_proximity(query_index, distance, attribute, word_index)
-        };
-
-        lhs.cmp(&rhs)
-    }
-
-    fn name(&self) -> &str {
-        "WordsProximity"
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn three_different_attributes() {
-        // "soup" "of the" "the day"
-        //
-        // { id: 0, attr: 0, attr_index: 0 }
-        // { id: 1, attr: 1, attr_index: 0 }
-        // { id: 2, attr: 1, attr_index: 1 }
-        // { id: 2, attr: 2, attr_index: 0 }
-        // { id: 3, attr: 3, attr_index: 1 }
-
-        let query_index = &[0, 1, 2, 2, 3];
-        let distance = &[0, 0, 0, 0, 0];
-        let attribute = &[0, 1, 1, 2, 3];
-        let word_index = &[0, 0, 1, 0, 1];
-
-        //   soup -> of = 8
-        // + of -> the  = 1
-        // + the -> day = 8 (not 1)
-        assert_eq!(
-            matches_proximity(query_index, distance, attribute, word_index),
-            17
-        );
-    }
-
-    #[test]
-    fn two_different_attributes() {
-        // "soup day" "soup of the day"
-        //
-        // { id: 0, attr: 0, attr_index: 0 }
-        // { id: 0, attr: 1, attr_index: 0 }
-        // { id: 1, attr: 1, attr_index: 1 }
-        // { id: 2, attr: 1, attr_index: 2 }
-        // { id: 3, attr: 0, attr_index: 1 }
-        // { id: 3, attr: 1, attr_index: 3 }
-
-        let query_index = &[0, 0, 1, 2, 3, 3];
-        let distance = &[0, 0, 0, 0, 0, 0];
-        let attribute = &[0, 1, 1, 1, 0, 1];
-        let word_index = &[0, 0, 1, 2, 1, 3];
-
-        //   soup -> of = 1
-        // + of -> the  = 1
-        // + the -> day = 1
-        assert_eq!(
-            matches_proximity(query_index, distance, attribute, word_index),
-            3
-        );
-    }
-}

meilidb-core/src/database.rs

@@ -1,362 +0,0 @@
-use std::collections::hash_map::{Entry, HashMap};
-use std::fs::File;
-use std::path::Path;
-use std::sync::{Arc, RwLock};
-use std::{fs, thread};
-
-use crossbeam_channel::Receiver;
-use heed::types::{Str, Unit};
-use heed::{CompactionOption, Result as ZResult};
-use log::debug;
-
-use crate::{store, update, Index, MResult};
-
-pub type BoxUpdateFn = Box<dyn Fn(update::ProcessedUpdateResult) + Send + Sync + 'static>;
-type ArcSwapFn = arc_swap::ArcSwapOption<BoxUpdateFn>;
-
-pub struct Database {
-    pub env: heed::Env,
-    common_store: heed::PolyDatabase,
-    indexes_store: heed::Database<Str, Unit>,
-    indexes: RwLock<HashMap<String, (Index, Arc<ArcSwapFn>, thread::JoinHandle<()>)>>,
-}
-
-macro_rules! r#break_try {
-    ($expr:expr, $msg:tt) => {
-        match $expr {
-            core::result::Result::Ok(val) => val,
-            core::result::Result::Err(err) => {
-                log::error!(concat!($msg, ": {}"), err);
-                break;
-            }
-        }
-    };
-}
-
-fn update_awaiter(receiver: Receiver<()>, env: heed::Env, update_fn: Arc<ArcSwapFn>, index: Index) {
-    for () in receiver {
-        // consume all updates in order (oldest first)
-        loop {
-            // instantiate a main/parent transaction
-            let mut writer = break_try!(env.write_txn(), "LMDB write transaction begin failed");
-
-            // retrieve the update that needs to be processed
-            let result = index.updates.pop_front(&mut writer);
-            let (update_id, update) = match break_try!(result, "pop front update failed") {
-                Some(value) => value,
-                None => {
-                    debug!("no more updates");
-                    writer.abort();
-                    break;
-                }
-            };
-
-            // instantiate a nested transaction
-            let result = env.nested_write_txn(&mut writer);
-            let mut nested_writer = break_try!(result, "LMDB nested write transaction failed");
-
-            // try to apply the update to the database using the nested transaction
-            let result = update::update_task(&mut nested_writer, index.clone(), update_id, update);
-            let status = break_try!(result, "update task failed");
-
-            // commit the nested transaction if the update was successful, abort it otherwise
-            if status.result.is_ok() {
-                break_try!(nested_writer.commit(), "commit nested transaction failed");
-            } else {
-                nested_writer.abort()
-            }
-
-            // write the result of the update in the updates-results store
-            let updates_results = index.updates_results;
-            let result = updates_results.put_update_result(&mut writer, update_id, &status);
-
-            // always commit the main/parent transaction, even if the update was unsuccessful
-            break_try!(result, "update result store commit failed");
-            break_try!(writer.commit(), "update parent transaction failed");
-
-            // call the user callback when the update and the result are written consistently
-            if let Some(ref callback) = *update_fn.load() {
-                (callback)(status);
-            }
-        }
-    }
-}
-
-impl Database {
-    pub fn open_or_create(path: impl AsRef<Path>) -> MResult<Database> {
-        fs::create_dir_all(path.as_ref())?;
-
-        let env = heed::EnvOpenOptions::new()
-            .map_size(10 * 1024 * 1024 * 1024) // 10GB
-            .max_dbs(3000)
-            .open(path)?;
-
-        let common_store = env.create_poly_database(Some("common"))?;
-        let indexes_store = env.create_database::<Str, Unit>(Some("indexes"))?;
-
-        // list all indexes that needs to be opened
-        let mut must_open = Vec::new();
-        let reader = env.read_txn()?;
-        for result in indexes_store.iter(&reader)? {
-            let (index_name, _) = result?;
-            must_open.push(index_name.to_owned());
-        }
-
-        reader.abort();
-
-        // open the previously aggregated indexes
-        let mut indexes = HashMap::new();
-        for index_name in must_open {
-            let (sender, receiver) = crossbeam_channel::bounded(100);
-            let index = match store::open(&env, &index_name, sender.clone())? {
-                Some(index) => index,
-                None => {
-                    log::warn!(
-                        "the index {} doesn't exist or has not all the databases",
-                        index_name
-                    );
-                    continue;
-                }
-            };
-            let update_fn = Arc::new(ArcSwapFn::empty());
-
-            let env_clone = env.clone();
-            let index_clone = index.clone();
-            let update_fn_clone = update_fn.clone();
-
-            let handle = thread::spawn(move || {
-                update_awaiter(receiver, env_clone, update_fn_clone, index_clone)
-            });
-
-            // send an update notification to make sure that
-            // possible pre-boot updates are consumed
-            sender.send(()).unwrap();
-
-            let result = indexes.insert(index_name, (index, update_fn, handle));
-            assert!(
-                result.is_none(),
-                "The index should not have been already open"
-            );
-        }
-
-        Ok(Database {
-            env,
-            common_store,
-            indexes_store,
-            indexes: RwLock::new(indexes),
-        })
-    }
-
-    pub fn open_index(&self, name: impl AsRef<str>) -> Option<Index> {
-        let indexes_lock = self.indexes.read().unwrap();
-        match indexes_lock.get(name.as_ref()) {
-            Some((index, ..)) => Some(index.clone()),
-            None => None,
-        }
-    }
-
-    pub fn create_index(&self, name: impl AsRef<str>) -> MResult<Index> {
-        let name = name.as_ref();
-        let mut indexes_lock = self.indexes.write().unwrap();
-
-        match indexes_lock.entry(name.to_owned()) {
-            Entry::Occupied(_) => Err(crate::Error::IndexAlreadyExists),
-            Entry::Vacant(entry) => {
-                let (sender, receiver) = crossbeam_channel::bounded(100);
-                let index = store::create(&self.env, name, sender)?;
-
-                let mut writer = self.env.write_txn()?;
-                self.indexes_store.put(&mut writer, name, &())?;
-
-                let env_clone = self.env.clone();
-                let index_clone = index.clone();
-
-                let no_update_fn = Arc::new(ArcSwapFn::empty());
-                let no_update_fn_clone = no_update_fn.clone();
-
-                let handle = thread::spawn(move || {
-                    update_awaiter(receiver, env_clone, no_update_fn_clone, index_clone)
-                });
-
-                writer.commit()?;
-                entry.insert((index.clone(), no_update_fn, handle));
-
-                Ok(index)
-            }
-        }
-    }
-
-    pub fn set_update_callback(&self, name: impl AsRef<str>, update_fn: BoxUpdateFn) -> bool {
-        let indexes_lock = self.indexes.read().unwrap();
-        match indexes_lock.get(name.as_ref()) {
-            Some((_, current_update_fn, _)) => {
-                let update_fn = Some(Arc::new(update_fn));
-                current_update_fn.swap(update_fn);
-                true
-            }
-            None => false,
-        }
-    }
-
-    pub fn unset_update_callback(&self, name: impl AsRef<str>) -> bool {
-        let indexes_lock = self.indexes.read().unwrap();
-        match indexes_lock.get(name.as_ref()) {
-            Some((_, current_update_fn, _)) => {
-                current_update_fn.swap(None);
-                true
-            }
-            None => false,
-        }
-    }
-
-    pub fn copy_and_compact_to_path<P: AsRef<Path>>(&self, path: P) -> ZResult<File> {
-        self.env.copy_to_path(path, CompactionOption::Enabled)
-    }
-
-    pub fn indexes_names(&self) -> MResult<Vec<String>> {
-        let indexes = self.indexes.read().unwrap();
-        Ok(indexes.keys().cloned().collect())
-    }
-
-    pub fn common_store(&self) -> heed::PolyDatabase {
-        self.common_store
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::update::{ProcessedUpdateResult, UpdateStatus};
-    use std::sync::mpsc;
-
-    #[test]
-    fn valid_updates() {
-        let dir = tempfile::tempdir().unwrap();
-
-        let database = Database::open_or_create(dir.path()).unwrap();
-        let env = &database.env;
-
-        let (sender, receiver) = mpsc::sync_channel(100);
-        let update_fn = move |update: ProcessedUpdateResult| sender.send(update.update_id).unwrap();
-        let index = database.create_index("test").unwrap();
-
-        let done = database.set_update_callback("test", Box::new(update_fn));
-        assert!(done, "could not set the index update function");
-
-        let schema = {
-            let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
-            "#;
-            toml::from_str(data).unwrap()
-        };
-
-        let mut writer = env.write_txn().unwrap();
-        let _update_id = index.schema_update(&mut writer, schema).unwrap();
-
-        // don't forget to commit...
-        writer.commit().unwrap();
-
-        let mut additions = index.documents_addition();
-
-        let doc1 = serde_json::json!({
-            "id": 123,
-            "name": "Marvin",
-            "description": "My name is Marvin",
-        });
-
-        let doc2 = serde_json::json!({
-            "id": 234,
-            "name": "Kevin",
-            "description": "My name is Kevin",
-        });
-
-        additions.update_document(doc1);
-        additions.update_document(doc2);
-
-        let mut writer = env.write_txn().unwrap();
-        let update_id = additions.finalize(&mut writer).unwrap();
-
-        // don't forget to commit...
-        writer.commit().unwrap();
-
-        // block until the transaction is processed
-        let _ = receiver.into_iter().find(|id| *id == update_id);
-
-        let reader = env.read_txn().unwrap();
-        let result = index.update_status(&reader, update_id).unwrap();
-        assert_matches!(result, UpdateStatus::Processed(status) if status.result.is_ok());
-    }
-
-    #[test]
-    fn invalid_updates() {
-        let dir = tempfile::tempdir().unwrap();
-
-        let database = Database::open_or_create(dir.path()).unwrap();
-        let env = &database.env;
-
-        let (sender, receiver) = mpsc::sync_channel(100);
-        let update_fn = move |update: ProcessedUpdateResult| sender.send(update.update_id).unwrap();
-        let index = database.create_index("test").unwrap();
-
-        let done = database.set_update_callback("test", Box::new(update_fn));
-        assert!(done, "could not set the index update function");
-
-        let schema = {
-            let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
-            "#;
-            toml::from_str(data).unwrap()
-        };
-
-        let mut writer = env.write_txn().unwrap();
-        let _update_id = index.schema_update(&mut writer, schema).unwrap();
-
-        // don't forget to commit...
-        writer.commit().unwrap();
-
-        let mut additions = index.documents_addition();
-
-        let doc1 = serde_json::json!({
-            "id": 123,
-            "name": "Marvin",
-            "description": "My name is Marvin",
-        });
-
-        let doc2 = serde_json::json!({
-            "name": "Kevin",
-            "description": "My name is Kevin",
-        });
-
-        additions.update_document(doc1);
-        additions.update_document(doc2);
-
-        let mut writer = env.write_txn().unwrap();
-        let update_id = additions.finalize(&mut writer).unwrap();
-
-        // don't forget to commit...
-        writer.commit().unwrap();
-
-        // block until the transaction is processed
-        let _ = receiver.into_iter().find(|id| *id == update_id);
-
-        let reader = env.read_txn().unwrap();
-        let result = index.update_status(&reader, update_id).unwrap();
-        assert_matches!(result, UpdateStatus::Processed(status) if status.result.is_err());
-    }
-}

meilidb-core/src/error.rs

@@ -1,117 +0,0 @@
-use crate::serde::{DeserializerError, SerializerError};
-use serde_json::Error as SerdeJsonError;
-use std::{error, fmt, io};
-
-pub type MResult<T> = Result<T, Error>;
-
-#[derive(Debug)]
-pub enum Error {
-    Io(io::Error),
-    IndexAlreadyExists,
-    SchemaDiffer,
-    SchemaMissing,
-    WordIndexMissing,
-    MissingDocumentId,
-    Zlmdb(heed::Error),
-    Fst(fst::Error),
-    SerdeJson(SerdeJsonError),
-    Bincode(bincode::Error),
-    Serializer(SerializerError),
-    Deserializer(DeserializerError),
-    UnsupportedOperation(UnsupportedOperation),
-}
-
-impl From<io::Error> for Error {
-    fn from(error: io::Error) -> Error {
-        Error::Io(error)
-    }
-}
-
-impl From<heed::Error> for Error {
-    fn from(error: heed::Error) -> Error {
-        Error::Zlmdb(error)
-    }
-}
-
-impl From<fst::Error> for Error {
-    fn from(error: fst::Error) -> Error {
-        Error::Fst(error)
-    }
-}
-
-impl From<SerdeJsonError> for Error {
-    fn from(error: SerdeJsonError) -> Error {
-        Error::SerdeJson(error)
-    }
-}
-
-impl From<bincode::Error> for Error {
-    fn from(error: bincode::Error) -> Error {
-        Error::Bincode(error)
-    }
-}
-
-impl From<SerializerError> for Error {
-    fn from(error: SerializerError) -> Error {
-        Error::Serializer(error)
-    }
-}
-
-impl From<DeserializerError> for Error {
-    fn from(error: DeserializerError) -> Error {
-        Error::Deserializer(error)
-    }
-}
-
-impl From<UnsupportedOperation> for Error {
-    fn from(op: UnsupportedOperation) -> Error {
-        Error::UnsupportedOperation(op)
-    }
-}
-
-impl fmt::Display for Error {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        use self::Error::*;
-        match self {
-            Io(e) => write!(f, "{}", e),
-            IndexAlreadyExists => write!(f, "index already exists"),
-            SchemaDiffer => write!(f, "schemas differ"),
-            SchemaMissing => write!(f, "this index does not have a schema"),
-            WordIndexMissing => write!(f, "this index does not have a word index"),
-            MissingDocumentId => write!(f, "document id is missing"),
-            Zlmdb(e) => write!(f, "heed error; {}", e),
-            Fst(e) => write!(f, "fst error; {}", e),
-            SerdeJson(e) => write!(f, "serde json error; {}", e),
-            Bincode(e) => write!(f, "bincode error; {}", e),
-            Serializer(e) => write!(f, "serializer error; {}", e),
-            Deserializer(e) => write!(f, "deserializer error; {}", e),
-            UnsupportedOperation(op) => write!(f, "unsupported operation; {}", op),
-        }
-    }
-}
-
-impl error::Error for Error {}
-
-#[derive(Debug)]
-pub enum UnsupportedOperation {
-    SchemaAlreadyExists,
-    CannotUpdateSchemaIdentifier,
-    CannotReorderSchemaAttribute,
-    CannotIntroduceNewSchemaAttribute,
-    CannotRemoveSchemaAttribute,
-}
-
-impl fmt::Display for UnsupportedOperation {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        use self::UnsupportedOperation::*;
-        match self {
-            SchemaAlreadyExists => write!(f, "Cannot update index which already have a schema"),
-            CannotUpdateSchemaIdentifier => write!(f, "Cannot update the identifier of a schema"),
-            CannotReorderSchemaAttribute => write!(f, "Cannot reorder the attributes of a schema"),
-            CannotIntroduceNewSchemaAttribute => {
-                write!(f, "Cannot introduce new attributes in a schema")
-            }
-            CannotRemoveSchemaAttribute => write!(f, "Cannot remove attributes from a schema"),
-        }
-    }
-}

meilidb-core/src/lib.rs

@@ -1,169 +0,0 @@
-#[cfg(test)]
-#[macro_use]
-extern crate assert_matches;
-
-mod automaton;
-pub mod criterion;
-mod database;
-mod distinct_map;
-mod error;
-mod levenshtein;
-mod number;
-mod query_builder;
-mod ranked_map;
-mod raw_document;
-pub mod raw_indexer;
-mod reordered_attrs;
-pub mod serde;
-pub mod store;
-mod update;
-
-pub use self::database::{BoxUpdateFn, Database};
-pub use self::error::{Error, MResult};
-pub use self::number::{Number, ParseNumberError};
-pub use self::ranked_map::RankedMap;
-pub use self::raw_document::RawDocument;
-pub use self::store::Index;
-pub use self::update::{EnqueuedUpdateResult, ProcessedUpdateResult, UpdateStatus, UpdateType};
-
-use ::serde::{Deserialize, Serialize};
-use zerocopy::{AsBytes, FromBytes};
-
-/// Represent an internally generated document unique identifier.
-///
-/// It is used to inform the database the document you want to deserialize.
-/// Helpful for custom ranking.
-#[derive(
-    Debug,
-    Copy,
-    Clone,
-    Eq,
-    PartialEq,
-    PartialOrd,
-    Ord,
-    Hash,
-    Serialize,
-    Deserialize,
-    AsBytes,
-    FromBytes,
-)]
-#[repr(C)]
-pub struct DocumentId(pub u64);
-
-/// This structure represent the position of a word
-/// in a document and its attributes.
-///
-/// This is stored in the map, generated at index time,
-/// extracted and interpreted at search time.
-#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, AsBytes, FromBytes)]
-#[repr(C)]
-pub struct DocIndex {
-    /// The document identifier where the word was found.
-    pub document_id: DocumentId,
-
-    /// The attribute in the document where the word was found
-    /// along with the index in it.
-    pub attribute: u16,
-    pub word_index: u16,
-
-    /// The position in bytes where the word was found
-    /// along with the length of it.
-    ///
-    /// It informs on the original word area in the text indexed
-    /// without needing to run the tokenizer again.
-    pub char_index: u16,
-    pub char_length: u16,
-}
-
-/// This structure represent a matching word with informations
-/// on the location of the word in the document.
-///
-/// The order of the field is important because it defines
-/// the way these structures are ordered between themselves.
-#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct Highlight {
-    /// The attribute in the document where the word was found
-    /// along with the index in it.
-    pub attribute: u16,
-
-    /// The position in bytes where the word was found.
-    ///
-    /// It informs on the original word area in the text indexed
-    /// without needing to run the tokenizer again.
-    pub char_index: u16,
-
-    /// The length in bytes of the found word.
-    ///
-    /// It informs on the original word area in the text indexed
-    /// without needing to run the tokenizer again.
-    pub char_length: u16,
-}
-
-#[doc(hidden)]
-#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct TmpMatch {
-    pub query_index: u32,
-    pub distance: u8,
-    pub attribute: u16,
-    pub word_index: u16,
-    pub is_exact: bool,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct Document {
-    pub id: DocumentId,
-    pub highlights: Vec<Highlight>,
-
-    #[cfg(test)]
-    pub matches: Vec<TmpMatch>,
-}
-
-impl Document {
-    #[cfg(not(test))]
-    fn from_raw(raw: RawDocument) -> Document {
-        Document {
-            id: raw.id,
-            highlights: raw.highlights,
-        }
-    }
-
-    #[cfg(test)]
-    fn from_raw(raw: RawDocument) -> Document {
-        let len = raw.query_index().len();
-        let mut matches = Vec::with_capacity(len);
-
-        let query_index = raw.query_index();
-        let distance = raw.distance();
-        let attribute = raw.attribute();
-        let word_index = raw.word_index();
-        let is_exact = raw.is_exact();
-
-        for i in 0..len {
-            let match_ = TmpMatch {
-                query_index: query_index[i],
-                distance: distance[i],
-                attribute: attribute[i],
-                word_index: word_index[i],
-                is_exact: is_exact[i],
-            };
-            matches.push(match_);
-        }
-
-        Document {
-            id: raw.id,
-            matches,
-            highlights: raw.highlights,
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::mem;
-
-    #[test]
-    fn docindex_mem_size() {
-        assert_eq!(mem::size_of::<DocIndex>(), 16);
-    }
-}

meilidb-core/src/query_enhancer.rs

@@ -1,398 +0,0 @@
-use std::ops::Range;
-use std::cmp::Ordering::{Less, Greater, Equal};
-
-/// Return `true` if the specified range can accept the given replacements words.
-/// Returns `false` if the replacements words are already present in the original query
-/// or if there is fewer replacement words than the range to replace.
-//
-//
-// ## Ignored because already present in original
-//
-//     new york city subway
-//     -------- ^^^^
-//   /          \
-//  [new york city]
-//
-//
-// ## Ignored because smaller than the original
-//
-//   new york city subway
-//   -------------
-//   \          /
-//    [new york]
-//
-//
-// ## Accepted because bigger than the original
-//
-//        NYC subway
-//        ---
-//       /   \
-//      /     \
-//     /       \
-//    /         \
-//   /           \
-//  [new york city]
-//
-fn rewrite_range_with<S, T>(query: &[S], range: Range<usize>, words: &[T]) -> bool
-where S: AsRef<str>,
-      T: AsRef<str>,
-{
-    if words.len() <= range.len() {
-        // there is fewer or equal replacement words
-        // than there is already in the replaced range
-        return false
-    }
-
-    // retrieve the part to rewrite but with the length
-    // of the replacement part
-    let original = query.iter().skip(range.start).take(words.len());
-
-    // check if the original query doesn't already contain
-    // the replacement words
-    !original.map(AsRef::as_ref).eq(words.iter().map(AsRef::as_ref))
-}
-
-type Origin = usize;
-type RealLength = usize;
-
-struct FakeIntervalTree {
-    intervals: Vec<(Range<usize>, (Origin, RealLength))>,
-}
-
-impl FakeIntervalTree {
-    fn new(mut intervals: Vec<(Range<usize>, (Origin, RealLength))>) -> FakeIntervalTree {
-        intervals.sort_unstable_by_key(|(r, _)| (r.start, r.end));
-        FakeIntervalTree { intervals }
-    }
-
-    fn query(&self, point: usize) -> Option<(Range<usize>, (Origin, RealLength))> {
-        let element = self.intervals.binary_search_by(|(r, _)| {
-            if point >= r.start {
-                if point < r.end { Equal } else { Less }
-            } else { Greater }
-        });
-
-        let n = match element { Ok(n) => n, Err(n) => n };
-
-        match self.intervals.get(n) {
-            Some((range, value)) if range.contains(&point) => Some((range.clone(), *value)),
-            _otherwise => None,
-        }
-    }
-}
-
-pub struct QueryEnhancerBuilder<'a, S> {
-    query: &'a [S],
-    origins: Vec<usize>,
-    real_to_origin: Vec<(Range<usize>, (Origin, RealLength))>,
-}
-
-impl<S: AsRef<str>> QueryEnhancerBuilder<'_, S> {
-    pub fn new(query: &[S]) -> QueryEnhancerBuilder<S> {
-        // we initialize origins query indices based on their positions
-        let origins: Vec<_> = (0..query.len() + 1).collect();
-        let real_to_origin = origins.iter().map(|&o| (o..o+1, (o, 1))).collect();
-
-        QueryEnhancerBuilder { query, origins, real_to_origin }
-    }
-
-    /// Update the final real to origin query indices mapping.
-    ///
-    /// `range` is the original words range that this `replacement` words replace
-    /// and `real` is the first real query index of these replacement words.
-    pub fn declare<T>(&mut self, range: Range<usize>, real: usize, replacement: &[T])
-    where T: AsRef<str>,
-    {
-        // check if the range of original words
-        // can be rewritten with the replacement words
-        if rewrite_range_with(self.query, range.clone(), replacement) {
-
-            // this range can be replaced so we need to
-            // modify the origins accordingly
-            let offset = replacement.len() - range.len();
-
-            let previous_padding = self.origins[range.end - 1];
-            let current_offset = (self.origins[range.end] - 1) - previous_padding;
-            let diff = offset.saturating_sub(current_offset);
-            self.origins[range.end] += diff;
-
-            for r in &mut self.origins[range.end + 1..] {
-                *r += diff;
-            }
-        }
-
-        // we need to store the real number and origins relations
-        // this way it will be possible to know by how many
-        // we need to pad real query indices
-        let real_range = real..real + replacement.len().max(range.len());
-        let real_length = replacement.len();
-        self.real_to_origin.push((real_range, (range.start, real_length)));
-    }
-
-    pub fn build(self) -> QueryEnhancer {
-        QueryEnhancer {
-            origins: self.origins,
-            real_to_origin: FakeIntervalTree::new(self.real_to_origin),
-        }
-    }
-}
-
-pub struct QueryEnhancer {
-    origins: Vec<usize>,
-    real_to_origin: FakeIntervalTree,
-}
-
-impl QueryEnhancer {
-    /// Returns the query indices to use to replace this real query index.
-    pub fn replacement(&self, real: u32) -> Range<u32> {
-        let real = real as usize;
-
-        // query the fake interval tree with the real query index
-        let (range, (origin, real_length)) =
-            self.real_to_origin
-                .query(real)
-                .expect("real has never been declared");
-
-        // if `real` is the end bound of the range
-        if (range.start + real_length - 1) == real {
-            let mut count = range.len();
-            let mut new_origin = origin;
-            for (i, slice) in self.origins[new_origin..].windows(2).enumerate() {
-                let len = slice[1] - slice[0];
-                count = count.saturating_sub(len);
-                if count == 0 { new_origin = origin + i; break }
-            }
-
-            let n = real - range.start;
-            let start = self.origins[origin];
-            let end = self.origins[new_origin + 1];
-            let remaining = (end - start) - n;
-
-            Range { start: (start + n) as u32, end: (start + n + remaining) as u32 }
-
-        } else {
-            // just return the origin along with
-            // the real position of the word
-            let n = real as usize - range.start;
-            let origin = self.origins[origin];
-
-            Range { start: (origin + n) as u32, end: (origin + n + 1) as u32 }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn original_unmodified() {
-        let query = ["new", "york", "city", "subway"];
-        //             0       1       2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // new york = new york city
-        builder.declare(0..2, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // new
-        assert_eq!(enhancer.replacement(1), 1..2); // york
-        assert_eq!(enhancer.replacement(2), 2..3); // city
-        assert_eq!(enhancer.replacement(3), 3..4); // subway
-        assert_eq!(enhancer.replacement(4), 0..1); // new
-        assert_eq!(enhancer.replacement(5), 1..2); // york
-        assert_eq!(enhancer.replacement(6), 2..3); // city
-    }
-
-    #[test]
-    fn simple_growing() {
-        let query = ["new", "york", "subway"];
-        //             0       1        2
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // new york = new york city
-        builder.declare(0..2, 3, &["new", "york", "city"]);
-        //                    ^      3       4       5
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // new
-        assert_eq!(enhancer.replacement(1), 1..3); // york
-        assert_eq!(enhancer.replacement(2), 3..4); // subway
-        assert_eq!(enhancer.replacement(3), 0..1); // new
-        assert_eq!(enhancer.replacement(4), 1..2); // york
-        assert_eq!(enhancer.replacement(5), 2..3); // city
-    }
-
-    #[test]
-    fn same_place_growings() {
-        let query = ["NY", "subway"];
-        //             0       1
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NY = new york
-        builder.declare(0..1, 2, &["new", "york"]);
-        //                    ^      2       3
-
-        // NY = new york city
-        builder.declare(0..1, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // NY = NYC
-        builder.declare(0..1, 7, &["NYC"]);
-        //                    ^      7
-
-        // NY = new york city
-        builder.declare(0..1, 8, &["new", "york", "city"]);
-        //                    ^      8       9      10
-
-        // subway = underground train
-        builder.declare(1..2, 11, &["underground", "train"]);
-        //                    ^          11          12
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..3); // NY
-        assert_eq!(enhancer.replacement(1), 3..5); // subway
-        assert_eq!(enhancer.replacement(2), 0..1); // new
-        assert_eq!(enhancer.replacement(3), 1..3); // york
-        assert_eq!(enhancer.replacement(4), 0..1); // new
-        assert_eq!(enhancer.replacement(5), 1..2); // york
-        assert_eq!(enhancer.replacement(6), 2..3); // city
-        assert_eq!(enhancer.replacement(7), 0..3); // NYC
-        assert_eq!(enhancer.replacement(8), 0..1); // new
-        assert_eq!(enhancer.replacement(9), 1..2); // york
-        assert_eq!(enhancer.replacement(10), 2..3); // city
-        assert_eq!(enhancer.replacement(11), 3..4); // underground
-        assert_eq!(enhancer.replacement(12), 4..5); // train
-    }
-
-    #[test]
-    fn bigger_growing() {
-        let query = ["NYC", "subway"];
-        //             0        1
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(0..1, 2, &["new", "york", "city"]);
-        //                    ^      2       3       4
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..3); // NYC
-        assert_eq!(enhancer.replacement(1), 3..4); // subway
-        assert_eq!(enhancer.replacement(2), 0..1); // new
-        assert_eq!(enhancer.replacement(3), 1..2); // york
-        assert_eq!(enhancer.replacement(4), 2..3); // city
-    }
-
-    #[test]
-    fn middle_query_growing() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // great
-        assert_eq!(enhancer.replacement(1), 1..2); // awesome
-        assert_eq!(enhancer.replacement(2), 2..5); // NYC
-        assert_eq!(enhancer.replacement(3), 5..6); // subway
-        assert_eq!(enhancer.replacement(4), 2..3); // new
-        assert_eq!(enhancer.replacement(5), 3..4); // york
-        assert_eq!(enhancer.replacement(6), 4..5); // city
-    }
-
-    #[test]
-    fn end_query_growing() {
-        let query = ["NYC", "subway"];
-        //             0        1
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(1..2, 2, &["underground", "train"]);
-        //                    ^         2            3
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // NYC
-        assert_eq!(enhancer.replacement(1), 1..3); // subway
-        assert_eq!(enhancer.replacement(2), 1..2); // underground
-        assert_eq!(enhancer.replacement(3), 2..3); // train
-    }
-
-    #[test]
-    fn multiple_growings() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // subway = underground train
-        builder.declare(3..4, 7, &["underground", "train"]);
-        //                    ^          7           8
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // great
-        assert_eq!(enhancer.replacement(1), 1..2); // awesome
-        assert_eq!(enhancer.replacement(2), 2..5); // NYC
-        assert_eq!(enhancer.replacement(3), 5..7); // subway
-        assert_eq!(enhancer.replacement(4), 2..3); // new
-        assert_eq!(enhancer.replacement(5), 3..4); // york
-        assert_eq!(enhancer.replacement(6), 4..5); // city
-        assert_eq!(enhancer.replacement(7), 5..6); // underground
-        assert_eq!(enhancer.replacement(8), 6..7); // train
-    }
-
-    #[test]
-    fn multiple_probable_growings() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // subway = underground train
-        builder.declare(3..4, 7, &["underground", "train"]);
-        //                    ^          7           8
-
-        // great awesome = good
-        builder.declare(0..2, 9, &["good"]);
-        //                    ^       9
-
-        // awesome NYC = NY
-        builder.declare(1..3, 10, &["NY"]);
-        //                    ^^     10
-
-        // NYC subway = metro
-        builder.declare(2..4, 11, &["metro"]);
-        //                    ^^      11
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0),  0..1); // great
-        assert_eq!(enhancer.replacement(1),  1..2); // awesome
-        assert_eq!(enhancer.replacement(2),  2..5); // NYC
-        assert_eq!(enhancer.replacement(3),  5..7); // subway
-        assert_eq!(enhancer.replacement(4),  2..3); // new
-        assert_eq!(enhancer.replacement(5),  3..4); // york
-        assert_eq!(enhancer.replacement(6),  4..5); // city
-        assert_eq!(enhancer.replacement(7),  5..6); // underground
-        assert_eq!(enhancer.replacement(8),  6..7); // train
-        assert_eq!(enhancer.replacement(9),  0..2); // good
-        assert_eq!(enhancer.replacement(10), 1..5); // NY
-        assert_eq!(enhancer.replacement(11), 2..5); // metro
-    }
-}

meilidb-core/src/raw_document.rs

@@ -1,186 +0,0 @@
-use std::fmt;
-use std::sync::Arc;
-
-use meilidb_schema::SchemaAttr;
-use sdset::SetBuf;
-use slice_group_by::GroupBy;
-
-use crate::{DocumentId, Highlight, TmpMatch};
-
-#[derive(Clone)]
-pub struct RawDocument {
-    pub id: DocumentId,
-    pub matches: SharedMatches,
-    pub highlights: Vec<Highlight>,
-    pub fields_counts: SetBuf<(SchemaAttr, u64)>,
-}
-
-impl RawDocument {
-    pub fn query_index(&self) -> &[u32] {
-        let r = self.matches.range;
-        // it is safe because construction/modifications
-        // can only be done in this module
-        unsafe {
-            &self
-                .matches
-                .matches
-                .query_index
-                .get_unchecked(r.start..r.end)
-        }
-    }
-
-    pub fn distance(&self) -> &[u8] {
-        let r = self.matches.range;
-        // it is safe because construction/modifications
-        // can only be done in this module
-        unsafe { &self.matches.matches.distance.get_unchecked(r.start..r.end) }
-    }
-
-    pub fn attribute(&self) -> &[u16] {
-        let r = self.matches.range;
-        // it is safe because construction/modifications
-        // can only be done in this module
-        unsafe { &self.matches.matches.attribute.get_unchecked(r.start..r.end) }
-    }
-
-    pub fn word_index(&self) -> &[u16] {
-        let r = self.matches.range;
-        // it is safe because construction/modifications
-        // can only be done in this module
-        unsafe {
-            &self
-                .matches
-                .matches
-                .word_index
-                .get_unchecked(r.start..r.end)
-        }
-    }
-
-    pub fn is_exact(&self) -> &[bool] {
-        let r = self.matches.range;
-        // it is safe because construction/modifications
-        // can only be done in this module
-        unsafe { &self.matches.matches.is_exact.get_unchecked(r.start..r.end) }
-    }
-}
-
-impl fmt::Debug for RawDocument {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        f.write_str("RawDocument {\r\n")?;
-        f.write_fmt(format_args!("{:>15}: {:?},\r\n", "id", self.id))?;
-        f.write_fmt(format_args!(
-            "{:>15}: {:^5?},\r\n",
-            "query_index",
-            self.query_index()
-        ))?;
-        f.write_fmt(format_args!(
-            "{:>15}: {:^5?},\r\n",
-            "distance",
-            self.distance()
-        ))?;
-        f.write_fmt(format_args!(
-            "{:>15}: {:^5?},\r\n",
-            "attribute",
-            self.attribute()
-        ))?;
-        f.write_fmt(format_args!(
-            "{:>15}: {:^5?},\r\n",
-            "word_index",
-            self.word_index()
-        ))?;
-        f.write_fmt(format_args!(
-            "{:>15}: {:^5?},\r\n",
-            "is_exact",
-            self.is_exact()
-        ))?;
-        f.write_str("}")?;
-        Ok(())
-    }
-}
-
-pub fn raw_documents_from(
-    matches: SetBuf<(DocumentId, TmpMatch)>,
-    highlights: SetBuf<(DocumentId, Highlight)>,
-    fields_counts: SetBuf<(DocumentId, SchemaAttr, u64)>,
-) -> Vec<RawDocument> {
-    let mut docs_ranges: Vec<(_, Range, _, _)> = Vec::new();
-    let mut matches2 = Matches::with_capacity(matches.len());
-
-    let matches = matches.linear_group_by_key(|(id, _)| *id);
-    let highlights = highlights.linear_group_by_key(|(id, _)| *id);
-    let fields_counts = fields_counts.linear_group_by_key(|(id, _, _)| *id);
-
-    for ((mgroup, hgroup), fgroup) in matches.zip(highlights).zip(fields_counts) {
-        debug_assert_eq!(mgroup[0].0, hgroup[0].0);
-        debug_assert_eq!(mgroup[0].0, fgroup[0].0);
-
-        let document_id = mgroup[0].0;
-        let start = docs_ranges.last().map(|(_, r, _, _)| r.end).unwrap_or(0);
-        let end = start + mgroup.len();
-        let highlights = hgroup.iter().map(|(_, h)| *h).collect();
-        let fields_counts = SetBuf::new(fgroup.iter().map(|(_, a, c)| (*a, *c)).collect()).unwrap();
-
-        docs_ranges.push((document_id, Range { start, end }, highlights, fields_counts));
-        matches2.extend_from_slice(mgroup);
-    }
-
-    let matches = Arc::new(matches2);
-    docs_ranges
-        .into_iter()
-        .map(|(id, range, highlights, fields_counts)| {
-            let matches = SharedMatches {
-                range,
-                matches: matches.clone(),
-            };
-            RawDocument {
-                id,
-                matches,
-                highlights,
-                fields_counts,
-            }
-        })
-        .collect()
-}
-
-#[derive(Debug, Copy, Clone)]
-struct Range {
-    start: usize,
-    end: usize,
-}
-
-#[derive(Clone)]
-pub struct SharedMatches {
-    range: Range,
-    matches: Arc<Matches>,
-}
-
-#[derive(Clone)]
-struct Matches {
-    query_index: Vec<u32>,
-    distance: Vec<u8>,
-    attribute: Vec<u16>,
-    word_index: Vec<u16>,
-    is_exact: Vec<bool>,
-}
-
-impl Matches {
-    fn with_capacity(cap: usize) -> Matches {
-        Matches {
-            query_index: Vec::with_capacity(cap),
-            distance: Vec::with_capacity(cap),
-            attribute: Vec::with_capacity(cap),
-            word_index: Vec::with_capacity(cap),
-            is_exact: Vec::with_capacity(cap),
-        }
-    }
-
-    fn extend_from_slice(&mut self, matches: &[(DocumentId, TmpMatch)]) {
-        for (_, match_) in matches {
-            self.query_index.push(match_.query_index);
-            self.distance.push(match_.distance);
-            self.attribute.push(match_.attribute);
-            self.word_index.push(match_.word_index);
-            self.is_exact.push(match_.is_exact);
-        }
-    }
-}

meilidb-core/src/reordered_attrs.rs

@@ -1,27 +0,0 @@
-#[derive(Default, Clone)]
-pub struct ReorderedAttrs {
-    count: usize,
-    reorders: Vec<Option<u16>>,
-}
-
-impl ReorderedAttrs {
-    pub fn new() -> ReorderedAttrs {
-        ReorderedAttrs {
-            count: 0,
-            reorders: Vec::new(),
-        }
-    }
-
-    pub fn insert_attribute(&mut self, attribute: u16) {
-        self.reorders.resize(attribute as usize + 1, None);
-        self.reorders[attribute as usize] = Some(self.count as u16);
-        self.count += 1;
-    }
-
-    pub fn get(&self, attribute: u16) -> Option<u16> {
-        match self.reorders.get(attribute as usize) {
-            Some(Some(attribute)) => Some(*attribute),
-            _ => None,
-        }
-    }
-}

meilidb-core/src/serde/convert_to_number.rs

@@ -1,198 +0,0 @@
-use std::str::FromStr;
-
-use ordered_float::OrderedFloat;
-use serde::ser;
-use serde::Serialize;
-
-use super::SerializerError;
-use crate::Number;
-
-pub struct ConvertToNumber;
-
-impl ser::Serializer for ConvertToNumber {
-    type Ok = Number;
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    fn serialize_bool(self, value: bool) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_char(self, _value: char) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "char" })
-    }
-
-    fn serialize_i8(self, value: i8) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(i64::from(value)))
-    }
-
-    fn serialize_i16(self, value: i16) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(i64::from(value)))
-    }
-
-    fn serialize_i32(self, value: i32) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(i64::from(value)))
-    }
-
-    fn serialize_i64(self, value: i64) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(value))
-    }
-
-    fn serialize_u8(self, value: u8) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_u16(self, value: u16) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_u32(self, value: u32) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_u64(self, value: u64) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(value))
-    }
-
-    fn serialize_f32(self, value: f32) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Float(OrderedFloat(f64::from(value))))
-    }
-
-    fn serialize_f64(self, value: f64) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Float(OrderedFloat(value)))
-    }
-
-    fn serialize_str(self, value: &str) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::from_str(value)?)
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnrankableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnrankableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "map" })
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "struct",
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "struct variant",
-        })
-    }
-}

meilidb-core/src/serde/convert_to_string.rs

@@ -1,258 +0,0 @@
-use serde::ser;
-use serde::Serialize;
-
-use super::SerializerError;
-
-pub struct ConvertToString;
-
-impl ser::Serializer for ConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = MapConvertToString;
-    type SerializeStruct = StructConvertToString;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    fn serialize_bool(self, _value: bool) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "boolean",
-        })
-    }
-
-    fn serialize_char(self, value: char) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i8(self, value: i8) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i16(self, value: i16) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i32(self, value: i32) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i64(self, value: i64) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u8(self, value: u8) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u16(self, value: u16) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u32(self, value: u32) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u64(self, value: u64) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_f32(self, value: f32) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_f64(self, value: f64) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_str(self, value: &str) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        Ok(MapConvertToString {
-            text: String::new(),
-        })
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Ok(StructConvertToString {
-            text: String::new(),
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct MapConvertToString {
-    text: String,
-}
-
-impl ser::SerializeMap for MapConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = key.serialize(ConvertToString)?;
-        self.text.push_str(&text);
-        self.text.push_str(" ");
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.text.push_str(&text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.text)
-    }
-}
-
-pub struct StructConvertToString {
-    text: String,
-}
-
-impl ser::SerializeStruct for StructConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let value = value.serialize(ConvertToString)?;
-        self.text.push_str(key);
-        self.text.push_str(" ");
-        self.text.push_str(&value);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.text)
-    }
-}

meilidb-core/src/serde/extract_document_id.rs

@@ -1,295 +0,0 @@
-use std::hash::{Hash, Hasher};
-
-use crate::DocumentId;
-use serde::{ser, Serialize};
-use serde_json::Value;
-use siphasher::sip::SipHasher;
-
-use super::{ConvertToString, SerializerError};
-
-pub fn extract_document_id<D>(
-    identifier: &str,
-    document: &D,
-) -> Result<Option<DocumentId>, SerializerError>
-where
-    D: serde::Serialize,
-{
-    let serializer = ExtractDocumentId { identifier };
-    document.serialize(serializer)
-}
-
-pub fn value_to_string(value: &Value) -> Option<String> {
-    match value {
-        Value::Null => None,
-        Value::Bool(_) => None,
-        Value::Number(value) => Some(value.to_string()),
-        Value::String(value) => Some(value.to_string()),
-        Value::Array(_) => None,
-        Value::Object(_) => None,
-    }
-}
-
-pub fn compute_document_id<H: Hash>(t: H) -> DocumentId {
-    let mut s = SipHasher::new();
-    t.hash(&mut s);
-    let hash = s.finish();
-    DocumentId(hash)
-}
-
-struct ExtractDocumentId<'a> {
-    identifier: &'a str,
-}
-
-impl<'a> ser::Serializer for ExtractDocumentId<'a> {
-    type Ok = Option<DocumentId>;
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = ExtractDocumentIdMapSerializer<'a>;
-    type SerializeStruct = ExtractDocumentIdStructSerializer<'a>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    forward_to_unserializable_type! {
-        bool => serialize_bool,
-        char => serialize_char,
-
-        i8  => serialize_i8,
-        i16 => serialize_i16,
-        i32 => serialize_i32,
-        i64 => serialize_i64,
-
-        u8  => serialize_u8,
-        u16 => serialize_u16,
-        u32 => serialize_u32,
-        u64 => serialize_u64,
-
-        f32 => serialize_f32,
-        f64 => serialize_f64,
-    }
-
-    fn serialize_str(self, _value: &str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "str" })
-    }
-
-    fn serialize_bytes(self, _value: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        let serializer = ExtractDocumentIdMapSerializer {
-            identifier: self.identifier,
-            document_id: None,
-            current_key_name: None,
-        };
-
-        Ok(serializer)
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        let serializer = ExtractDocumentIdStructSerializer {
-            identifier: self.identifier,
-            document_id: None,
-        };
-
-        Ok(serializer)
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct ExtractDocumentIdMapSerializer<'a> {
-    identifier: &'a str,
-    document_id: Option<DocumentId>,
-    current_key_name: Option<String>,
-}
-
-impl<'a> ser::SerializeMap for ExtractDocumentIdMapSerializer<'a> {
-    type Ok = Option<DocumentId>;
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-        self.current_key_name = Some(key);
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        let key = self.current_key_name.take().unwrap();
-        self.serialize_entry(&key, value)
-    }
-
-    fn serialize_entry<K: ?Sized, V: ?Sized>(
-        &mut self,
-        key: &K,
-        value: &V,
-    ) -> Result<(), Self::Error>
-    where
-        K: Serialize,
-        V: Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-
-        if self.identifier == key {
-            let value = serde_json::to_string(value).and_then(|s| serde_json::from_str(&s))?;
-            match value_to_string(&value).map(|s| compute_document_id(&s)) {
-                Some(document_id) => self.document_id = Some(document_id),
-                None => return Err(SerializerError::InvalidDocumentIdType),
-            }
-        }
-
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.document_id)
-    }
-}
-
-pub struct ExtractDocumentIdStructSerializer<'a> {
-    identifier: &'a str,
-    document_id: Option<DocumentId>,
-}
-
-impl<'a> ser::SerializeStruct for ExtractDocumentIdStructSerializer<'a> {
-    type Ok = Option<DocumentId>;
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        if self.identifier == key {
-            let value = serde_json::to_string(value).and_then(|s| serde_json::from_str(&s))?;
-            match value_to_string(&value).map(compute_document_id) {
-                Some(document_id) => self.document_id = Some(document_id),
-                None => return Err(SerializerError::InvalidDocumentIdType),
-            }
-        }
-
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.document_id)
-    }
-}

meilidb-core/src/serde/indexer.rs

@@ -1,365 +0,0 @@
-use meilidb_schema::SchemaAttr;
-use serde::ser;
-use serde::Serialize;
-
-use super::{ConvertToString, SerializerError};
-use crate::raw_indexer::RawIndexer;
-use crate::DocumentId;
-
-pub struct Indexer<'a> {
-    pub attribute: SchemaAttr,
-    pub indexer: &'a mut RawIndexer,
-    pub document_id: DocumentId,
-}
-
-impl<'a> ser::Serializer for Indexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-    type SerializeSeq = SeqIndexer<'a>;
-    type SerializeTuple = TupleIndexer<'a>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = MapIndexer<'a>;
-    type SerializeStruct = StructIndexer<'a>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    fn serialize_bool(self, _value: bool) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "boolean",
-        })
-    }
-
-    fn serialize_char(self, value: char) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i8(self, value: i8) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i16(self, value: i16) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i32(self, value: i32) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i64(self, value: i64) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u8(self, value: u8) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u16(self, value: u16) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u32(self, value: u32) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u64(self, value: u64) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_f32(self, value: f32) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_f64(self, value: f64) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_str(self, text: &str) -> Result<Self::Ok, Self::Error> {
-        let number_of_words = self
-            .indexer
-            .index_text(self.document_id, self.attribute, text);
-        Ok(Some(number_of_words))
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        let number_of_words = self
-            .indexer
-            .index_text(self.document_id, self.attribute, &text);
-        Ok(Some(number_of_words))
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        Err(SerializerError::UnindexableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        let indexer = SeqIndexer {
-            attribute: self.attribute,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        let indexer = TupleIndexer {
-            attribute: self.attribute,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        let indexer = MapIndexer {
-            attribute: self.attribute,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "struct",
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct SeqIndexer<'a> {
-    attribute: SchemaAttr,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeSeq for SeqIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_element<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.attribute, texts);
-        Ok(None)
-    }
-}
-
-pub struct MapIndexer<'a> {
-    attribute: SchemaAttr,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeMap for MapIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = key.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.attribute, texts);
-        Ok(None)
-    }
-}
-
-pub struct StructIndexer<'a> {
-    attribute: SchemaAttr,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeStruct for StructIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let key_text = key.to_owned();
-        let value_text = value.serialize(ConvertToString)?;
-        self.texts.push(key_text);
-        self.texts.push(value_text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.attribute, texts);
-        Ok(None)
-    }
-}
-
-pub struct TupleIndexer<'a> {
-    attribute: SchemaAttr,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeTuple for TupleIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_element<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.attribute, texts);
-        Ok(None)
-    }
-}

meilidb-core/src/serde/serializer.rs

@@ -1,338 +0,0 @@
-use meilidb_schema::{Schema, SchemaAttr, SchemaProps};
-use serde::ser;
-
-use crate::raw_indexer::RawIndexer;
-use crate::store::{DocumentsFields, DocumentsFieldsCounts};
-use crate::{DocumentId, RankedMap};
-
-use super::{ConvertToNumber, ConvertToString, Indexer, SerializerError};
-
-pub struct Serializer<'a, 'b> {
-    pub txn: &'a mut heed::RwTxn<'b>,
-    pub schema: &'a Schema,
-    pub document_store: DocumentsFields,
-    pub document_fields_counts: DocumentsFieldsCounts,
-    pub indexer: &'a mut RawIndexer,
-    pub ranked_map: &'a mut RankedMap,
-    pub document_id: DocumentId,
-}
-
-impl<'a, 'b> ser::Serializer for Serializer<'a, 'b> {
-    type Ok = ();
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = MapSerializer<'a, 'b>;
-    type SerializeStruct = StructSerializer<'a, 'b>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    forward_to_unserializable_type! {
-        bool => serialize_bool,
-        char => serialize_char,
-
-        i8  => serialize_i8,
-        i16 => serialize_i16,
-        i32 => serialize_i32,
-        i64 => serialize_i64,
-
-        u8  => serialize_u8,
-        u16 => serialize_u16,
-        u32 => serialize_u32,
-        u64 => serialize_u64,
-
-        f32 => serialize_f32,
-        f64 => serialize_f64,
-    }
-
-    fn serialize_str(self, _v: &str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "str" })
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        Ok(MapSerializer {
-            txn: self.txn,
-            schema: self.schema,
-            document_id: self.document_id,
-            document_store: self.document_store,
-            document_fields_counts: self.document_fields_counts,
-            indexer: self.indexer,
-            ranked_map: self.ranked_map,
-            current_key_name: None,
-        })
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Ok(StructSerializer {
-            txn: self.txn,
-            schema: self.schema,
-            document_id: self.document_id,
-            document_store: self.document_store,
-            document_fields_counts: self.document_fields_counts,
-            indexer: self.indexer,
-            ranked_map: self.ranked_map,
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct MapSerializer<'a, 'b> {
-    txn: &'a mut heed::RwTxn<'b>,
-    schema: &'a Schema,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    document_fields_counts: DocumentsFieldsCounts,
-    indexer: &'a mut RawIndexer,
-    ranked_map: &'a mut RankedMap,
-    current_key_name: Option<String>,
-}
-
-impl<'a, 'b> ser::SerializeMap for MapSerializer<'a, 'b> {
-    type Ok = ();
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-        self.current_key_name = Some(key);
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let key = self.current_key_name.take().unwrap();
-        self.serialize_entry(&key, value)
-    }
-
-    fn serialize_entry<K: ?Sized, V: ?Sized>(
-        &mut self,
-        key: &K,
-        value: &V,
-    ) -> Result<(), Self::Error>
-    where
-        K: ser::Serialize,
-        V: ser::Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-        match self.schema.attribute(&key) {
-            Some(attribute) => serialize_value(
-                self.txn,
-                attribute,
-                self.schema.props(attribute),
-                self.document_id,
-                self.document_store,
-                self.document_fields_counts,
-                self.indexer,
-                self.ranked_map,
-                value,
-            ),
-            None => Ok(()),
-        }
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(())
-    }
-}
-
-pub struct StructSerializer<'a, 'b> {
-    txn: &'a mut heed::RwTxn<'b>,
-    schema: &'a Schema,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    document_fields_counts: DocumentsFieldsCounts,
-    indexer: &'a mut RawIndexer,
-    ranked_map: &'a mut RankedMap,
-}
-
-impl<'a, 'b> ser::SerializeStruct for StructSerializer<'a, 'b> {
-    type Ok = ();
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        match self.schema.attribute(key) {
-            Some(attribute) => serialize_value(
-                self.txn,
-                attribute,
-                self.schema.props(attribute),
-                self.document_id,
-                self.document_store,
-                self.document_fields_counts,
-                self.indexer,
-                self.ranked_map,
-                value,
-            ),
-            None => Ok(()),
-        }
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(())
-    }
-}
-
-pub fn serialize_value<T: ?Sized>(
-    txn: &mut heed::RwTxn,
-    attribute: SchemaAttr,
-    props: SchemaProps,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    documents_fields_counts: DocumentsFieldsCounts,
-    indexer: &mut RawIndexer,
-    ranked_map: &mut RankedMap,
-    value: &T,
-) -> Result<(), SerializerError>
-where
-    T: ser::Serialize,
-{
-    let serialized = serde_json::to_vec(value)?;
-    document_store.put_document_field(txn, document_id, attribute, &serialized)?;
-
-    if props.is_indexed() {
-        let indexer = Indexer {
-            attribute,
-            indexer,
-            document_id,
-        };
-        if let Some(number_of_words) = value.serialize(indexer)? {
-            documents_fields_counts.put_document_field_count(
-                txn,
-                document_id,
-                attribute,
-                number_of_words as u64,
-            )?;
-        }
-    }
-
-    if props.is_ranked() {
-        let number = value.serialize(ConvertToNumber)?;
-        ranked_map.insert(document_id, attribute, number);
-    }
-
-    Ok(())
-}

meilidb-core/src/store/docs_words.rs

@@ -1,49 +0,0 @@
-use super::BEU64;
-use crate::DocumentId;
-use heed::types::{ByteSlice, OwnedType};
-use heed::Result as ZResult;
-use std::sync::Arc;
-
-#[derive(Copy, Clone)]
-pub struct DocsWords {
-    pub(crate) docs_words: heed::Database<OwnedType<BEU64>, ByteSlice>,
-}
-
-impl DocsWords {
-    pub fn put_doc_words(
-        self,
-        writer: &mut heed::RwTxn,
-        document_id: DocumentId,
-        words: &fst::Set,
-    ) -> ZResult<()> {
-        let document_id = BEU64::new(document_id.0);
-        let bytes = words.as_fst().as_bytes();
-        self.docs_words.put(writer, &document_id, bytes)
-    }
-
-    pub fn del_doc_words(self, writer: &mut heed::RwTxn, document_id: DocumentId) -> ZResult<bool> {
-        let document_id = BEU64::new(document_id.0);
-        self.docs_words.delete(writer, &document_id)
-    }
-
-    pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> {
-        self.docs_words.clear(writer)
-    }
-
-    pub fn doc_words(
-        self,
-        reader: &heed::RoTxn,
-        document_id: DocumentId,
-    ) -> ZResult<Option<fst::Set>> {
-        let document_id = BEU64::new(document_id.0);
-        match self.docs_words.get(reader, &document_id)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::from(bytes);
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
-        }
-    }
-}

meilidb-core/src/store/main.rs

@@ -1,121 +0,0 @@
-use crate::RankedMap;
-use heed::types::{ByteSlice, OwnedType, SerdeBincode, Str};
-use heed::Result as ZResult;
-use meilidb_schema::Schema;
-use std::sync::Arc;
-
-const CUSTOMS_KEY: &str = "customs-key";
-const NUMBER_OF_DOCUMENTS_KEY: &str = "number-of-documents";
-const RANKED_MAP_KEY: &str = "ranked-map";
-const SCHEMA_KEY: &str = "schema";
-const SYNONYMS_KEY: &str = "synonyms";
-const STOP_WORDS_KEY: &str = "stop-words";
-const WORDS_KEY: &str = "words";
-
-#[derive(Copy, Clone)]
-pub struct Main {
-    pub(crate) main: heed::PolyDatabase,
-}
-
-impl Main {
-    pub fn put_words_fst(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> ZResult<()> {
-        let bytes = fst.as_fst().as_bytes();
-        self.main.put::<Str, ByteSlice>(writer, WORDS_KEY, bytes)
-    }
-
-    pub fn words_fst(self, reader: &heed::RoTxn) -> ZResult<Option<fst::Set>> {
-        match self.main.get::<Str, ByteSlice>(reader, WORDS_KEY)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::from(bytes);
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
-        }
-    }
-
-    pub fn put_schema(self, writer: &mut heed::RwTxn, schema: &Schema) -> ZResult<()> {
-        self.main
-            .put::<Str, SerdeBincode<Schema>>(writer, SCHEMA_KEY, schema)
-    }
-
-    pub fn schema(self, reader: &heed::RoTxn) -> ZResult<Option<Schema>> {
-        self.main
-            .get::<Str, SerdeBincode<Schema>>(reader, SCHEMA_KEY)
-    }
-
-    pub fn put_ranked_map(self, writer: &mut heed::RwTxn, ranked_map: &RankedMap) -> ZResult<()> {
-        self.main
-            .put::<Str, SerdeBincode<RankedMap>>(writer, RANKED_MAP_KEY, &ranked_map)
-    }
-
-    pub fn ranked_map(self, reader: &heed::RoTxn) -> ZResult<Option<RankedMap>> {
-        self.main
-            .get::<Str, SerdeBincode<RankedMap>>(reader, RANKED_MAP_KEY)
-    }
-
-    pub fn put_synonyms_fst(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> ZResult<()> {
-        let bytes = fst.as_fst().as_bytes();
-        self.main.put::<Str, ByteSlice>(writer, SYNONYMS_KEY, bytes)
-    }
-
-    pub fn synonyms_fst(self, reader: &heed::RoTxn) -> ZResult<Option<fst::Set>> {
-        match self.main.get::<Str, ByteSlice>(reader, SYNONYMS_KEY)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::from(bytes);
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
-        }
-    }
-
-    pub fn put_stop_words_fst(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> ZResult<()> {
-        let bytes = fst.as_fst().as_bytes();
-        self.main
-            .put::<Str, ByteSlice>(writer, STOP_WORDS_KEY, bytes)
-    }
-
-    pub fn stop_words_fst(self, reader: &heed::RoTxn) -> ZResult<Option<fst::Set>> {
-        match self.main.get::<Str, ByteSlice>(reader, STOP_WORDS_KEY)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::from(bytes);
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
-        }
-    }
-
-    pub fn put_number_of_documents<F>(self, writer: &mut heed::RwTxn, f: F) -> ZResult<u64>
-    where
-        F: Fn(u64) -> u64,
-    {
-        let new = self.number_of_documents(writer).map(f)?;
-        self.main
-            .put::<Str, OwnedType<u64>>(writer, NUMBER_OF_DOCUMENTS_KEY, &new)?;
-        Ok(new)
-    }
-
-    pub fn number_of_documents(self, reader: &heed::RoTxn) -> ZResult<u64> {
-        match self
-            .main
-            .get::<Str, OwnedType<u64>>(reader, NUMBER_OF_DOCUMENTS_KEY)?
-        {
-            Some(value) => Ok(value),
-            None => Ok(0),
-        }
-    }
-
-    pub fn put_customs(self, writer: &mut heed::RwTxn, customs: &[u8]) -> ZResult<()> {
-        self.main
-            .put::<Str, ByteSlice>(writer, CUSTOMS_KEY, customs)
-    }
-
-    pub fn customs<'txn>(self, reader: &'txn heed::RoTxn) -> ZResult<Option<&'txn [u8]>> {
-        self.main.get::<Str, ByteSlice>(reader, CUSTOMS_KEY)
-    }
-}

meilidb-core/src/store/mod.rs

@@ -1,372 +0,0 @@
-mod docs_words;
-mod documents_fields;
-mod documents_fields_counts;
-mod main;
-mod postings_lists;
-mod synonyms;
-mod updates;
-mod updates_results;
-
-pub use self::docs_words::DocsWords;
-pub use self::documents_fields::{DocumentFieldsIter, DocumentsFields};
-pub use self::documents_fields_counts::{
-    DocumentFieldsCountsIter, DocumentsFieldsCounts, DocumentsIdsIter,
-};
-pub use self::main::Main;
-pub use self::postings_lists::PostingsLists;
-pub use self::synonyms::Synonyms;
-pub use self::updates::Updates;
-pub use self::updates_results::UpdatesResults;
-
-use std::collections::HashSet;
-
-use heed::Result as ZResult;
-use meilidb_schema::{Schema, SchemaAttr};
-use serde::de;
-use zerocopy::{AsBytes, FromBytes};
-
-use crate::criterion::Criteria;
-use crate::serde::Deserializer;
-use crate::{query_builder::QueryBuilder, update, DocumentId, Error, MResult};
-
-type BEU64 = zerocopy::U64<byteorder::BigEndian>;
-type BEU16 = zerocopy::U16<byteorder::BigEndian>;
-
-#[derive(Debug, Copy, Clone, AsBytes, FromBytes)]
-#[repr(C)]
-pub struct DocumentAttrKey {
-    docid: BEU64,
-    attr: BEU16,
-}
-
-impl DocumentAttrKey {
-    fn new(docid: DocumentId, attr: SchemaAttr) -> DocumentAttrKey {
-        DocumentAttrKey {
-            docid: BEU64::new(docid.0),
-            attr: BEU16::new(attr.0),
-        }
-    }
-}
-
-fn main_name(name: &str) -> String {
-    format!("store-{}", name)
-}
-
-fn postings_lists_name(name: &str) -> String {
-    format!("store-{}-postings-lists", name)
-}
-
-fn documents_fields_name(name: &str) -> String {
-    format!("store-{}-documents-fields", name)
-}
-
-fn documents_fields_counts_name(name: &str) -> String {
-    format!("store-{}-documents-fields-counts", name)
-}
-
-fn synonyms_name(name: &str) -> String {
-    format!("store-{}-synonyms", name)
-}
-
-fn docs_words_name(name: &str) -> String {
-    format!("store-{}-docs-words", name)
-}
-
-fn updates_name(name: &str) -> String {
-    format!("store-{}-updates", name)
-}
-
-fn updates_results_name(name: &str) -> String {
-    format!("store-{}-updates-results", name)
-}
-
-#[derive(Clone)]
-pub struct Index {
-    pub main: Main,
-    pub postings_lists: PostingsLists,
-    pub documents_fields: DocumentsFields,
-    pub documents_fields_counts: DocumentsFieldsCounts,
-    pub synonyms: Synonyms,
-    pub docs_words: DocsWords,
-
-    pub updates: Updates,
-    pub updates_results: UpdatesResults,
-    updates_notifier: crossbeam_channel::Sender<()>,
-}
-
-impl Index {
-    pub fn document<T: de::DeserializeOwned>(
-        &self,
-        reader: &heed::RoTxn,
-        attributes: Option<&HashSet<&str>>,
-        document_id: DocumentId,
-    ) -> MResult<Option<T>> {
-        let schema = self.main.schema(reader)?;
-        let schema = schema.ok_or(Error::SchemaMissing)?;
-
-        let attributes = match attributes {
-            Some(attributes) => attributes
-                .iter()
-                .map(|name| schema.attribute(name))
-                .collect(),
-            None => None,
-        };
-
-        let mut deserializer = Deserializer {
-            document_id,
-            reader,
-            documents_fields: self.documents_fields,
-            schema: &schema,
-            attributes: attributes.as_ref(),
-        };
-
-        // TODO: currently we return an error if all document fields are missing,
-        //       returning None would have been better
-        Ok(T::deserialize(&mut deserializer).map(Some)?)
-    }
-
-    pub fn document_attribute<T: de::DeserializeOwned>(
-        &self,
-        reader: &heed::RoTxn,
-        document_id: DocumentId,
-        attribute: SchemaAttr,
-    ) -> MResult<Option<T>> {
-        let bytes = self
-            .documents_fields
-            .document_attribute(reader, document_id, attribute)?;
-        match bytes {
-            Some(bytes) => Ok(Some(serde_json::from_slice(bytes)?)),
-            None => Ok(None),
-        }
-    }
-
-    pub fn schema_update(&self, writer: &mut heed::RwTxn, schema: Schema) -> MResult<u64> {
-        let _ = self.updates_notifier.send(());
-        update::push_schema_update(writer, self.updates, self.updates_results, schema)
-    }
-
-    pub fn customs_update(&self, writer: &mut heed::RwTxn, customs: Vec<u8>) -> ZResult<u64> {
-        let _ = self.updates_notifier.send(());
-        update::push_customs_update(writer, self.updates, self.updates_results, customs)
-    }
-
-    pub fn documents_addition<D>(&self) -> update::DocumentsAddition<D> {
-        update::DocumentsAddition::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn documents_deletion(&self) -> update::DocumentsDeletion {
-        update::DocumentsDeletion::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn clear_all(&self, writer: &mut heed::RwTxn) -> MResult<u64> {
-        let _ = self.updates_notifier.send(());
-        update::push_clear_all(writer, self.updates, self.updates_results)
-    }
-
-    pub fn synonyms_addition(&self) -> update::SynonymsAddition {
-        update::SynonymsAddition::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn synonyms_deletion(&self) -> update::SynonymsDeletion {
-        update::SynonymsDeletion::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn stop_words_addition(&self) -> update::StopWordsAddition {
-        update::StopWordsAddition::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn stop_words_deletion(&self) -> update::StopWordsDeletion {
-        update::StopWordsDeletion::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn current_update_id(&self, reader: &heed::RoTxn) -> MResult<Option<u64>> {
-        match self.updates.last_update_id(reader)? {
-            Some((id, _)) => Ok(Some(id)),
-            None => Ok(None),
-        }
-    }
-
-    pub fn update_status(
-        &self,
-        reader: &heed::RoTxn,
-        update_id: u64,
-    ) -> MResult<update::UpdateStatus> {
-        update::update_status(reader, self.updates, self.updates_results, update_id)
-    }
-
-    pub fn all_updates_status(&self, reader: &heed::RoTxn) -> MResult<Vec<update::UpdateStatus>> {
-        let mut updates = Vec::new();
-        let mut last_update_result_id = 0;
-
-        // retrieve all updates results
-        if let Some((last_id, _)) = self.updates_results.last_update_id(reader)? {
-            updates.reserve(last_id as usize);
-
-            for id in 0..=last_id {
-                let update = self.update_status(reader, id)?;
-                updates.push(update);
-                last_update_result_id = id;
-            }
-        }
-
-        // retrieve all enqueued updates
-        if let Some((last_id, _)) = self.updates.last_update_id(reader)? {
-            for id in last_update_result_id + 1..last_id {
-                let update = self.update_status(reader, id)?;
-                updates.push(update);
-            }
-        }
-
-        Ok(updates)
-    }
-
-    pub fn query_builder(&self) -> QueryBuilder {
-        QueryBuilder::new(
-            self.main,
-            self.postings_lists,
-            self.documents_fields_counts,
-            self.synonyms,
-        )
-    }
-
-    pub fn query_builder_with_criteria<'c, 'f, 'd>(
-        &self,
-        criteria: Criteria<'c>,
-    ) -> QueryBuilder<'c, 'f, 'd> {
-        QueryBuilder::with_criteria(
-            self.main,
-            self.postings_lists,
-            self.documents_fields_counts,
-            self.synonyms,
-            criteria,
-        )
-    }
-}
-
-pub fn create(
-    env: &heed::Env,
-    name: &str,
-    updates_notifier: crossbeam_channel::Sender<()>,
-) -> MResult<Index> {
-    // create all the store names
-    let main_name = main_name(name);
-    let postings_lists_name = postings_lists_name(name);
-    let documents_fields_name = documents_fields_name(name);
-    let documents_fields_counts_name = documents_fields_counts_name(name);
-    let synonyms_name = synonyms_name(name);
-    let docs_words_name = docs_words_name(name);
-    let updates_name = updates_name(name);
-    let updates_results_name = updates_results_name(name);
-
-    // open all the stores
-    let main = env.create_poly_database(Some(&main_name))?;
-    let postings_lists = env.create_database(Some(&postings_lists_name))?;
-    let documents_fields = env.create_database(Some(&documents_fields_name))?;
-    let documents_fields_counts = env.create_database(Some(&documents_fields_counts_name))?;
-    let synonyms = env.create_database(Some(&synonyms_name))?;
-    let docs_words = env.create_database(Some(&docs_words_name))?;
-    let updates = env.create_database(Some(&updates_name))?;
-    let updates_results = env.create_database(Some(&updates_results_name))?;
-
-    Ok(Index {
-        main: Main { main },
-        postings_lists: PostingsLists { postings_lists },
-        documents_fields: DocumentsFields { documents_fields },
-        documents_fields_counts: DocumentsFieldsCounts {
-            documents_fields_counts,
-        },
-        synonyms: Synonyms { synonyms },
-        docs_words: DocsWords { docs_words },
-        updates: Updates { updates },
-        updates_results: UpdatesResults { updates_results },
-        updates_notifier,
-    })
-}
-
-pub fn open(
-    env: &heed::Env,
-    name: &str,
-    updates_notifier: crossbeam_channel::Sender<()>,
-) -> MResult<Option<Index>> {
-    // create all the store names
-    let main_name = main_name(name);
-    let postings_lists_name = postings_lists_name(name);
-    let documents_fields_name = documents_fields_name(name);
-    let documents_fields_counts_name = documents_fields_counts_name(name);
-    let synonyms_name = synonyms_name(name);
-    let docs_words_name = docs_words_name(name);
-    let updates_name = updates_name(name);
-    let updates_results_name = updates_results_name(name);
-
-    // open all the stores
-    let main = match env.open_poly_database(Some(&main_name))? {
-        Some(main) => main,
-        None => return Ok(None),
-    };
-    let postings_lists = match env.open_database(Some(&postings_lists_name))? {
-        Some(postings_lists) => postings_lists,
-        None => return Ok(None),
-    };
-    let documents_fields = match env.open_database(Some(&documents_fields_name))? {
-        Some(documents_fields) => documents_fields,
-        None => return Ok(None),
-    };
-    let documents_fields_counts = match env.open_database(Some(&documents_fields_counts_name))? {
-        Some(documents_fields_counts) => documents_fields_counts,
-        None => return Ok(None),
-    };
-    let synonyms = match env.open_database(Some(&synonyms_name))? {
-        Some(synonyms) => synonyms,
-        None => return Ok(None),
-    };
-    let docs_words = match env.open_database(Some(&docs_words_name))? {
-        Some(docs_words) => docs_words,
-        None => return Ok(None),
-    };
-    let updates = match env.open_database(Some(&updates_name))? {
-        Some(updates) => updates,
-        None => return Ok(None),
-    };
-    let updates_results = match env.open_database(Some(&updates_results_name))? {
-        Some(updates_results) => updates_results,
-        None => return Ok(None),
-    };
-
-    Ok(Some(Index {
-        main: Main { main },
-        postings_lists: PostingsLists { postings_lists },
-        documents_fields: DocumentsFields { documents_fields },
-        documents_fields_counts: DocumentsFieldsCounts {
-            documents_fields_counts,
-        },
-        synonyms: Synonyms { synonyms },
-        docs_words: DocsWords { docs_words },
-        updates: Updates { updates },
-        updates_results: UpdatesResults { updates_results },
-        updates_notifier,
-    }))
-}

meilidb-core/src/store/postings_lists.rs

@@ -1,41 +0,0 @@
-use crate::DocIndex;
-use heed::types::{ByteSlice, CowSlice};
-use heed::Result as ZResult;
-use sdset::{Set, SetBuf};
-use std::borrow::Cow;
-
-#[derive(Copy, Clone)]
-pub struct PostingsLists {
-    pub(crate) postings_lists: heed::Database<ByteSlice, CowSlice<DocIndex>>,
-}
-
-impl PostingsLists {
-    pub fn put_postings_list(
-        self,
-        writer: &mut heed::RwTxn,
-        word: &[u8],
-        words_indexes: &Set<DocIndex>,
-    ) -> ZResult<()> {
-        self.postings_lists.put(writer, word, words_indexes)
-    }
-
-    pub fn del_postings_list(self, writer: &mut heed::RwTxn, word: &[u8]) -> ZResult<bool> {
-        self.postings_lists.delete(writer, word)
-    }
-
-    pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> {
-        self.postings_lists.clear(writer)
-    }
-
-    pub fn postings_list<'txn>(
-        self,
-        reader: &'txn heed::RoTxn,
-        word: &[u8],
-    ) -> ZResult<Option<Cow<'txn, Set<DocIndex>>>> {
-        match self.postings_lists.get(reader, word)? {
-            Some(Cow::Borrowed(slice)) => Ok(Some(Cow::Borrowed(Set::new_unchecked(slice)))),
-            Some(Cow::Owned(vec)) => Ok(Some(Cow::Owned(SetBuf::new_unchecked(vec)))),
-            None => Ok(None),
-        }
-    }
-}

meilidb-core/src/store/synonyms.rs

@@ -1,36 +0,0 @@
-use heed::types::ByteSlice;
-use heed::Result as ZResult;
-use std::sync::Arc;
-
-#[derive(Copy, Clone)]
-pub struct Synonyms {
-    pub(crate) synonyms: heed::Database<ByteSlice, ByteSlice>,
-}
-
-impl Synonyms {
-    pub fn put_synonyms(
-        self,
-        writer: &mut heed::RwTxn,
-        word: &[u8],
-        synonyms: &fst::Set,
-    ) -> ZResult<()> {
-        let bytes = synonyms.as_fst().as_bytes();
-        self.synonyms.put(writer, word, bytes)
-    }
-
-    pub fn del_synonyms(self, writer: &mut heed::RwTxn, word: &[u8]) -> ZResult<bool> {
-        self.synonyms.delete(writer, word)
-    }
-
-    pub fn synonyms(self, reader: &heed::RoTxn, word: &[u8]) -> ZResult<Option<fst::Set>> {
-        match self.synonyms.get(reader, word)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::from(bytes);
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
-        }
-    }
-}

meilidb-core/src/update/clear_all.rs

@@ -1,33 +0,0 @@
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult, RankedMap};
-
-pub fn apply_clear_all(
-    writer: &mut heed::RwTxn,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-) -> MResult<()> {
-    main_store.put_words_fst(writer, &fst::Set::default())?;
-    main_store.put_ranked_map(writer, &RankedMap::default())?;
-    main_store.put_number_of_documents(writer, |_| 0)?;
-    documents_fields_store.clear(writer)?;
-    documents_fields_counts_store.clear(writer)?;
-    postings_lists_store.clear(writer)?;
-    docs_words_store.clear(writer)?;
-
-    Ok(())
-}
-
-pub fn push_clear_all(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-    let update = Update::ClearAll;
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}

meilidb-core/src/update/documents_addition.rs

@@ -1,290 +0,0 @@
-use std::collections::HashMap;
-
-use fst::{set::OpBuilder, SetBuilder};
-use sdset::{duo::Union, SetOperation};
-use serde::Serialize;
-
-use crate::raw_indexer::RawIndexer;
-use crate::serde::{extract_document_id, serialize_value, Serializer};
-use crate::store;
-use crate::update::{apply_documents_deletion, next_update_id, Update};
-use crate::{Error, MResult, RankedMap};
-
-pub struct DocumentsAddition<D> {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: crossbeam_channel::Sender<()>,
-    documents: Vec<D>,
-}
-
-impl<D> DocumentsAddition<D> {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: crossbeam_channel::Sender<()>,
-    ) -> DocumentsAddition<D> {
-        DocumentsAddition {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            documents: Vec::new(),
-        }
-    }
-
-    pub fn update_document(&mut self, document: D) {
-        self.documents.push(document);
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64>
-    where
-        D: serde::Serialize,
-    {
-        let _ = self.updates_notifier.send(());
-        let update_id = push_documents_addition(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.documents,
-        )?;
-        Ok(update_id)
-    }
-}
-
-impl<D> Extend<D> for DocumentsAddition<D> {
-    fn extend<T: IntoIterator<Item = D>>(&mut self, iter: T) {
-        self.documents.extend(iter)
-    }
-}
-
-pub fn push_documents_addition<D: serde::Serialize>(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    addition: Vec<D>,
-) -> MResult<u64> {
-    let mut values = Vec::with_capacity(addition.len());
-    for add in addition {
-        let vec = serde_json::to_vec(&add)?;
-        let add = serde_json::from_slice(&vec)?;
-        values.push(add);
-    }
-
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::DocumentsAddition(values);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_documents_addition<'a, 'b>(
-    writer: &'a mut heed::RwTxn<'b>,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-    addition: Vec<serde_json::Value>,
-) -> MResult<()> {
-    let mut documents_additions = HashMap::new();
-
-    let schema = match main_store.schema(writer)? {
-        Some(schema) => schema,
-        None => return Err(Error::SchemaMissing),
-    };
-
-    let identifier = schema.identifier_name();
-
-    // 1. store documents ids for future deletion
-    for document in addition {
-        let document_id = match extract_document_id(identifier, &document)? {
-            Some(id) => id,
-            None => return Err(Error::MissingDocumentId),
-        };
-
-        documents_additions.insert(document_id, document);
-    }
-
-    // 2. remove the documents posting lists
-    let number_of_inserted_documents = documents_additions.len();
-    let documents_ids = documents_additions.iter().map(|(id, _)| *id).collect();
-    apply_documents_deletion(
-        writer,
-        main_store,
-        documents_fields_store,
-        documents_fields_counts_store,
-        postings_lists_store,
-        docs_words_store,
-        documents_ids,
-    )?;
-
-    let mut ranked_map = match main_store.ranked_map(writer)? {
-        Some(ranked_map) => ranked_map,
-        None => RankedMap::default(),
-    };
-
-    let stop_words = match main_store.stop_words_fst(writer)? {
-        Some(stop_words) => stop_words,
-        None => fst::Set::default(),
-    };
-
-    // 3. index the documents fields in the stores
-    let mut indexer = RawIndexer::new(stop_words);
-
-    for (document_id, document) in documents_additions {
-        let serializer = Serializer {
-            txn: writer,
-            schema: &schema,
-            document_store: documents_fields_store,
-            document_fields_counts: documents_fields_counts_store,
-            indexer: &mut indexer,
-            ranked_map: &mut ranked_map,
-            document_id,
-        };
-
-        document.serialize(serializer)?;
-    }
-
-    write_documents_addition_index(
-        writer,
-        main_store,
-        postings_lists_store,
-        docs_words_store,
-        &ranked_map,
-        number_of_inserted_documents,
-        indexer,
-    )
-}
-
-pub fn reindex_all_documents(
-    writer: &mut heed::RwTxn,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-) -> MResult<()> {
-    let schema = match main_store.schema(writer)? {
-        Some(schema) => schema,
-        None => return Err(Error::SchemaMissing),
-    };
-
-    let mut ranked_map = RankedMap::default();
-
-    // 1. retrieve all documents ids
-    let mut documents_ids_to_reindex = Vec::new();
-    for result in documents_fields_counts_store.documents_ids(writer)? {
-        let document_id = result?;
-        documents_ids_to_reindex.push(document_id);
-    }
-
-    // 2. remove the documents posting lists
-    main_store.put_words_fst(writer, &fst::Set::default())?;
-    main_store.put_ranked_map(writer, &ranked_map)?;
-    main_store.put_number_of_documents(writer, |_| 0)?;
-    postings_lists_store.clear(writer)?;
-    docs_words_store.clear(writer)?;
-
-    // 3. re-index chunks of documents (otherwise we make the borrow checker unhappy)
-    for documents_ids in documents_ids_to_reindex.chunks(100) {
-        let stop_words = match main_store.stop_words_fst(writer)? {
-            Some(stop_words) => stop_words,
-            None => fst::Set::default(),
-        };
-
-        let number_of_inserted_documents = documents_ids.len();
-        let mut indexer = RawIndexer::new(stop_words);
-        let mut ram_store = HashMap::new();
-
-        for document_id in documents_ids {
-            for result in documents_fields_store.document_fields(writer, *document_id)? {
-                let (attr, bytes) = result?;
-                let value: serde_json::Value = serde_json::from_slice(bytes)?;
-                ram_store.insert((document_id, attr), value);
-            }
-
-            for ((docid, attr), value) in ram_store.drain() {
-                serialize_value(
-                    writer,
-                    attr,
-                    schema.props(attr),
-                    *docid,
-                    documents_fields_store,
-                    documents_fields_counts_store,
-                    &mut indexer,
-                    &mut ranked_map,
-                    &value,
-                )?;
-            }
-        }
-
-        // 4. write the new index in the main store
-        write_documents_addition_index(
-            writer,
-            main_store,
-            postings_lists_store,
-            docs_words_store,
-            &ranked_map,
-            number_of_inserted_documents,
-            indexer,
-        )?;
-    }
-
-    Ok(())
-}
-
-pub fn write_documents_addition_index(
-    writer: &mut heed::RwTxn,
-    main_store: store::Main,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-    ranked_map: &RankedMap,
-    number_of_inserted_documents: usize,
-    indexer: RawIndexer,
-) -> MResult<()> {
-    let indexed = indexer.build();
-    let mut delta_words_builder = SetBuilder::memory();
-
-    for (word, delta_set) in indexed.words_doc_indexes {
-        delta_words_builder.insert(&word).unwrap();
-
-        let set = match postings_lists_store.postings_list(writer, &word)? {
-            Some(set) => Union::new(&set, &delta_set).into_set_buf(),
-            None => delta_set,
-        };
-
-        postings_lists_store.put_postings_list(writer, &word, &set)?;
-    }
-
-    for (id, words) in indexed.docs_words {
-        docs_words_store.put_doc_words(writer, id, &words)?;
-    }
-
-    let delta_words = delta_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    let words = match main_store.words_fst(writer)? {
-        Some(words) => {
-            let op = OpBuilder::new()
-                .add(words.stream())
-                .add(delta_words.stream())
-                .r#union();
-
-            let mut words_builder = SetBuilder::memory();
-            words_builder.extend_stream(op).unwrap();
-            words_builder
-                .into_inner()
-                .and_then(fst::Set::from_bytes)
-                .unwrap()
-        }
-        None => delta_words,
-    };
-
-    main_store.put_words_fst(writer, &words)?;
-    main_store.put_ranked_map(writer, ranked_map)?;
-    main_store.put_number_of_documents(writer, |old| old + number_of_inserted_documents as u64)?;
-
-    Ok(())
-}

meilidb-core/src/update/documents_deletion.rs

@@ -1,191 +0,0 @@
-use std::collections::{BTreeSet, HashMap, HashSet};
-
-use fst::{SetBuilder, Streamer};
-use meilidb_schema::Schema;
-use sdset::{duo::DifferenceByKey, SetBuf, SetOperation};
-
-use crate::serde::extract_document_id;
-use crate::store;
-use crate::update::{next_update_id, Update};
-use crate::{DocumentId, Error, MResult, RankedMap};
-
-pub struct DocumentsDeletion {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: crossbeam_channel::Sender<()>,
-    documents: Vec<DocumentId>,
-}
-
-impl DocumentsDeletion {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: crossbeam_channel::Sender<()>,
-    ) -> DocumentsDeletion {
-        DocumentsDeletion {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            documents: Vec::new(),
-        }
-    }
-
-    pub fn delete_document_by_id(&mut self, document_id: DocumentId) {
-        self.documents.push(document_id);
-    }
-
-    pub fn delete_document<D>(&mut self, schema: &Schema, document: D) -> MResult<()>
-    where
-        D: serde::Serialize,
-    {
-        let identifier = schema.identifier_name();
-        let document_id = match extract_document_id(identifier, &document)? {
-            Some(id) => id,
-            None => return Err(Error::MissingDocumentId),
-        };
-
-        self.delete_document_by_id(document_id);
-
-        Ok(())
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64> {
-        let _ = self.updates_notifier.send(());
-        let update_id = push_documents_deletion(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.documents,
-        )?;
-        Ok(update_id)
-    }
-}
-
-impl Extend<DocumentId> for DocumentsDeletion {
-    fn extend<T: IntoIterator<Item = DocumentId>>(&mut self, iter: T) {
-        self.documents.extend(iter)
-    }
-}
-
-pub fn push_documents_deletion(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    deletion: Vec<DocumentId>,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::DocumentsDeletion(deletion);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_documents_deletion(
-    writer: &mut heed::RwTxn,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-    deletion: Vec<DocumentId>,
-) -> MResult<()> {
-    let idset = SetBuf::from_dirty(deletion);
-
-    let schema = match main_store.schema(writer)? {
-        Some(schema) => schema,
-        None => return Err(Error::SchemaMissing),
-    };
-
-    let mut ranked_map = match main_store.ranked_map(writer)? {
-        Some(ranked_map) => ranked_map,
-        None => RankedMap::default(),
-    };
-
-    // collect the ranked attributes according to the schema
-    let ranked_attrs: Vec<_> = schema
-        .iter()
-        .filter_map(
-            |(_, attr, prop)| {
-                if prop.is_ranked() {
-                    Some(attr)
-                } else {
-                    None
-                }
-            },
-        )
-        .collect();
-
-    let mut words_document_ids = HashMap::new();
-    for id in idset {
-        // remove all the ranked attributes from the ranked_map
-        for ranked_attr in &ranked_attrs {
-            ranked_map.remove(id, *ranked_attr);
-        }
-
-        if let Some(words) = docs_words_store.doc_words(writer, id)? {
-            let mut stream = words.stream();
-            while let Some(word) = stream.next() {
-                let word = word.to_vec();
-                words_document_ids
-                    .entry(word)
-                    .or_insert_with(Vec::new)
-                    .push(id);
-            }
-        }
-    }
-
-    let mut deleted_documents = HashSet::new();
-    let mut removed_words = BTreeSet::new();
-    for (word, document_ids) in words_document_ids {
-        let document_ids = SetBuf::from_dirty(document_ids);
-
-        if let Some(doc_indexes) = postings_lists_store.postings_list(writer, &word)? {
-            let op = DifferenceByKey::new(&doc_indexes, &document_ids, |d| d.document_id, |id| *id);
-            let doc_indexes = op.into_set_buf();
-
-            if !doc_indexes.is_empty() {
-                postings_lists_store.put_postings_list(writer, &word, &doc_indexes)?;
-            } else {
-                postings_lists_store.del_postings_list(writer, &word)?;
-                removed_words.insert(word);
-            }
-        }
-
-        for id in document_ids {
-            documents_fields_counts_store.del_all_document_fields_counts(writer, id)?;
-            if documents_fields_store.del_all_document_fields(writer, id)? != 0 {
-                deleted_documents.insert(id);
-            }
-        }
-    }
-
-    let deleted_documents_len = deleted_documents.len() as u64;
-    for id in deleted_documents {
-        docs_words_store.del_doc_words(writer, id)?;
-    }
-
-    let removed_words = fst::Set::from_iter(removed_words).unwrap();
-    let words = match main_store.words_fst(writer)? {
-        Some(words_set) => {
-            let op = fst::set::OpBuilder::new()
-                .add(words_set.stream())
-                .add(removed_words.stream())
-                .difference();
-
-            let mut words_builder = SetBuilder::memory();
-            words_builder.extend_stream(op).unwrap();
-            words_builder
-                .into_inner()
-                .and_then(fst::Set::from_bytes)
-                .unwrap()
-        }
-        None => fst::Set::default(),
-    };
-
-    main_store.put_words_fst(writer, &words)?;
-    main_store.put_ranked_map(writer, &ranked_map)?;
-    main_store.put_number_of_documents(writer, |old| old - deleted_documents_len)?;
-
-    Ok(())
-}

meilidb-core/src/update/mod.rs

@@ -1,309 +0,0 @@
-mod clear_all;
-mod customs_update;
-mod documents_addition;
-mod documents_deletion;
-mod schema_update;
-mod stop_words_addition;
-mod stop_words_deletion;
-mod synonyms_addition;
-mod synonyms_deletion;
-
-pub use self::clear_all::{apply_clear_all, push_clear_all};
-pub use self::customs_update::{apply_customs_update, push_customs_update};
-pub use self::documents_addition::{apply_documents_addition, DocumentsAddition};
-pub use self::documents_deletion::{apply_documents_deletion, DocumentsDeletion};
-pub use self::schema_update::{apply_schema_update, push_schema_update};
-pub use self::stop_words_addition::{apply_stop_words_addition, StopWordsAddition};
-pub use self::stop_words_deletion::{apply_stop_words_deletion, StopWordsDeletion};
-pub use self::synonyms_addition::{apply_synonyms_addition, SynonymsAddition};
-pub use self::synonyms_deletion::{apply_synonyms_deletion, SynonymsDeletion};
-
-use std::cmp;
-use std::collections::{BTreeMap, BTreeSet};
-use std::time::{Duration, Instant};
-
-use heed::Result as ZResult;
-use log::debug;
-use serde::{Deserialize, Serialize};
-
-use crate::{store, DocumentId, MResult};
-use meilidb_schema::Schema;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum Update {
-    ClearAll,
-    Schema(Schema),
-    Customs(Vec<u8>),
-    DocumentsAddition(Vec<serde_json::Value>),
-    DocumentsDeletion(Vec<DocumentId>),
-    SynonymsAddition(BTreeMap<String, Vec<String>>),
-    SynonymsDeletion(BTreeMap<String, Option<Vec<String>>>),
-    StopWordsAddition(BTreeSet<String>),
-    StopWordsDeletion(BTreeSet<String>),
-}
-
-impl Update {
-    pub fn update_type(&self) -> UpdateType {
-        match self {
-            Update::ClearAll => UpdateType::ClearAll,
-            Update::Schema(schema) => UpdateType::Schema {
-                schema: schema.clone(),
-            },
-            Update::Customs(_) => UpdateType::Customs,
-            Update::DocumentsAddition(addition) => UpdateType::DocumentsAddition {
-                number: addition.len(),
-            },
-            Update::DocumentsDeletion(deletion) => UpdateType::DocumentsDeletion {
-                number: deletion.len(),
-            },
-            Update::SynonymsAddition(addition) => UpdateType::SynonymsAddition {
-                number: addition.len(),
-            },
-            Update::SynonymsDeletion(deletion) => UpdateType::SynonymsDeletion {
-                number: deletion.len(),
-            },
-            Update::StopWordsAddition(addition) => UpdateType::StopWordsAddition {
-                number: addition.len(),
-            },
-            Update::StopWordsDeletion(deletion) => UpdateType::StopWordsDeletion {
-                number: deletion.len(),
-            },
-        }
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum UpdateType {
-    ClearAll,
-    Schema { schema: Schema },
-    Customs,
-    DocumentsAddition { number: usize },
-    DocumentsDeletion { number: usize },
-    SynonymsAddition { number: usize },
-    SynonymsDeletion { number: usize },
-    StopWordsAddition { number: usize },
-    StopWordsDeletion { number: usize },
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct DetailedDuration {
-    pub main: Duration,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ProcessedUpdateResult {
-    pub update_id: u64,
-    pub update_type: UpdateType,
-    pub result: Result<(), String>,
-    pub detailed_duration: DetailedDuration,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct EnqueuedUpdateResult {
-    pub update_id: u64,
-    pub update_type: UpdateType,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum UpdateStatus {
-    Enqueued(EnqueuedUpdateResult),
-    Processed(ProcessedUpdateResult),
-    Unknown,
-}
-
-pub fn update_status(
-    reader: &heed::RoTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    update_id: u64,
-) -> MResult<UpdateStatus> {
-    match updates_results_store.update_result(reader, update_id)? {
-        Some(result) => Ok(UpdateStatus::Processed(result)),
-        None => {
-            if let Some(update) = updates_store.get(reader, update_id)? {
-                Ok(UpdateStatus::Enqueued(EnqueuedUpdateResult {
-                    update_id,
-                    update_type: update.update_type(),
-                }))
-            } else {
-                Ok(UpdateStatus::Unknown)
-            }
-        }
-    }
-}
-
-pub fn next_update_id(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-) -> ZResult<u64> {
-    let last_update_id = updates_store.last_update_id(writer)?;
-    let last_update_id = last_update_id.map(|(n, _)| n);
-
-    let last_update_results_id = updates_results_store.last_update_id(writer)?;
-    let last_update_results_id = last_update_results_id.map(|(n, _)| n);
-
-    let max_update_id = cmp::max(last_update_id, last_update_results_id);
-    let new_update_id = max_update_id.map_or(0, |n| n + 1);
-
-    Ok(new_update_id)
-}
-
-pub fn update_task<'a, 'b>(
-    writer: &'a mut heed::RwTxn<'b>,
-    index: store::Index,
-    update_id: u64,
-    update: Update,
-) -> MResult<ProcessedUpdateResult> {
-    debug!("Processing update number {}", update_id);
-
-    let (update_type, result, duration) = match update {
-        Update::ClearAll => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::ClearAll;
-            let result = apply_clear_all(
-                writer,
-                index.main,
-                index.documents_fields,
-                index.documents_fields_counts,
-                index.postings_lists,
-                index.docs_words,
-            );
-
-            (update_type, result, start.elapsed())
-        }
-        Update::Schema(schema) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::Schema {
-                schema: schema.clone(),
-            };
-            let result = apply_schema_update(
-                writer,
-                &schema,
-                index.main,
-                index.documents_fields,
-                index.documents_fields_counts,
-                index.postings_lists,
-                index.docs_words,
-            );
-
-            (update_type, result, start.elapsed())
-        }
-        Update::Customs(customs) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::Customs;
-            let result = apply_customs_update(writer, index.main, &customs).map_err(Into::into);
-
-            (update_type, result, start.elapsed())
-        }
-        Update::DocumentsAddition(documents) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::DocumentsAddition {
-                number: documents.len(),
-            };
-
-            let result = apply_documents_addition(
-                writer,
-                index.main,
-                index.documents_fields,
-                index.documents_fields_counts,
-                index.postings_lists,
-                index.docs_words,
-                documents,
-            );
-
-            (update_type, result, start.elapsed())
-        }
-        Update::DocumentsDeletion(documents) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::DocumentsDeletion {
-                number: documents.len(),
-            };
-
-            let result = apply_documents_deletion(
-                writer,
-                index.main,
-                index.documents_fields,
-                index.documents_fields_counts,
-                index.postings_lists,
-                index.docs_words,
-                documents,
-            );
-
-            (update_type, result, start.elapsed())
-        }
-        Update::SynonymsAddition(synonyms) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::SynonymsAddition {
-                number: synonyms.len(),
-            };
-
-            let result = apply_synonyms_addition(writer, index.main, index.synonyms, synonyms);
-
-            (update_type, result, start.elapsed())
-        }
-        Update::SynonymsDeletion(synonyms) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::SynonymsDeletion {
-                number: synonyms.len(),
-            };
-
-            let result = apply_synonyms_deletion(writer, index.main, index.synonyms, synonyms);
-
-            (update_type, result, start.elapsed())
-        }
-        Update::StopWordsAddition(stop_words) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::StopWordsAddition {
-                number: stop_words.len(),
-            };
-
-            let result =
-                apply_stop_words_addition(writer, index.main, index.postings_lists, stop_words);
-
-            (update_type, result, start.elapsed())
-        }
-        Update::StopWordsDeletion(stop_words) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::StopWordsDeletion {
-                number: stop_words.len(),
-            };
-
-            let result = apply_stop_words_deletion(
-                writer,
-                index.main,
-                index.documents_fields,
-                index.documents_fields_counts,
-                index.postings_lists,
-                index.docs_words,
-                stop_words,
-            );
-
-            (update_type, result, start.elapsed())
-        }
-    };
-
-    debug!(
-        "Processed update number {} {:?} {:?}",
-        update_id, update_type, result
-    );
-
-    let detailed_duration = DetailedDuration { main: duration };
-    let status = ProcessedUpdateResult {
-        update_id,
-        update_type,
-        result: result.map_err(|e| e.to_string()),
-        detailed_duration,
-    };
-
-    Ok(status)
-}

meilidb-core/src/update/schema_update.rs

@@ -1,70 +0,0 @@
-use meilidb_schema::{Diff, Schema};
-
-use crate::update::documents_addition::reindex_all_documents;
-use crate::update::{next_update_id, Update};
-use crate::{error::UnsupportedOperation, store, MResult};
-
-pub fn apply_schema_update(
-    writer: &mut heed::RwTxn,
-    new_schema: &Schema,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-) -> MResult<()> {
-    use UnsupportedOperation::{
-        CannotIntroduceNewSchemaAttribute, CannotRemoveSchemaAttribute,
-        CannotReorderSchemaAttribute, CannotUpdateSchemaIdentifier,
-    };
-
-    let mut need_full_reindexing = false;
-
-    if let Some(old_schema) = main_store.schema(writer)? {
-        for diff in meilidb_schema::diff(&old_schema, new_schema) {
-            match diff {
-                Diff::IdentChange { .. } => return Err(CannotUpdateSchemaIdentifier.into()),
-                Diff::AttrMove { .. } => return Err(CannotReorderSchemaAttribute.into()),
-                Diff::AttrPropsChange { old, new, .. } => {
-                    if new.indexed != old.indexed {
-                        need_full_reindexing = true;
-                    }
-                    if new.ranked != old.ranked {
-                        need_full_reindexing = true;
-                    }
-                }
-                Diff::NewAttr { .. } => return Err(CannotIntroduceNewSchemaAttribute.into()),
-                Diff::RemovedAttr { .. } => return Err(CannotRemoveSchemaAttribute.into()),
-            }
-        }
-    }
-
-    main_store.put_schema(writer, new_schema)?;
-
-    if need_full_reindexing {
-        reindex_all_documents(
-            writer,
-            main_store,
-            documents_fields_store,
-            documents_fields_counts_store,
-            postings_lists_store,
-            docs_words_store,
-        )?
-    }
-
-    Ok(())
-}
-
-pub fn push_schema_update(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    schema: Schema,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::Schema(schema);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}

meilidb-core/src/update/stop_words_addition.rs

@@ -1,116 +0,0 @@
-use std::collections::BTreeSet;
-
-use fst::{set::OpBuilder, SetBuilder};
-
-use crate::automaton::normalize_str;
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult};
-
-pub struct StopWordsAddition {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: crossbeam_channel::Sender<()>,
-    stop_words: BTreeSet<String>,
-}
-
-impl StopWordsAddition {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: crossbeam_channel::Sender<()>,
-    ) -> StopWordsAddition {
-        StopWordsAddition {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            stop_words: BTreeSet::new(),
-        }
-    }
-
-    pub fn add_stop_word<S: AsRef<str>>(&mut self, stop_word: S) {
-        let stop_word = normalize_str(stop_word.as_ref());
-        self.stop_words.insert(stop_word);
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64> {
-        let _ = self.updates_notifier.send(());
-        let update_id = push_stop_words_addition(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.stop_words,
-        )?;
-        Ok(update_id)
-    }
-}
-
-pub fn push_stop_words_addition(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    addition: BTreeSet<String>,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::StopWordsAddition(addition);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_stop_words_addition(
-    writer: &mut heed::RwTxn,
-    main_store: store::Main,
-    postings_lists_store: store::PostingsLists,
-    addition: BTreeSet<String>,
-) -> MResult<()> {
-    let mut stop_words_builder = SetBuilder::memory();
-
-    for word in addition {
-        stop_words_builder.insert(&word).unwrap();
-        // we remove every posting list associated to a new stop word
-        postings_lists_store.del_postings_list(writer, word.as_bytes())?;
-    }
-
-    // create the new delta stop words fst
-    let delta_stop_words = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    // we also need to remove all the stop words from the main fst
-    if let Some(word_fst) = main_store.words_fst(writer)? {
-        let op = OpBuilder::new()
-            .add(&word_fst)
-            .add(&delta_stop_words)
-            .difference();
-
-        let mut word_fst_builder = SetBuilder::memory();
-        word_fst_builder.extend_stream(op).unwrap();
-        let word_fst = word_fst_builder
-            .into_inner()
-            .and_then(fst::Set::from_bytes)
-            .unwrap();
-
-        main_store.put_words_fst(writer, &word_fst)?;
-    }
-
-    // now we add all of these stop words from the main store
-    let stop_words_fst = main_store.stop_words_fst(writer)?.unwrap_or_default();
-
-    let op = OpBuilder::new()
-        .add(&stop_words_fst)
-        .add(&delta_stop_words)
-        .r#union();
-
-    let mut stop_words_builder = SetBuilder::memory();
-    stop_words_builder.extend_stream(op).unwrap();
-    let stop_words_fst = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    main_store.put_stop_words_fst(writer, &stop_words_fst)?;
-
-    Ok(())
-}

meilidb-core/src/update/stop_words_deletion.rs

@@ -1,112 +0,0 @@
-use std::collections::BTreeSet;
-
-use fst::{set::OpBuilder, SetBuilder};
-
-use crate::automaton::normalize_str;
-use crate::update::documents_addition::reindex_all_documents;
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult};
-
-pub struct StopWordsDeletion {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: crossbeam_channel::Sender<()>,
-    stop_words: BTreeSet<String>,
-}
-
-impl StopWordsDeletion {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: crossbeam_channel::Sender<()>,
-    ) -> StopWordsDeletion {
-        StopWordsDeletion {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            stop_words: BTreeSet::new(),
-        }
-    }
-
-    pub fn delete_stop_word<S: AsRef<str>>(&mut self, stop_word: S) {
-        let stop_word = normalize_str(stop_word.as_ref());
-        self.stop_words.insert(stop_word);
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64> {
-        let _ = self.updates_notifier.send(());
-        let update_id = push_stop_words_deletion(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.stop_words,
-        )?;
-        Ok(update_id)
-    }
-}
-
-pub fn push_stop_words_deletion(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    deletion: BTreeSet<String>,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::StopWordsDeletion(deletion);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_stop_words_deletion(
-    writer: &mut heed::RwTxn,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-    deletion: BTreeSet<String>,
-) -> MResult<()> {
-    let mut stop_words_builder = SetBuilder::memory();
-
-    for word in deletion {
-        stop_words_builder.insert(&word).unwrap();
-    }
-
-    // create the new delta stop words fst
-    let delta_stop_words = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    // now we delete all of these stop words from the main store
-    let stop_words_fst = main_store.stop_words_fst(writer)?.unwrap_or_default();
-
-    let op = OpBuilder::new()
-        .add(&stop_words_fst)
-        .add(&delta_stop_words)
-        .difference();
-
-    let mut stop_words_builder = SetBuilder::memory();
-    stop_words_builder.extend_stream(op).unwrap();
-    let stop_words_fst = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    main_store.put_stop_words_fst(writer, &stop_words_fst)?;
-
-    // now that we have setup the stop words
-    // lets reindex everything...
-    reindex_all_documents(
-        writer,
-        main_store,
-        documents_fields_store,
-        documents_fields_counts_store,
-        postings_lists_store,
-        docs_words_store,
-    )?;
-
-    Ok(())
-}

meilidb-core/src/update/synonyms_addition.rs

@@ -1,118 +0,0 @@
-use std::collections::BTreeMap;
-
-use fst::{set::OpBuilder, SetBuilder};
-use sdset::SetBuf;
-
-use crate::automaton::normalize_str;
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult};
-
-pub struct SynonymsAddition {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: crossbeam_channel::Sender<()>,
-    synonyms: BTreeMap<String, Vec<String>>,
-}
-
-impl SynonymsAddition {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: crossbeam_channel::Sender<()>,
-    ) -> SynonymsAddition {
-        SynonymsAddition {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            synonyms: BTreeMap::new(),
-        }
-    }
-
-    pub fn add_synonym<S, T, I>(&mut self, synonym: S, alternatives: I)
-    where
-        S: AsRef<str>,
-        T: AsRef<str>,
-        I: IntoIterator<Item = T>,
-    {
-        let synonym = normalize_str(synonym.as_ref());
-        let alternatives = alternatives.into_iter().map(|s| s.as_ref().to_lowercase());
-        self.synonyms
-            .entry(synonym)
-            .or_insert_with(Vec::new)
-            .extend(alternatives);
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64> {
-        let _ = self.updates_notifier.send(());
-        let update_id = push_synonyms_addition(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.synonyms,
-        )?;
-        Ok(update_id)
-    }
-}
-
-pub fn push_synonyms_addition(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    addition: BTreeMap<String, Vec<String>>,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::SynonymsAddition(addition);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_synonyms_addition(
-    writer: &mut heed::RwTxn,
-    main_store: store::Main,
-    synonyms_store: store::Synonyms,
-    addition: BTreeMap<String, Vec<String>>,
-) -> MResult<()> {
-    let mut synonyms_builder = SetBuilder::memory();
-
-    for (word, alternatives) in addition {
-        synonyms_builder.insert(&word).unwrap();
-
-        let alternatives = {
-            let alternatives = SetBuf::from_dirty(alternatives);
-            let mut alternatives_builder = SetBuilder::memory();
-            alternatives_builder.extend_iter(alternatives).unwrap();
-            let bytes = alternatives_builder.into_inner().unwrap();
-            fst::Set::from_bytes(bytes).unwrap()
-        };
-
-        synonyms_store.put_synonyms(writer, word.as_bytes(), &alternatives)?;
-    }
-
-    let delta_synonyms = synonyms_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    let synonyms = match main_store.synonyms_fst(writer)? {
-        Some(synonyms) => {
-            let op = OpBuilder::new()
-                .add(synonyms.stream())
-                .add(delta_synonyms.stream())
-                .r#union();
-
-            let mut synonyms_builder = SetBuilder::memory();
-            synonyms_builder.extend_stream(op).unwrap();
-            synonyms_builder
-                .into_inner()
-                .and_then(fst::Set::from_bytes)
-                .unwrap()
-        }
-        None => delta_synonyms,
-    };
-
-    main_store.put_synonyms_fst(writer, &synonyms)?;
-
-    Ok(())
-}

meilidb-core/src/update/synonyms_deletion.rs

@@ -1,156 +0,0 @@
-use std::collections::BTreeMap;
-use std::iter::FromIterator;
-
-use fst::{set::OpBuilder, SetBuilder};
-use sdset::SetBuf;
-
-use crate::automaton::normalize_str;
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult};
-
-pub struct SynonymsDeletion {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: crossbeam_channel::Sender<()>,
-    synonyms: BTreeMap<String, Option<Vec<String>>>,
-}
-
-impl SynonymsDeletion {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: crossbeam_channel::Sender<()>,
-    ) -> SynonymsDeletion {
-        SynonymsDeletion {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            synonyms: BTreeMap::new(),
-        }
-    }
-
-    pub fn delete_all_alternatives_of<S: AsRef<str>>(&mut self, synonym: S) {
-        let synonym = normalize_str(synonym.as_ref());
-        self.synonyms.insert(synonym, None);
-    }
-
-    pub fn delete_specific_alternatives_of<S, T, I>(&mut self, synonym: S, alternatives: I)
-    where
-        S: AsRef<str>,
-        T: AsRef<str>,
-        I: Iterator<Item = T>,
-    {
-        let synonym = normalize_str(synonym.as_ref());
-        let value = self.synonyms.entry(synonym).or_insert(None);
-        let alternatives = alternatives.map(|s| s.as_ref().to_lowercase());
-        match value {
-            Some(v) => v.extend(alternatives),
-            None => *value = Some(Vec::from_iter(alternatives)),
-        }
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64> {
-        let _ = self.updates_notifier.send(());
-        let update_id = push_synonyms_deletion(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.synonyms,
-        )?;
-        Ok(update_id)
-    }
-}
-
-pub fn push_synonyms_deletion(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    deletion: BTreeMap<String, Option<Vec<String>>>,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::SynonymsDeletion(deletion);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_synonyms_deletion(
-    writer: &mut heed::RwTxn,
-    main_store: store::Main,
-    synonyms_store: store::Synonyms,
-    deletion: BTreeMap<String, Option<Vec<String>>>,
-) -> MResult<()> {
-    let mut delete_whole_synonym_builder = SetBuilder::memory();
-
-    for (synonym, alternatives) in deletion {
-        match alternatives {
-            Some(alternatives) => {
-                let prev_alternatives = synonyms_store.synonyms(writer, synonym.as_bytes())?;
-                let prev_alternatives = match prev_alternatives {
-                    Some(alternatives) => alternatives,
-                    None => continue,
-                };
-
-                let delta_alternatives = {
-                    let alternatives = SetBuf::from_dirty(alternatives);
-                    let mut builder = SetBuilder::memory();
-                    builder.extend_iter(alternatives).unwrap();
-                    builder.into_inner().and_then(fst::Set::from_bytes).unwrap()
-                };
-
-                let op = OpBuilder::new()
-                    .add(prev_alternatives.stream())
-                    .add(delta_alternatives.stream())
-                    .difference();
-
-                let (alternatives, empty_alternatives) = {
-                    let mut builder = SetBuilder::memory();
-                    let len = builder.get_ref().len();
-                    builder.extend_stream(op).unwrap();
-                    let is_empty = len == builder.get_ref().len();
-                    let bytes = builder.into_inner().unwrap();
-                    let alternatives = fst::Set::from_bytes(bytes).unwrap();
-
-                    (alternatives, is_empty)
-                };
-
-                if empty_alternatives {
-                    delete_whole_synonym_builder.insert(synonym.as_bytes())?;
-                } else {
-                    synonyms_store.put_synonyms(writer, synonym.as_bytes(), &alternatives)?;
-                }
-            }
-            None => {
-                delete_whole_synonym_builder.insert(&synonym).unwrap();
-                synonyms_store.del_synonyms(writer, synonym.as_bytes())?;
-            }
-        }
-    }
-
-    let delta_synonyms = delete_whole_synonym_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    let synonyms = match main_store.synonyms_fst(writer)? {
-        Some(synonyms) => {
-            let op = OpBuilder::new()
-                .add(synonyms.stream())
-                .add(delta_synonyms.stream())
-                .difference();
-
-            let mut synonyms_builder = SetBuilder::memory();
-            synonyms_builder.extend_stream(op).unwrap();
-            synonyms_builder
-                .into_inner()
-                .and_then(fst::Set::from_bytes)
-                .unwrap()
-        }
-        None => fst::Set::default(),
-    };
-
-    main_store.put_synonyms_fst(writer, &synonyms)?;
-
-    Ok(())
-}

meilidb-http/Cargo.toml

@@ -1,55 +0,0 @@
-[package]
-name = "meilidb-http"
-version = "0.1.0"
-authors = [
-    "Quentin de Quelen <quentin@dequelen.me>",
-    "Clément Renault <clement@meilisearch.com>",
-]
-edition = "2018"
-
-[dependencies]
-bincode = "1.2.0"
-chrono = { version = "0.4.9", features = ["serde"] }
-crossbeam-channel = "0.3.9"
-envconfig = "0.5.1"
-envconfig_derive = "0.5.1"
-heed = "0.5.0"
-http = "0.1.19"
-indexmap = { version = "1.3.0", features = ["serde-1"] }
-jemallocator = "0.3.2"
-log = "0.4.8"
-main_error = "0.1.0"
-meilidb-core = { path = "../meilidb-core", version = "0.6.0" }
-meilidb-schema = { path = "../meilidb-schema", version = "0.6.0" }
-pretty-bytes = "0.2.2"
-rand = "0.7.2"
-rayon = "1.2.0"
-serde = { version = "1.0.101", features = ["derive"] }
-serde_json = { version = "1.0.41", features = ["preserve_order"] }
-structopt = "0.3.3"
-sysinfo = "0.9.5"
-walkdir = "2.2.9"
-
-[dependencies.async-compression]
-default-features = false
-features = ["stream", "gzip", "zlib", "brotli", "zstd"]
-version = "0.1.0-alpha.7"
-
-[dependencies.tide]
-git = "https://github.com/rustasync/tide"
-rev = "e77709370bb24cf776fe6da902467c35131535b1"
-
-[dependencies.tide-log]
-git = "https://github.com/rustasync/tide"
-rev = "e77709370bb24cf776fe6da902467c35131535b1"
-
-[dependencies.tide-slog]
-git = "https://github.com/rustasync/tide"
-rev = "e77709370bb24cf776fe6da902467c35131535b1"
-
-[dependencies.tide-compression]
-git = "https://github.com/rustasync/tide"
-rev = "e77709370bb24cf776fe6da902467c35131535b1"
-
-[build-dependencies]
-vergen = "3.0.4"

meilidb-http/src/data.rs

@@ -1,190 +0,0 @@
-use std::collections::HashMap;
-use std::ops::Deref;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::Arc;
-
-use chrono::{DateTime, Utc};
-use heed::types::{SerdeBincode, Str};
-use log::*;
-use meilidb_core::{Database, MResult};
-use sysinfo::Pid;
-
-use crate::option::Opt;
-use crate::routes::index::index_update_callback;
-
-pub type FreqsMap = HashMap<String, usize>;
-type SerdeFreqsMap = SerdeBincode<FreqsMap>;
-type SerdeDatetime = SerdeBincode<DateTime<Utc>>;
-
-#[derive(Clone)]
-pub struct Data {
-    inner: Arc<DataInner>,
-}
-
-impl Deref for Data {
-    type Target = DataInner;
-
-    fn deref(&self) -> &Self::Target {
-        &self.inner
-    }
-}
-
-#[derive(Clone)]
-pub struct DataInner {
-    pub db: Arc<Database>,
-    pub db_path: String,
-    pub admin_token: Option<String>,
-    pub server_pid: Pid,
-    pub accept_updates: Arc<AtomicBool>,
-}
-
-impl DataInner {
-    pub fn is_indexing(&self, reader: &heed::RoTxn, index: &str) -> MResult<Option<bool>> {
-        match self.db.open_index(&index) {
-            Some(index) => index.current_update_id(&reader).map(|u| Some(u.is_some())),
-            None => Ok(None),
-        }
-    }
-
-    pub fn last_update(
-        &self,
-        reader: &heed::RoTxn,
-        index_name: &str,
-    ) -> MResult<Option<DateTime<Utc>>> {
-        let key = format!("last-update-{}", index_name);
-        match self
-            .db
-            .common_store()
-            .get::<Str, SerdeDatetime>(&reader, &key)?
-        {
-            Some(datetime) => Ok(Some(datetime)),
-            None => Ok(None),
-        }
-    }
-
-    pub fn set_last_update(&self, writer: &mut heed::RwTxn, index_name: &str) -> MResult<()> {
-        let key = format!("last-update-{}", index_name);
-        self.db
-            .common_store()
-            .put::<Str, SerdeDatetime>(writer, &key, &Utc::now())
-            .map_err(Into::into)
-    }
-
-    pub fn last_backup(&self, reader: &heed::RoTxn) -> MResult<Option<DateTime<Utc>>> {
-        match self
-            .db
-            .common_store()
-            .get::<Str, SerdeDatetime>(&reader, "last-backup")?
-        {
-            Some(datetime) => Ok(Some(datetime)),
-            None => Ok(None),
-        }
-    }
-
-    pub fn set_last_backup(&self, writer: &mut heed::RwTxn) -> MResult<()> {
-        self.db
-            .common_store()
-            .put::<Str, SerdeDatetime>(writer, "last-backup", &Utc::now())?;
-
-        Ok(())
-    }
-
-    pub fn fields_frequency(
-        &self,
-        reader: &heed::RoTxn,
-        index_name: &str,
-    ) -> MResult<Option<FreqsMap>> {
-        let key = format!("fields-frequency-{}", index_name);
-        match self
-            .db
-            .common_store()
-            .get::<Str, SerdeFreqsMap>(&reader, &key)?
-        {
-            Some(freqs) => Ok(Some(freqs)),
-            None => Ok(None),
-        }
-    }
-
-    pub fn compute_stats(&self, writer: &mut heed::RwTxn, index_name: &str) -> MResult<()> {
-        let index = match self.db.open_index(&index_name) {
-            Some(index) => index,
-            None => {
-                error!("Impossible to retrieve index {}", index_name);
-                return Ok(());
-            }
-        };
-
-        let schema = match index.main.schema(&writer)? {
-            Some(schema) => schema,
-            None => return Ok(()),
-        };
-
-        let all_documents_fields = index
-            .documents_fields_counts
-            .all_documents_fields_counts(&writer)?;
-
-        // count fields frequencies
-        let mut fields_frequency = HashMap::<_, usize>::new();
-        for result in all_documents_fields {
-            let (_, attr, _) = result?;
-            *fields_frequency.entry(attr).or_default() += 1;
-        }
-
-        // convert attributes to their names
-        let frequency: HashMap<_, _> = fields_frequency
-            .into_iter()
-            .map(|(a, c)| (schema.attribute_name(a).to_owned(), c))
-            .collect();
-
-        let key = format!("fields-frequency-{}", index_name);
-        self.db
-            .common_store()
-            .put::<Str, SerdeFreqsMap>(writer, &key, &frequency)?;
-
-        Ok(())
-    }
-
-    pub fn stop_accept_updates(&self) {
-        self.accept_updates.store(false, Ordering::Relaxed);
-    }
-
-    pub fn accept_updates(&self) -> bool {
-        self.accept_updates.load(Ordering::Relaxed)
-    }
-}
-
-impl Data {
-    pub fn new(opt: Opt) -> Data {
-        let db_path = opt.database_path.clone();
-        let admin_token = opt.admin_token.clone();
-        let server_pid = sysinfo::get_current_pid().unwrap();
-
-        let db = Arc::new(Database::open_or_create(opt.database_path.clone()).unwrap());
-        let accept_updates = Arc::new(AtomicBool::new(true));
-
-        let inner_data = DataInner {
-            db: db.clone(),
-            db_path,
-            admin_token,
-            server_pid,
-            accept_updates,
-        };
-
-        let data = Data {
-            inner: Arc::new(inner_data),
-        };
-
-        for index_name in db.indexes_names().unwrap() {
-            let callback_context = data.clone();
-            let callback_name = index_name.clone();
-            db.set_update_callback(
-                index_name,
-                Box::new(move |status| {
-                    index_update_callback(&callback_name, &callback_context, status);
-                }),
-            );
-        }
-
-        data
-    }
-}

meilidb-http/src/error.rs

@@ -1,117 +0,0 @@
-use std::fmt::Display;
-
-use http::status::StatusCode;
-use log::{error, warn};
-use serde::{Deserialize, Serialize};
-use tide::response::IntoResponse;
-use tide::Response;
-
-pub type SResult<T> = Result<T, ResponseError>;
-
-pub enum ResponseError {
-    Internal(String),
-    BadRequest(String),
-    InvalidToken(String),
-    NotFound(String),
-    IndexNotFound(String),
-    DocumentNotFound(String),
-    MissingHeader(String),
-    BadParameter(String, String),
-    CreateIndex(String),
-    Maintenance,
-}
-
-impl ResponseError {
-    pub fn internal(message: impl Display) -> ResponseError {
-        ResponseError::Internal(message.to_string())
-    }
-
-    pub fn bad_request(message: impl Display) -> ResponseError {
-        ResponseError::BadRequest(message.to_string())
-    }
-
-    pub fn invalid_token(message: impl Display) -> ResponseError {
-        ResponseError::InvalidToken(message.to_string())
-    }
-
-    pub fn not_found(message: impl Display) -> ResponseError {
-        ResponseError::NotFound(message.to_string())
-    }
-
-    pub fn index_not_found(message: impl Display) -> ResponseError {
-        ResponseError::IndexNotFound(message.to_string())
-    }
-
-    pub fn document_not_found(message: impl Display) -> ResponseError {
-        ResponseError::DocumentNotFound(message.to_string())
-    }
-
-    pub fn missing_header(message: impl Display) -> ResponseError {
-        ResponseError::MissingHeader(message.to_string())
-    }
-
-    pub fn bad_parameter(name: impl Display, message: impl Display) -> ResponseError {
-        ResponseError::BadParameter(name.to_string(), message.to_string())
-    }
-
-    pub fn create_index(message: impl Display) -> ResponseError {
-        ResponseError::CreateIndex(message.to_string())
-    }
-}
-
-impl IntoResponse for ResponseError {
-    fn into_response(self) -> Response {
-        match self {
-            ResponseError::Internal(err) => {
-                error!("internal server error: {}", err);
-                error(
-                    String::from("Internal server error"),
-                    StatusCode::INTERNAL_SERVER_ERROR,
-                )
-            }
-            ResponseError::BadRequest(err) => {
-                warn!("bad request: {}", err);
-                error(err, StatusCode::BAD_REQUEST)
-            }
-            ResponseError::InvalidToken(err) => {
-                error(format!("Invalid Token: {}", err), StatusCode::FORBIDDEN)
-            }
-            ResponseError::NotFound(err) => error(err, StatusCode::NOT_FOUND),
-            ResponseError::IndexNotFound(index) => {
-                error(format!("Index {} not found", index), StatusCode::NOT_FOUND)
-            }
-            ResponseError::DocumentNotFound(id) => error(
-                format!("Document with id {} not found", id),
-                StatusCode::NOT_FOUND,
-            ),
-            ResponseError::MissingHeader(header) => error(
-                format!("Header {} is missing", header),
-                StatusCode::UNAUTHORIZED,
-            ),
-            ResponseError::BadParameter(param, e) => error(
-                format!("Url parameter {} error: {}", param, e),
-                StatusCode::BAD_REQUEST,
-            ),
-            ResponseError::CreateIndex(err) => error(
-                format!("Impossible to create index; {}", err),
-                StatusCode::BAD_REQUEST,
-            ),
-            ResponseError::Maintenance => error(
-                String::from("Server is in maintenance, please try again later"),
-                StatusCode::SERVICE_UNAVAILABLE,
-            ),
-        }
-    }
-}
-
-#[derive(Serialize, Deserialize)]
-struct ErrorMessage {
-    message: String,
-}
-
-fn error(message: String, status: StatusCode) -> Response {
-    let message = ErrorMessage { message };
-    tide::response::json(message)
-        .with_status(status)
-        .into_response()
-}

meilidb-http/src/helpers/meilidb.rs

@@ -1,568 +0,0 @@
-use crate::routes::setting::{RankingOrdering, SettingBody};
-use indexmap::IndexMap;
-use log::*;
-use meilidb_core::criterion::*;
-use meilidb_core::Highlight;
-use meilidb_core::{Index, RankedMap};
-use meilidb_schema::{Schema, SchemaAttr};
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
-use std::cmp::Ordering;
-use std::collections::{HashMap, HashSet};
-use std::convert::From;
-use std::error;
-use std::fmt;
-use std::time::{Duration, Instant};
-
-#[derive(Debug)]
-pub enum Error {
-    SearchDocuments(String),
-    RetrieveDocument(u64, String),
-    DocumentNotFound(u64),
-    CropFieldWrongType(String),
-    AttributeNotFoundOnDocument(String),
-    AttributeNotFoundOnSchema(String),
-    MissingFilterValue,
-    UnknownFilteredAttribute,
-    Internal(String),
-}
-
-impl error::Error for Error {}
-
-impl fmt::Display for Error {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        use Error::*;
-
-        match self {
-            SearchDocuments(err) => write!(f, "impossible to search documents; {}", err),
-            RetrieveDocument(id, err) => write!(
-                f,
-                "impossible to retrieve the document with id: {}; {}",
-                id, err
-            ),
-            DocumentNotFound(id) => write!(f, "document {} not found", id),
-            CropFieldWrongType(field) => {
-                write!(f, "the field {} cannot be cropped it's not a string", field)
-            }
-            AttributeNotFoundOnDocument(field) => {
-                write!(f, "field {} is not found on document", field)
-            }
-            AttributeNotFoundOnSchema(field) => write!(f, "field {} is not found on schema", field),
-            MissingFilterValue => f.write_str("a filter doesn't have a value to compare it with"),
-            UnknownFilteredAttribute => {
-                f.write_str("a filter is specifying an unknown schema attribute")
-            }
-            Internal(err) => write!(f, "internal error; {}", err),
-        }
-    }
-}
-
-impl From<meilidb_core::Error> for Error {
-    fn from(error: meilidb_core::Error) -> Self {
-        Error::Internal(error.to_string())
-    }
-}
-
-pub trait IndexSearchExt {
-    fn new_search(&self, query: String) -> SearchBuilder;
-}
-
-impl IndexSearchExt for Index {
-    fn new_search(&self, query: String) -> SearchBuilder {
-        SearchBuilder {
-            index: self,
-            query,
-            offset: 0,
-            limit: 20,
-            attributes_to_crop: None,
-            attributes_to_retrieve: None,
-            attributes_to_search_in: None,
-            attributes_to_highlight: None,
-            filters: None,
-            timeout: Duration::from_millis(30),
-            matches: false,
-        }
-    }
-}
-
-pub struct SearchBuilder<'a> {
-    index: &'a Index,
-    query: String,
-    offset: usize,
-    limit: usize,
-    attributes_to_crop: Option<HashMap<String, usize>>,
-    attributes_to_retrieve: Option<HashSet<String>>,
-    attributes_to_search_in: Option<HashSet<String>>,
-    attributes_to_highlight: Option<HashSet<String>>,
-    filters: Option<String>,
-    timeout: Duration,
-    matches: bool,
-}
-
-impl<'a> SearchBuilder<'a> {
-    pub fn offset(&mut self, value: usize) -> &SearchBuilder {
-        self.offset = value;
-        self
-    }
-
-    pub fn limit(&mut self, value: usize) -> &SearchBuilder {
-        self.limit = value;
-        self
-    }
-
-    pub fn attributes_to_crop(&mut self, value: HashMap<String, usize>) -> &SearchBuilder {
-        self.attributes_to_crop = Some(value);
-        self
-    }
-
-    pub fn attributes_to_retrieve(&mut self, value: HashSet<String>) -> &SearchBuilder {
-        self.attributes_to_retrieve = Some(value);
-        self
-    }
-
-    pub fn add_retrievable_field(&mut self, value: String) -> &SearchBuilder {
-        let attributes_to_retrieve = self.attributes_to_retrieve.get_or_insert(HashSet::new());
-        attributes_to_retrieve.insert(value);
-        self
-    }
-
-    pub fn attributes_to_search_in(&mut self, value: HashSet<String>) -> &SearchBuilder {
-        self.attributes_to_search_in = Some(value);
-        self
-    }
-
-    pub fn attributes_to_highlight(&mut self, value: HashSet<String>) -> &SearchBuilder {
-        self.attributes_to_highlight = Some(value);
-        self
-    }
-
-    pub fn filters(&mut self, value: String) -> &SearchBuilder {
-        self.filters = Some(value);
-        self
-    }
-
-    pub fn timeout(&mut self, value: Duration) -> &SearchBuilder {
-        self.timeout = value;
-        self
-    }
-
-    pub fn get_matches(&mut self) -> &SearchBuilder {
-        self.matches = true;
-        self
-    }
-
-    pub fn search(&self, reader: &heed::RoTxn) -> Result<SearchResult, Error> {
-        let schema = self.index.main.schema(reader);
-        let schema = schema.map_err(|e| Error::Internal(e.to_string()))?;
-        let schema = match schema {
-            Some(schema) => schema,
-            None => return Err(Error::Internal(String::from("missing schema"))),
-        };
-
-        let ranked_map = self.index.main.ranked_map(reader);
-        let ranked_map = ranked_map.map_err(|e| Error::Internal(e.to_string()))?;
-        let ranked_map = ranked_map.unwrap_or_default();
-
-        let start = Instant::now();
-
-        // Change criteria
-        let mut query_builder = match self.get_criteria(reader, &ranked_map, &schema)? {
-            Some(criteria) => self.index.query_builder_with_criteria(criteria),
-            None => self.index.query_builder(),
-        };
-
-        // Filter searchable fields
-        if let Some(fields) = &self.attributes_to_search_in {
-            for attribute in fields.iter().filter_map(|f| schema.attribute(f)) {
-                query_builder.add_searchable_attribute(attribute.0);
-            }
-        }
-
-        if let Some(filters) = &self.filters {
-            let mut split = filters.split(':');
-            match (split.next(), split.next()) {
-                (Some(_), None) | (Some(_), Some("")) => return Err(Error::MissingFilterValue),
-                (Some(attr), Some(value)) => {
-                    let ref_reader = reader;
-                    let ref_index = &self.index;
-                    let value = value.trim().to_lowercase();
-
-                    let attr = match schema.attribute(attr) {
-                        Some(attr) => attr,
-                        None => return Err(Error::UnknownFilteredAttribute),
-                    };
-
-                    query_builder.with_filter(move |id| {
-                        let attr = attr;
-                        let index = ref_index;
-                        let reader = ref_reader;
-
-                        match index.document_attribute::<Value>(reader, id, attr) {
-                            Ok(Some(Value::String(s))) => s.to_lowercase() == value,
-                            Ok(Some(Value::Bool(b))) => {
-                                (value == "true" && b) || (value == "false" && !b)
-                            }
-                            Ok(Some(Value::Array(a))) => {
-                                a.into_iter().any(|s| s.as_str() == Some(&value))
-                            }
-                            _ => false,
-                        }
-                    });
-                }
-                (_, _) => (),
-            }
-        }
-
-        query_builder.with_fetch_timeout(self.timeout);
-
-        let docs =
-            query_builder.query(reader, &self.query, self.offset..(self.offset + self.limit));
-
-        let mut hits = Vec::with_capacity(self.limit);
-        for doc in docs.map_err(|e| Error::SearchDocuments(e.to_string()))? {
-            // retrieve the content of document in kv store
-            let mut fields: Option<HashSet<&str>> = None;
-            if let Some(attributes_to_retrieve) = &self.attributes_to_retrieve {
-                let mut set = HashSet::new();
-                for field in attributes_to_retrieve {
-                    set.insert(field.as_str());
-                }
-                fields = Some(set);
-            }
-            let mut document: IndexMap<String, Value> = self
-                .index
-                .document(reader, fields.as_ref(), doc.id)
-                .map_err(|e| Error::RetrieveDocument(doc.id.0, e.to_string()))?
-                .ok_or(Error::DocumentNotFound(doc.id.0))?;
-
-            let mut matches = doc.highlights.clone();
-
-            // Crops fields if needed
-            if let Some(fields) = self.attributes_to_crop.clone() {
-                for (field, length) in fields {
-                    let _ = crop_document(&mut document, &mut matches, &schema, &field, length);
-                }
-            }
-
-            // Transform to readable matches
-            let matches = calculate_matches(matches, self.attributes_to_retrieve.clone(), &schema);
-
-            if !self.matches {
-                if let Some(attributes_to_highlight) = self.attributes_to_highlight.clone() {
-                    let highlights = calculate_highlights(
-                        document.clone(),
-                        matches.clone(),
-                        attributes_to_highlight,
-                    );
-                    for (key, value) in highlights {
-                        if let Some(content) = document.get_mut(&key) {
-                            *content = value;
-                        }
-                    }
-                }
-            }
-
-            let matches_info = if self.matches { Some(matches) } else { None };
-
-            let hit = SearchHit {
-                hit: document,
-                matches_info,
-            };
-
-            hits.push(hit);
-        }
-
-        let time_ms = start.elapsed().as_millis() as usize;
-
-        let results = SearchResult {
-            hits,
-            offset: self.offset,
-            limit: self.limit,
-            processing_time_ms: time_ms,
-            query: self.query.to_string(),
-        };
-
-        Ok(results)
-    }
-
-    pub fn get_criteria(
-        &self,
-        reader: &heed::RoTxn,
-        ranked_map: &'a RankedMap,
-        schema: &Schema,
-    ) -> Result<Option<Criteria<'a>>, Error> {
-        let current_settings = match self.index.main.customs(reader).unwrap() {
-            Some(bytes) => bincode::deserialize(bytes).unwrap(),
-            None => SettingBody::default(),
-        };
-
-        let ranking_rules = &current_settings.ranking_rules;
-        let ranking_order = &current_settings.ranking_order;
-
-        if let Some(ranking_rules) = ranking_rules {
-            let mut builder = CriteriaBuilder::with_capacity(7 + ranking_rules.len());
-            if let Some(ranking_rules_order) = ranking_order {
-                for rule in ranking_rules_order {
-                    match rule.as_str() {
-                        "_sum_of_typos" => builder.push(SumOfTypos),
-                        "_number_of_words" => builder.push(NumberOfWords),
-                        "_word_proximity" => builder.push(WordsProximity),
-                        "_sum_of_words_attribute" => builder.push(SumOfWordsAttribute),
-                        "_sum_of_words_position" => builder.push(SumOfWordsPosition),
-                        "_exact" => builder.push(Exact),
-                        _ => {
-                            let order = match ranking_rules.get(rule.as_str()) {
-                                Some(o) => o,
-                                None => continue,
-                            };
-
-                            let custom_ranking = match order {
-                                RankingOrdering::Asc => {
-                                    SortByAttr::lower_is_better(&ranked_map, &schema, &rule)
-                                        .unwrap()
-                                }
-                                RankingOrdering::Dsc => {
-                                    SortByAttr::higher_is_better(&ranked_map, &schema, &rule)
-                                        .unwrap()
-                                }
-                            };
-
-                            builder.push(custom_ranking);
-                        }
-                    }
-                }
-                builder.push(DocumentId);
-                return Ok(Some(builder.build()));
-            } else {
-                builder.push(SumOfTypos);
-                builder.push(NumberOfWords);
-                builder.push(WordsProximity);
-                builder.push(SumOfWordsAttribute);
-                builder.push(SumOfWordsPosition);
-                builder.push(Exact);
-                for (rule, order) in ranking_rules.iter() {
-                    let custom_ranking = match order {
-                        RankingOrdering::Asc => {
-                            SortByAttr::lower_is_better(&ranked_map, &schema, &rule).unwrap()
-                        }
-                        RankingOrdering::Dsc => {
-                            SortByAttr::higher_is_better(&ranked_map, &schema, &rule).unwrap()
-                        }
-                    };
-                    builder.push(custom_ranking);
-                }
-                builder.push(DocumentId);
-                return Ok(Some(builder.build()));
-            }
-        }
-
-        Ok(None)
-    }
-}
-
-#[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Serialize, Deserialize)]
-pub struct MatchPosition {
-    pub start: usize,
-    pub length: usize,
-}
-
-impl Ord for MatchPosition {
-    fn cmp(&self, other: &Self) -> Ordering {
-        match self.start.cmp(&other.start) {
-            Ordering::Equal => self.length.cmp(&other.length),
-            _ => self.start.cmp(&other.start),
-        }
-    }
-}
-
-pub type HighlightInfos = HashMap<String, Value>;
-pub type MatchesInfos = HashMap<String, Vec<MatchPosition>>;
-// pub type RankingInfos = HashMap<String, u64>;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct SearchHit {
-    #[serde(flatten)]
-    pub hit: IndexMap<String, Value>,
-    #[serde(rename = "_matchesInfo", skip_serializing_if = "Option::is_none")]
-    pub matches_info: Option<MatchesInfos>,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct SearchResult {
-    pub hits: Vec<SearchHit>,
-    pub offset: usize,
-    pub limit: usize,
-    pub processing_time_ms: usize,
-    pub query: String,
-    // pub parsed_query: String,
-    // pub params: Option<String>,
-}
-
-fn crop_text(
-    text: &str,
-    matches: impl IntoIterator<Item = Highlight>,
-    context: usize,
-) -> (String, Vec<Highlight>) {
-    let mut matches = matches.into_iter().peekable();
-
-    let char_index = matches.peek().map(|m| m.char_index as usize).unwrap_or(0);
-    let start = char_index.saturating_sub(context);
-    let text = text.chars().skip(start).take(context * 2).collect();
-
-    let matches = matches
-        .take_while(|m| (m.char_index as usize) + (m.char_length as usize) <= start + (context * 2))
-        .map(|match_| Highlight {
-            char_index: match_.char_index - start as u16,
-            ..match_
-        })
-        .collect();
-
-    (text, matches)
-}
-
-fn crop_document(
-    document: &mut IndexMap<String, Value>,
-    matches: &mut Vec<Highlight>,
-    schema: &Schema,
-    field: &str,
-    length: usize,
-) -> Result<(), Error> {
-    matches.sort_unstable_by_key(|m| (m.char_index, m.char_length));
-
-    let attribute = schema
-        .attribute(field)
-        .ok_or(Error::AttributeNotFoundOnSchema(field.to_string()))?;
-    let selected_matches = matches
-        .iter()
-        .filter(|m| SchemaAttr::new(m.attribute) == attribute)
-        .cloned();
-    let original_text = match document.get(field) {
-        Some(Value::String(text)) => text,
-        Some(_) => return Err(Error::CropFieldWrongType(field.to_string())),
-        None => return Err(Error::AttributeNotFoundOnDocument(field.to_string())),
-    };
-    let (cropped_text, cropped_matches) = crop_text(&original_text, selected_matches, length);
-
-    document.insert(
-        field.to_string(),
-        serde_json::value::Value::String(cropped_text),
-    );
-    matches.retain(|m| SchemaAttr::new(m.attribute) != attribute);
-    matches.extend_from_slice(&cropped_matches);
-    Ok(())
-}
-
-fn calculate_matches(
-    matches: Vec<Highlight>,
-    attributes_to_retrieve: Option<HashSet<String>>,
-    schema: &Schema,
-) -> MatchesInfos {
-    let mut matches_result: HashMap<String, Vec<MatchPosition>> = HashMap::new();
-    for m in matches.iter() {
-        let attribute = schema
-            .attribute_name(SchemaAttr::new(m.attribute))
-            .to_string();
-        if let Some(attributes_to_retrieve) = attributes_to_retrieve.clone() {
-            if !attributes_to_retrieve.contains(attribute.as_str()) {
-                continue;
-            }
-        };
-        if let Some(pos) = matches_result.get_mut(&attribute) {
-            pos.push(MatchPosition {
-                start: m.char_index as usize,
-                length: m.char_length as usize,
-            });
-        } else {
-            let mut positions = Vec::new();
-            positions.push(MatchPosition {
-                start: m.char_index as usize,
-                length: m.char_length as usize,
-            });
-            matches_result.insert(attribute, positions);
-        }
-    }
-    for (_, val) in matches_result.iter_mut() {
-        val.sort_unstable();
-        val.dedup();
-    }
-    matches_result
-}
-
-fn calculate_highlights(
-    document: IndexMap<String, Value>,
-    matches: MatchesInfos,
-    attributes_to_highlight: HashSet<String>,
-) -> HighlightInfos {
-    let mut highlight_result: HashMap<String, Value> = HashMap::new();
-    for (attribute, matches) in matches.iter() {
-        if attributes_to_highlight.contains("*") || attributes_to_highlight.contains(attribute) {
-            if let Some(Value::String(value)) = document.get(attribute) {
-                let value: Vec<_> = value.chars().collect();
-                let mut highlighted_value = String::new();
-                let mut index = 0;
-                for m in matches {
-                    if m.start >= index {
-                        let before = value.get(index..m.start);
-                        let highlighted = value.get(m.start..(m.start + m.length));
-                        if let (Some(before), Some(highlighted)) = (before, highlighted) {
-                            highlighted_value.extend(before);
-                            highlighted_value.push_str("<em>");
-                            highlighted_value.extend(highlighted);
-                            highlighted_value.push_str("</em>");
-                            index = m.start + m.length;
-                        } else {
-                            error!("value: {:?}; index: {:?}, match: {:?}", value, index, m);
-                        }
-                    }
-                }
-                highlighted_value.extend(value[index..].iter());
-                highlight_result.insert(attribute.to_string(), Value::String(highlighted_value));
-            };
-        }
-    }
-    highlight_result
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn calculate_highlights() {
-        let data = r#"{
-            "title": "Fondation (Isaac ASIMOV)",
-            "description": "En ce début de trentième millénaire, l'Empire n'a jamais été aussi puissant, aussi étendu à travers toute la galaxie. C'est dans sa capitale, Trantor, que l'éminent savant Hari Seldon invente la psychohistoire, une science toute nouvelle, à base de psychologie et de mathématiques, qui lui permet de prédire l'avenir... C'est-à-dire l'effondrement de l'Empire d'ici cinq siècles et au-delà, trente mille années de chaos et de ténèbres. Pour empêcher cette catastrophe et sauver la civilisation, Seldon crée la Fondation."
-        }"#;
-
-        let document: IndexMap<String, Value> = serde_json::from_str(data).unwrap();
-        let mut attributes_to_highlight = HashSet::new();
-        attributes_to_highlight.insert("*".to_string());
-
-        let mut matches: HashMap<String, Vec<MatchPosition>> = HashMap::new();
-
-        let mut m = Vec::new();
-        m.push(MatchPosition {
-            start: 0,
-            length: 9,
-        });
-        matches.insert("title".to_string(), m);
-
-        let mut m = Vec::new();
-        m.push(MatchPosition {
-            start: 510,
-            length: 9,
-        });
-        matches.insert("description".to_string(), m);
-        let result = super::calculate_highlights(document, matches, attributes_to_highlight);
-
-        let mut result_expected = HashMap::new();
-        result_expected.insert(
-            "title".to_string(),
-            Value::String("<em>Fondation</em> (Isaac ASIMOV)".to_string()),
-        );
-        result_expected.insert("description".to_string(), Value::String("En ce début de trentième millénaire, l'Empire n'a jamais été aussi puissant, aussi étendu à travers toute la galaxie. C'est dans sa capitale, Trantor, que l'éminent savant Hari Seldon invente la psychohistoire, une science toute nouvelle, à base de psychologie et de mathématiques, qui lui permet de prédire l'avenir... C'est-à-dire l'effondrement de l'Empire d'ici cinq siècles et au-delà, trente mille années de chaos et de ténèbres. Pour empêcher cette catastrophe et sauver la civilisation, Seldon crée la <em>Fondation</em>.".to_string()));
-
-        assert_eq!(result, result_expected);
-    }
-}

meilidb-http/src/helpers/mod.rs

@@ -1,2 +0,0 @@
-pub mod meilidb;
-pub mod tide;

meilidb-http/src/helpers/tide.rs

@@ -1,118 +0,0 @@
-use crate::error::{ResponseError, SResult};
-use crate::models::token::*;
-use crate::Data;
-use chrono::Utc;
-use heed::types::{SerdeBincode, Str};
-use meilidb_core::Index;
-use serde_json::Value;
-use tide::Context;
-
-pub trait ContextExt {
-    fn is_allowed(&self, acl: ACL) -> SResult<()>;
-    fn header(&self, name: &str) -> Result<String, ResponseError>;
-    fn url_param(&self, name: &str) -> Result<String, ResponseError>;
-    fn index(&self) -> Result<Index, ResponseError>;
-    fn identifier(&self) -> Result<String, ResponseError>;
-}
-
-impl ContextExt for Context<Data> {
-    fn is_allowed(&self, acl: ACL) -> SResult<()> {
-        let admin_token = match &self.state().admin_token {
-            Some(admin_token) => admin_token,
-            None => return Ok(()),
-        };
-
-        let user_api_key = self.header("X-Meili-API-Key")?;
-        if user_api_key == *admin_token {
-            return Ok(());
-        }
-        let request_index: Option<String> = None; //self.param::<String>("index").ok();
-
-        let db = &self.state().db;
-        let env = &db.env;
-        let reader = env.read_txn().map_err(ResponseError::internal)?;
-
-        let token_key = format!("{}{}", TOKEN_PREFIX_KEY, user_api_key);
-
-        let token_config = db
-            .common_store()
-            .get::<Str, SerdeBincode<Token>>(&reader, &token_key)
-            .map_err(ResponseError::internal)?
-            .ok_or(ResponseError::not_found(format!(
-                "token key: {}",
-                token_key
-            )))?;
-
-        if token_config.revoked {
-            return Err(ResponseError::invalid_token("token revoked"));
-        }
-
-        if let Some(index) = request_index {
-            if !token_config
-                .indexes
-                .iter()
-                .any(|r| match_wildcard(&r, &index))
-            {
-                return Err(ResponseError::invalid_token(
-                    "token is not allowed to access to this index",
-                ));
-            }
-        }
-
-        if token_config.expires_at < Utc::now() {
-            return Err(ResponseError::invalid_token("token expired"));
-        }
-
-        if token_config.acl.contains(&ACL::All) {
-            return Ok(());
-        }
-
-        if !token_config.acl.contains(&acl) {
-            return Err(ResponseError::invalid_token("token do not have this ACL"));
-        }
-
-        Ok(())
-    }
-
-    fn header(&self, name: &str) -> Result<String, ResponseError> {
-        let header = self
-            .headers()
-            .get(name)
-            .ok_or(ResponseError::missing_header(name))?
-            .to_str()
-            .map_err(|_| ResponseError::missing_header("X-Meili-API-Key"))?
-            .to_string();
-        Ok(header)
-    }
-
-    fn url_param(&self, name: &str) -> Result<String, ResponseError> {
-        let param = self
-            .param::<String>(name)
-            .map_err(|e| ResponseError::bad_parameter(name, e))?;
-        Ok(param)
-    }
-
-    fn index(&self) -> Result<Index, ResponseError> {
-        let index_name = self.url_param("index")?;
-        let index = self
-            .state()
-            .db
-            .open_index(&index_name)
-            .ok_or(ResponseError::index_not_found(index_name))?;
-        Ok(index)
-    }
-
-    fn identifier(&self) -> Result<String, ResponseError> {
-        let name = self
-            .param::<Value>("identifier")
-            .as_ref()
-            .map(meilidb_core::serde::value_to_string)
-            .map_err(|e| ResponseError::bad_parameter("identifier", e))?
-            .ok_or(ResponseError::bad_parameter(
-                "identifier",
-                "missing parameter",
-            ))?;
-
-        Ok(name)
-    }
-}

meilidb-http/src/lib.rs

@@ -1,11 +0,0 @@
-#[macro_use]
-extern crate envconfig_derive;
-
-pub mod data;
-pub mod error;
-pub mod helpers;
-pub mod models;
-pub mod option;
-pub mod routes;
-
-use self::data::Data;

meilidb-http/src/main.rs

@@ -1,36 +0,0 @@
-use http::header::HeaderValue;
-use log::info;
-use main_error::MainError;
-use tide::middleware::{CorsMiddleware, CorsOrigin};
-use tide_log::RequestLogger;
-
-use meilidb_http::data::Data;
-use meilidb_http::option::Opt;
-use meilidb_http::routes;
-
-#[cfg(not(target_os = "macos"))]
-#[global_allocator]
-static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
-
-pub fn main() -> Result<(), MainError> {
-    let opt = Opt::new();
-
-    let data = Data::new(opt.clone());
-    let mut app = tide::App::with_state(data);
-
-    app.middleware(
-        CorsMiddleware::new()
-            .allow_origin(CorsOrigin::from("*"))
-            .allow_methods(HeaderValue::from_static("GET, POST, OPTIONS")),
-    );
-    app.middleware(RequestLogger::new());
-    app.middleware(tide_compression::Compression::new());
-    app.middleware(tide_compression::Decompression::new());
-
-    routes::load_routes(&mut app);
-
-    info!("Server HTTP enabled");
-    app.run(opt.http_addr)?;
-
-    Ok(())
-}

meilidb-http/src/models/mod.rs

@@ -1,3 +0,0 @@
-pub mod schema;
-pub mod token;
-pub mod update_operation;

meilidb-http/src/models/schema.rs

@@ -1,118 +0,0 @@
-use std::collections::HashSet;
-
-use indexmap::IndexMap;
-use meilidb_schema::{Schema, SchemaBuilder, SchemaProps};
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub enum FieldProperties {
-    Identifier,
-    Indexed,
-    Displayed,
-    Ranked,
-}
-
-#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
-pub struct SchemaBody(IndexMap<String, HashSet<FieldProperties>>);
-
-impl From<Schema> for SchemaBody {
-    fn from(value: Schema) -> SchemaBody {
-        let mut map = IndexMap::new();
-        for (name, _attr, props) in value.iter() {
-            let old_properties = map.entry(name.to_owned()).or_insert(HashSet::new());
-            if props.is_indexed() {
-                old_properties.insert(FieldProperties::Indexed);
-            }
-            if props.is_displayed() {
-                old_properties.insert(FieldProperties::Displayed);
-            }
-            if props.is_ranked() {
-                old_properties.insert(FieldProperties::Ranked);
-            }
-        }
-        let old_properties = map
-            .entry(value.identifier_name().to_string())
-            .or_insert(HashSet::new());
-        old_properties.insert(FieldProperties::Identifier);
-        old_properties.insert(FieldProperties::Displayed);
-        SchemaBody(map)
-    }
-}
-
-impl Into<Schema> for SchemaBody {
-    fn into(self) -> Schema {
-        let mut identifier = "documentId".to_string();
-        let mut attributes = IndexMap::new();
-        for (field, properties) in self.0 {
-            let mut indexed = false;
-            let mut displayed = false;
-            let mut ranked = false;
-            for property in properties {
-                match property {
-                    FieldProperties::Indexed => indexed = true,
-                    FieldProperties::Displayed => displayed = true,
-                    FieldProperties::Ranked => ranked = true,
-                    FieldProperties::Identifier => identifier = field.clone(),
-                }
-            }
-            attributes.insert(
-                field,
-                SchemaProps {
-                    indexed,
-                    displayed,
-                    ranked,
-                },
-            );
-        }
-
-        let mut builder = SchemaBuilder::with_identifier(identifier);
-        for (field, props) in attributes {
-            builder.new_attribute(field, props);
-        }
-        builder.build()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_schema_body_conversion() {
-        let schema_body = r#"
-        {
-            "id": ["identifier", "indexed", "displayed"],
-            "title": ["indexed", "displayed"],
-            "date": ["displayed"]
-        }
-        "#;
-
-        let schema_builder = r#"
-        {
-            "identifier": "id",
-            "attributes": {
-                "id": {
-                    "indexed": true,
-                    "displayed": true
-                },
-                "title": {
-                    "indexed": true,
-                    "displayed": true
-                },
-                "date": {
-                    "displayed": true
-                }
-            }
-        }
-        "#;
-
-        let schema_body: SchemaBody = serde_json::from_str(schema_body).unwrap();
-        let schema_builder: SchemaBuilder = serde_json::from_str(schema_builder).unwrap();
-
-        let schema_from_body: Schema = schema_body.into();
-        let schema_from_builder: Schema = schema_builder.build();
-
-        assert_eq!(schema_from_body, schema_from_builder);
-    }
-}

meilidb-http/src/models/token.rs

@@ -1,72 +0,0 @@
-use chrono::{DateTime, Utc};
-use serde::{Deserialize, Serialize};
-
-pub const TOKEN_PREFIX_KEY: &str = "_token_";
-
-#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub enum ACL {
-    IndexesRead,
-    IndexesWrite,
-    DocumentsRead,
-    DocumentsWrite,
-    SettingsRead,
-    SettingsWrite,
-    Admin,
-    #[serde(rename = "*")]
-    All,
-}
-
-pub type Wildcard = String;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct Token {
-    pub key: String,
-    pub description: String,
-    pub acl: Vec<ACL>,
-    pub indexes: Vec<Wildcard>,
-    pub created_at: DateTime<Utc>,
-    pub updated_at: DateTime<Utc>,
-    pub expires_at: DateTime<Utc>,
-    pub revoked: bool,
-}
-
-fn cleanup_wildcard(input: &str) -> (bool, &str, bool) {
-    let first = input.chars().next().filter(|&c| c == '*').is_some();
-    let last = input.chars().last().filter(|&c| c == '*').is_some();
-    let bound_last = std::cmp::max(input.len().saturating_sub(last as usize), first as usize);
-    let output = input.get(first as usize..bound_last).unwrap();
-    (first, output, last)
-}
-
-pub fn match_wildcard(pattern: &str, input: &str) -> bool {
-    let (first, pattern, last) = cleanup_wildcard(pattern);
-
-    match (first, last) {
-        (false, false) => pattern == input,
-        (true, false) => input.ends_with(pattern),
-        (false, true) => input.starts_with(pattern),
-        (true, true) => input.contains(pattern),
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_match_wildcard() {
-        assert!(match_wildcard("*", "qqq"));
-        assert!(match_wildcard("*", ""));
-        assert!(match_wildcard("*ab", "qqqab"));
-        assert!(match_wildcard("*ab*", "qqqabqq"));
-        assert!(match_wildcard("ab*", "abqqq"));
-        assert!(match_wildcard("**", "ab"));
-        assert!(match_wildcard("ab", "ab"));
-        assert!(match_wildcard("ab*", "ab"));
-        assert!(match_wildcard("*ab", "ab"));
-        assert!(match_wildcard("*ab*", "ab"));
-        assert!(match_wildcard("*😆*", "ab😆dsa"));
-    }
-}

meilidb-http/src/option.rs

@@ -1,56 +0,0 @@
-use envconfig::Envconfig;
-use structopt::StructOpt;
-
-#[derive(Debug, Clone, StructOpt, Envconfig)]
-struct Vars {
-    /// The destination where the database must be created.
-    #[structopt(long)]
-    #[envconfig(from = "MEILI_DATABASE_PATH")]
-    pub database_path: Option<String>,
-
-    /// The addr on which the http server will listen.
-    #[structopt(long)]
-    #[envconfig(from = "MEILI_HTTP_ADDR")]
-    pub http_addr: Option<String>,
-
-    #[structopt(long)]
-    #[envconfig(from = "MEILI_ADMIN_TOKEN")]
-    pub admin_token: Option<String>,
-}
-
-#[derive(Clone, Debug)]
-pub struct Opt {
-    pub database_path: String,
-    pub http_addr: String,
-    pub admin_token: Option<String>,
-}
-
-impl Default for Opt {
-    fn default() -> Self {
-        Opt {
-            database_path: String::from("/tmp/meilidb"),
-            http_addr: String::from("127.0.0.1:8080"),
-            admin_token: None,
-        }
-    }
-}
-
-impl Opt {
-    pub fn new() -> Self {
-        let default = Self::default();
-        let args = Vars::from_args();
-        let env = Vars::init().unwrap();
-
-        Self {
-            database_path: env
-                .database_path
-                .or(args.database_path)
-                .unwrap_or(default.database_path),
-            http_addr: env
-                .http_addr
-                .or(args.http_addr)
-                .unwrap_or(default.http_addr),
-            admin_token: env.admin_token.or(args.admin_token).or(default.admin_token),
-        }
-    }
-}

meilidb-http/src/routes/document.rs

@@ -1,250 +0,0 @@
-use std::collections::{BTreeSet, HashSet};
-
-use http::StatusCode;
-use indexmap::IndexMap;
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
-use tide::querystring::ContextExt as QSContextExt;
-use tide::response::IntoResponse;
-use tide::{Context, Response};
-
-use crate::error::{ResponseError, SResult};
-use crate::helpers::tide::ContextExt;
-use crate::models::token::ACL::*;
-use crate::Data;
-
-pub async fn get_document(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(DocumentsRead)?;
-
-    let index = ctx.index()?;
-
-    let identifier = ctx.identifier()?;
-    let document_id = meilidb_core::serde::compute_document_id(identifier.clone());
-
-    let env = &ctx.state().db.env;
-    let reader = env.read_txn().map_err(ResponseError::internal)?;
-
-    let response = index
-        .document::<IndexMap<String, Value>>(&reader, None, document_id)
-        .map_err(ResponseError::internal)?
-        .ok_or(ResponseError::document_not_found(&identifier))?;
-
-    if response.is_empty() {
-        return Err(ResponseError::document_not_found(identifier));
-    }
-
-    Ok(tide::response::json(response))
-}
-
-#[derive(Default, Serialize)]
-#[serde(rename_all = "camelCase")]
-pub struct IndexUpdateResponse {
-    pub update_id: u64,
-}
-
-pub async fn delete_document(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(DocumentsWrite)?;
-
-    if !ctx.state().accept_updates() {
-        return Err(ResponseError::Maintenance);
-    }
-
-    let index = ctx.index()?;
-    let identifier = ctx.identifier()?;
-    let document_id = meilidb_core::serde::compute_document_id(identifier.clone());
-
-    let env = &ctx.state().db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-
-    let mut documents_deletion = index.documents_deletion();
-    documents_deletion.delete_document_by_id(document_id);
-    let update_id = documents_deletion
-        .finalize(&mut writer)
-        .map_err(ResponseError::internal)?;
-
-    writer.commit().map_err(ResponseError::internal)?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
-}
-
-#[derive(Default, Deserialize)]
-#[serde(rename_all = "camelCase", deny_unknown_fields)]
-struct BrowseQuery {
-    offset: Option<usize>,
-    limit: Option<usize>,
-    attributes_to_retrieve: Option<String>,
-}
-
-pub async fn browse_documents(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(DocumentsRead)?;
-
-    let index = ctx.index()?;
-    let query: BrowseQuery = ctx.url_query().unwrap_or(BrowseQuery::default());
-
-    let offset = query.offset.unwrap_or(0);
-    let limit = query.limit.unwrap_or(20);
-
-    let env = &ctx.state().db.env;
-    let reader = env.read_txn().map_err(ResponseError::internal)?;
-
-    let documents_ids: Result<BTreeSet<_>, _> =
-        match index.documents_fields_counts.documents_ids(&reader) {
-            Ok(documents_ids) => documents_ids.skip(offset).take(limit).collect(),
-            Err(e) => return Err(ResponseError::internal(e)),
-        };
-
-    let documents_ids = match documents_ids {
-        Ok(documents_ids) => documents_ids,
-        Err(e) => return Err(ResponseError::internal(e)),
-    };
-
-    let mut response_body = Vec::<IndexMap<String, Value>>::new();
-
-    if let Some(attributes) = query.attributes_to_retrieve {
-        let attributes = attributes.split(',').collect::<HashSet<&str>>();
-        for document_id in documents_ids {
-            if let Ok(Some(document)) = index.document(&reader, Some(&attributes), document_id) {
-                response_body.push(document);
-            }
-        }
-    } else {
-        for document_id in documents_ids {
-            if let Ok(Some(document)) = index.document(&reader, None, document_id) {
-                response_body.push(document);
-            }
-        }
-    }
-
-    if response_body.is_empty() {
-        Ok(tide::response::json(response_body)
-            .with_status(StatusCode::NO_CONTENT)
-            .into_response())
-    } else {
-        Ok(tide::response::json(response_body)
-            .with_status(StatusCode::OK)
-            .into_response())
-    }
-}
-
-fn infered_schema(document: &IndexMap<String, Value>) -> Option<meilidb_schema::Schema> {
-    use meilidb_schema::{SchemaBuilder, DISPLAYED, INDEXED};
-
-    let mut identifier = None;
-    for key in document.keys() {
-        if identifier.is_none() && key.to_lowercase().contains("id") {
-            identifier = Some(key);
-        }
-    }
-
-    match identifier {
-        Some(identifier) => {
-            let mut builder = SchemaBuilder::with_identifier(identifier);
-            for key in document.keys() {
-                builder.new_attribute(key, DISPLAYED | INDEXED);
-            }
-            Some(builder.build())
-        }
-        None => None,
-    }
-}
-
-pub async fn add_or_update_multiple_documents(mut ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(DocumentsWrite)?;
-
-    if !ctx.state().accept_updates() {
-        return Err(ResponseError::Maintenance);
-    }
-    let data: Vec<IndexMap<String, Value>> =
-        ctx.body_json().await.map_err(ResponseError::bad_request)?;
-    let index = ctx.index()?;
-
-    let env = &ctx.state().db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-
-    let current_schema = index
-        .main
-        .schema(&writer)
-        .map_err(ResponseError::internal)?;
-    if current_schema.is_none() {
-        match data.first().and_then(infered_schema) {
-            Some(schema) => {
-                index
-                    .schema_update(&mut writer, schema)
-                    .map_err(ResponseError::internal)?;
-            }
-            None => return Err(ResponseError::bad_request("Could not infer a schema")),
-        }
-    }
-
-    let mut document_addition = index.documents_addition();
-
-    for document in data {
-        document_addition.update_document(document);
-    }
-
-    let update_id = document_addition
-        .finalize(&mut writer)
-        .map_err(ResponseError::internal)?;
-
-    writer.commit().map_err(ResponseError::internal)?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
-}
-
-pub async fn delete_multiple_documents(mut ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(DocumentsWrite)?;
-    if !ctx.state().accept_updates() {
-        return Err(ResponseError::Maintenance);
-    }
-    let data: Vec<Value> = ctx.body_json().await.map_err(ResponseError::bad_request)?;
-    let index = ctx.index()?;
-
-    let env = &ctx.state().db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-
-    let mut documents_deletion = index.documents_deletion();
-
-    for identifier in data {
-        if let Some(identifier) = meilidb_core::serde::value_to_string(&identifier) {
-            documents_deletion
-                .delete_document_by_id(meilidb_core::serde::compute_document_id(identifier));
-        }
-    }
-
-    let update_id = documents_deletion
-        .finalize(&mut writer)
-        .map_err(ResponseError::internal)?;
-
-    writer.commit().map_err(ResponseError::internal)?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
-}
-
-pub async fn clear_all_documents(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(DocumentsWrite)?;
-    if !ctx.state().accept_updates() {
-        return Err(ResponseError::Maintenance);
-    }
-    let index = ctx.index()?;
-
-    let env = &ctx.state().db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-    let update_id = index
-        .clear_all(&mut writer)
-        .map_err(ResponseError::internal)?;
-    writer.commit().map_err(ResponseError::internal)?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
-}

meilidb-http/src/routes/health.rs

@@ -1,79 +0,0 @@
-use crate::error::{ResponseError, SResult};
-use crate::helpers::tide::ContextExt;
-use crate::models::token::ACL::*;
-use crate::Data;
-
-use heed::types::{Str, Unit};
-use serde::Deserialize;
-use tide::Context;
-
-const UNHEALTHY_KEY: &str = "_is_unhealthy";
-
-pub async fn get_health(ctx: Context<Data>) -> SResult<()> {
-    let db = &ctx.state().db;
-    let env = &db.env;
-    let reader = env.read_txn().map_err(ResponseError::internal)?;
-
-    let common_store = ctx.state().db.common_store();
-
-    if let Ok(Some(_)) = common_store.get::<Str, Unit>(&reader, UNHEALTHY_KEY) {
-        return Err(ResponseError::Maintenance);
-    }
-
-    Ok(())
-}
-
-pub async fn set_healthy(ctx: Context<Data>) -> SResult<()> {
-    ctx.is_allowed(Admin)?;
-
-    let db = &ctx.state().db;
-    let env = &db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-
-    let common_store = ctx.state().db.common_store();
-    match common_store.delete::<Str>(&mut writer, UNHEALTHY_KEY) {
-        Ok(_) => (),
-        Err(e) => return Err(ResponseError::internal(e)),
-    }
-
-    if let Err(e) = writer.commit() {
-        return Err(ResponseError::internal(e));
-    }
-
-    Ok(())
-}
-
-pub async fn set_unhealthy(ctx: Context<Data>) -> SResult<()> {
-    ctx.is_allowed(Admin)?;
-
-    let db = &ctx.state().db;
-    let env = &db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-
-    let common_store = ctx.state().db.common_store();
-
-    if let Err(e) = common_store.put::<Str, Unit>(&mut writer, UNHEALTHY_KEY, &()) {
-        return Err(ResponseError::internal(e));
-    }
-
-    if let Err(e) = writer.commit() {
-        return Err(ResponseError::internal(e));
-    }
-
-    Ok(())
-}
-
-#[derive(Deserialize, Clone)]
-struct HealtBody {
-    health: bool,
-}
-
-pub async fn change_healthyness(mut ctx: Context<Data>) -> SResult<()> {
-    let body: HealtBody = ctx.body_json().await.map_err(ResponseError::bad_request)?;
-
-    if body.health {
-        set_healthy(ctx).await
-    } else {
-        set_unhealthy(ctx).await
-    }
-}

meilidb-http/src/routes/index.rs

@@ -1,210 +0,0 @@
-use http::StatusCode;
-use meilidb_core::{ProcessedUpdateResult, UpdateStatus};
-use meilidb_schema::Schema;
-use serde_json::json;
-use tide::response::IntoResponse;
-use tide::{Context, Response};
-
-use crate::error::{ResponseError, SResult};
-use crate::helpers::tide::ContextExt;
-use crate::models::schema::SchemaBody;
-use crate::models::token::ACL::*;
-use crate::routes::document::IndexUpdateResponse;
-use crate::Data;
-
-pub async fn list_indexes(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(IndexesRead)?;
-    let list = ctx
-        .state()
-        .db
-        .indexes_names()
-        .map_err(ResponseError::internal)?;
-    Ok(tide::response::json(list))
-}
-
-pub async fn get_index_schema(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(IndexesRead)?;
-
-    let index = ctx.index()?;
-
-    let env = &ctx.state().db.env;
-    let reader = env.read_txn().map_err(ResponseError::internal)?;
-
-    let schema = index
-        .main
-        .schema(&reader)
-        .map_err(ResponseError::create_index)?;
-
-    match schema {
-        Some(schema) => {
-            let schema = SchemaBody::from(schema);
-            Ok(tide::response::json(schema))
-        }
-        None => Ok(
-            tide::response::json(json!({ "message": "missing index schema" }))
-                .with_status(StatusCode::NOT_FOUND)
-                .into_response(),
-        ),
-    }
-}
-
-pub async fn create_index(mut ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(IndexesWrite)?;
-
-    let index_name = ctx.url_param("index")?;
-
-    let body = ctx.body_bytes().await.map_err(ResponseError::bad_request)?;
-    let schema: Option<Schema> = if body.is_empty() {
-        None
-    } else {
-        serde_json::from_slice::<SchemaBody>(&body)
-            .map_err(ResponseError::bad_request)
-            .map(|s| Some(s.into()))?
-    };
-
-    let db = &ctx.state().db;
-
-    let created_index = match db.create_index(&index_name) {
-        Ok(index) => index,
-        Err(meilidb_core::Error::IndexAlreadyExists) => db.open_index(&index_name).ok_or(
-            ResponseError::internal("index not found but must have been found"),
-        )?,
-        Err(e) => return Err(ResponseError::create_index(e)),
-    };
-
-    let callback_context = ctx.state().clone();
-    let callback_name = index_name.clone();
-    db.set_update_callback(
-        &index_name,
-        Box::new(move |status| {
-            index_update_callback(&callback_name, &callback_context, status);
-        }),
-    );
-
-    let env = &db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-
-    match schema {
-        Some(schema) => {
-            let update_id = created_index
-                .schema_update(&mut writer, schema.clone())
-                .map_err(ResponseError::internal)?;
-
-            writer.commit().map_err(ResponseError::internal)?;
-
-            let response_body = IndexUpdateResponse { update_id };
-            Ok(tide::response::json(response_body)
-                .with_status(StatusCode::CREATED)
-                .into_response())
-        }
-        None => Ok(Response::new(tide::Body::empty())
-            .with_status(StatusCode::NO_CONTENT)
-            .into_response()),
-    }
-}
-
-pub async fn update_schema(mut ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(IndexesWrite)?;
-
-    let index_name = ctx.url_param("index")?;
-
-    let schema = ctx
-        .body_json::<SchemaBody>()
-        .await
-        .map_err(ResponseError::bad_request)?;
-
-    let db = &ctx.state().db;
-    let env = &db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-
-    let index = db
-        .open_index(&index_name)
-        .ok_or(ResponseError::index_not_found(index_name))?;
-
-    let schema: meilidb_schema::Schema = schema.into();
-    let update_id = index
-        .schema_update(&mut writer, schema.clone())
-        .map_err(ResponseError::internal)?;
-
-    writer.commit().map_err(ResponseError::internal)?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
-}
-
-pub async fn get_update_status(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(IndexesRead)?;
-
-    let env = &ctx.state().db.env;
-    let reader = env.read_txn().map_err(ResponseError::internal)?;
-
-    let update_id = ctx
-        .param::<u64>("update_id")
-        .map_err(|e| ResponseError::bad_parameter("update_id", e))?;
-
-    let index = ctx.index()?;
-    let status = index
-        .update_status(&reader, update_id)
-        .map_err(ResponseError::internal)?;
-
-    let response = match status {
-        UpdateStatus::Enqueued(data) => {
-            tide::response::json(json!({ "status": "enqueued", "data": data }))
-                .with_status(StatusCode::OK)
-                .into_response()
-        }
-        UpdateStatus::Processed(data) => {
-            tide::response::json(json!({ "status": "processed", "data": data }))
-                .with_status(StatusCode::OK)
-                .into_response()
-        }
-        UpdateStatus::Unknown => tide::response::json(json!({ "message": "unknown update id" }))
-            .with_status(StatusCode::NOT_FOUND)
-            .into_response(),
-    };
-
-    Ok(response)
-}
-
-pub async fn get_all_updates_status(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(IndexesRead)?;
-
-    let env = &ctx.state().db.env;
-    let reader = env.read_txn().map_err(ResponseError::internal)?;
-
-    let index = ctx.index()?;
-    let all_status = index
-        .all_updates_status(&reader)
-        .map_err(ResponseError::internal)?;
-
-    let response = tide::response::json(all_status)
-        .with_status(StatusCode::OK)
-        .into_response();
-
-    Ok(response)
-}
-
-pub async fn delete_index(ctx: Context<Data>) -> SResult<StatusCode> {
-    ctx.is_allowed(IndexesWrite)?;
-    let _index_name = ctx.url_param("index")?;
-    let _index = ctx.index()?;
-
-    // ctx.state()
-    //     .db
-    //     .delete_index(&index_name)
-    //     .map_err(ResponseError::internal)?;
-
-    Ok(StatusCode::NOT_IMPLEMENTED)
-}
-
-pub fn index_update_callback(index_name: &str, data: &Data, _status: ProcessedUpdateResult) {
-    let env = &data.db.env;
-    let mut writer = env.write_txn().unwrap();
-
-    data.compute_stats(&mut writer, &index_name).unwrap();
-    data.set_last_update(&mut writer, &index_name).unwrap();
-
-    writer.commit().unwrap();
-}

meilidb-http/src/routes/key.rs

@@ -1,189 +0,0 @@
-use chrono::serde::ts_seconds;
-use chrono::{DateTime, Utc};
-use heed::types::{SerdeBincode, Str};
-use http::StatusCode;
-use rand::seq::SliceRandom;
-use serde::{Deserialize, Serialize};
-use tide::response::IntoResponse;
-use tide::{Context, Response};
-
-use crate::error::{ResponseError, SResult};
-use crate::helpers::tide::ContextExt;
-use crate::models::token::ACL::*;
-use crate::models::token::*;
-use crate::Data;
-
-fn generate_api_key() -> String {
-    let mut rng = rand::thread_rng();
-    let sample = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
-    sample
-        .choose_multiple(&mut rng, 40)
-        .map(|c| *c as char)
-        .collect()
-}
-
-pub async fn list(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(Admin)?;
-
-    let db = &ctx.state().db;
-    let env = &db.env;
-    let reader = env.read_txn().map_err(ResponseError::internal)?;
-
-    let common_store = db.common_store();
-
-    let mut response: Vec<Token> = Vec::new();
-
-    let iter = common_store
-        .prefix_iter::<Str, SerdeBincode<Token>>(&reader, TOKEN_PREFIX_KEY)
-        .map_err(ResponseError::internal)?;
-
-    for result in iter {
-        let (_, token) = result.map_err(ResponseError::internal)?;
-        response.push(token);
-    }
-
-    Ok(tide::response::json(response))
-}
-
-pub async fn get(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(Admin)?;
-    let request_key = ctx.url_param("key")?;
-
-    let db = &ctx.state().db;
-    let env = &db.env;
-    let reader = env.read_txn().map_err(ResponseError::internal)?;
-
-    let token_key = format!("{}{}", TOKEN_PREFIX_KEY, request_key);
-
-    let token_config = db
-        .common_store()
-        .get::<Str, SerdeBincode<Token>>(&reader, &token_key)
-        .map_err(ResponseError::internal)?
-        .ok_or(ResponseError::not_found(format!(
-            "token key: {}",
-            token_key
-        )))?;
-
-    Ok(tide::response::json(token_config))
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase", deny_unknown_fields)]
-pub struct CreatedRequest {
-    description: String,
-    acl: Vec<ACL>,
-    indexes: Vec<Wildcard>,
-    #[serde(with = "ts_seconds")]
-    expires_at: DateTime<Utc>,
-}
-
-pub async fn create(mut ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(Admin)?;
-
-    let data: CreatedRequest = ctx.body_json().await.map_err(ResponseError::bad_request)?;
-
-    let key = generate_api_key();
-    let token_key = format!("{}{}", TOKEN_PREFIX_KEY, key);
-
-    let token_definition = Token {
-        key,
-        description: data.description,
-        acl: data.acl,
-        indexes: data.indexes,
-        expires_at: data.expires_at,
-        created_at: Utc::now(),
-        updated_at: Utc::now(),
-        revoked: false,
-    };
-
-    let db = &ctx.state().db;
-    let env = &db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-
-    db.common_store()
-        .put::<Str, SerdeBincode<Token>>(&mut writer, &token_key, &token_definition)
-        .map_err(ResponseError::internal)?;
-
-    writer.commit().map_err(ResponseError::internal)?;
-
-    Ok(tide::response::json(token_definition)
-        .with_status(StatusCode::CREATED)
-        .into_response())
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase", deny_unknown_fields)]
-pub struct UpdatedRequest {
-    description: Option<String>,
-    acl: Option<Vec<ACL>>,
-    indexes: Option<Vec<Wildcard>>,
-}
-
-pub async fn update(mut ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(Admin)?;
-    let request_key = ctx.url_param("key")?;
-
-    let data: UpdatedRequest = ctx.body_json().await.map_err(ResponseError::bad_request)?;
-
-    let db = &ctx.state().db;
-    let env = &db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-
-    let common_store = db.common_store();
-
-    let token_key = format!("{}{}", TOKEN_PREFIX_KEY, request_key);
-
-    let mut token_config = common_store
-        .get::<Str, SerdeBincode<Token>>(&writer, &token_key)
-        .map_err(ResponseError::internal)?
-        .ok_or(ResponseError::not_found(format!(
-            "token key: {}",
-            token_key
-        )))?;
-
-    // apply the modifications
-    if let Some(description) = data.description {
-        token_config.description = description;
-    }
-
-    if let Some(acl) = data.acl {
-        token_config.acl = acl;
-    }
-
-    if let Some(indexes) = data.indexes {
-        token_config.indexes = indexes;
-    }
-
-    token_config.updated_at = Utc::now();
-
-    common_store
-        .put::<Str, SerdeBincode<Token>>(&mut writer, &token_key, &token_config)
-        .map_err(ResponseError::internal)?;
-
-    writer.commit().map_err(ResponseError::internal)?;
-
-    Ok(tide::response::json(token_config)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
-}
-
-pub async fn delete(ctx: Context<Data>) -> SResult<StatusCode> {
-    ctx.is_allowed(Admin)?;
-    let request_key = ctx.url_param("key")?;
-
-    let db = &ctx.state().db;
-    let env = &db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-
-    let common_store = db.common_store();
-
-    let token_key = format!("{}{}", TOKEN_PREFIX_KEY, request_key);
-
-    common_store
-        .delete::<Str>(&mut writer, &token_key)
-        .map_err(ResponseError::internal)?;
-
-    writer.commit().map_err(ResponseError::internal)?;
-
-    Ok(StatusCode::ACCEPTED)
-}

meilidb-http/src/routes/mod.rs

@@ -1,111 +0,0 @@
-use crate::data::Data;
-
-pub mod document;
-pub mod health;
-pub mod index;
-pub mod key;
-pub mod search;
-pub mod setting;
-pub mod stats;
-pub mod stop_words;
-pub mod synonym;
-
-pub fn load_routes(app: &mut tide::App<Data>) {
-    app.at("").nest(|router| {
-        router.at("/indexes").nest(|router| {
-            router.at("/").get(index::list_indexes);
-
-            router.at("/search").post(search::search_multi_index);
-
-            router.at("/:index").nest(|router| {
-                router.at("/search").get(search::search_with_url_query);
-
-                router.at("/updates").nest(|router| {
-                    router.at("/").get(index::get_all_updates_status);
-
-                    router.at("/:update_id").get(index::get_update_status);
-                });
-
-                router
-                    .at("/")
-                    .get(index::get_index_schema)
-                    .post(index::create_index)
-                    .put(index::update_schema)
-                    .delete(index::delete_index);
-
-                router.at("/documents").nest(|router| {
-                    router
-                        .at("/")
-                        .get(document::browse_documents)
-                        .post(document::add_or_update_multiple_documents)
-                        .delete(document::clear_all_documents);
-
-                    router.at("/:identifier").nest(|router| {
-                        router
-                            .at("/")
-                            .get(document::get_document)
-                            .delete(document::delete_document);
-                    });
-
-                    router
-                        .at("/delete")
-                        .post(document::delete_multiple_documents);
-                });
-
-                router.at("/synonym").nest(|router| {
-                    router.at("/").get(synonym::list).post(synonym::create);
-
-                    router
-                        .at("/:synonym")
-                        .get(synonym::get)
-                        .put(synonym::update)
-                        .delete(synonym::delete);
-
-                    router.at("/batch").post(synonym::batch_write);
-                    router.at("/clear").post(synonym::clear);
-                });
-
-                router.at("/stop-words").nest(|router| {
-                    router
-                        .at("/")
-                        .get(stop_words::list)
-                        .put(stop_words::add)
-                        .delete(stop_words::delete);
-                });
-
-                router
-                    .at("/settings")
-                    .get(setting::get)
-                    .post(setting::update);
-            });
-        });
-
-        router.at("/keys").nest(|router| {
-            router.at("/").get(key::list).post(key::create);
-
-            router
-                .at("/:key")
-                .get(key::get)
-                .put(key::update)
-                .delete(key::delete);
-        });
-    });
-
-    // Private
-    app.at("").nest(|router| {
-        router
-            .at("/health")
-            .get(health::get_health)
-            .post(health::set_healthy)
-            .put(health::change_healthyness)
-            .delete(health::set_unhealthy);
-
-        router.at("/stats").get(stats::get_stats);
-        router.at("/stats/:index").get(stats::index_stat);
-        router.at("/version").get(stats::get_version);
-        router.at("/sys-info").get(stats::get_sys_info);
-        router
-            .at("/sys-info/pretty")
-            .get(stats::get_sys_info_pretty);
-    });
-}

meilidb-http/src/routes/search.rs

@@ -1,231 +0,0 @@
-use std::collections::HashMap;
-use std::collections::HashSet;
-use std::time::Duration;
-
-use meilidb_core::Index;
-use rayon::iter::{IntoParallelIterator, ParallelIterator};
-use serde::{Deserialize, Serialize};
-use tide::querystring::ContextExt as QSContextExt;
-use tide::{Context, Response};
-
-use crate::error::{ResponseError, SResult};
-use crate::helpers::meilidb::{Error, IndexSearchExt, SearchHit};
-use crate::helpers::tide::ContextExt;
-use crate::Data;
-
-#[derive(Deserialize)]
-#[serde(rename_all = "camelCase", deny_unknown_fields)]
-struct SearchQuery {
-    q: String,
-    offset: Option<usize>,
-    limit: Option<usize>,
-    attributes_to_retrieve: Option<String>,
-    attributes_to_search_in: Option<String>,
-    attributes_to_crop: Option<String>,
-    crop_length: Option<usize>,
-    attributes_to_highlight: Option<String>,
-    filters: Option<String>,
-    timeout_ms: Option<u64>,
-    matches: Option<bool>,
-}
-
-pub async fn search_with_url_query(ctx: Context<Data>) -> SResult<Response> {
-    // ctx.is_allowed(DocumentsRead)?;
-
-    let index = ctx.index()?;
-    let env = &ctx.state().db.env;
-    let reader = env.read_txn().map_err(ResponseError::internal)?;
-
-    let query: SearchQuery = ctx
-        .url_query()
-        .map_err(|_| ResponseError::bad_request("invalid query parameter"))?;
-
-    let mut search_builder = index.new_search(query.q.clone());
-
-    if let Some(offset) = query.offset {
-        search_builder.offset(offset);
-    }
-    if let Some(limit) = query.limit {
-        search_builder.limit(limit);
-    }
-
-    if let Some(attributes_to_retrieve) = query.attributes_to_retrieve {
-        for attr in attributes_to_retrieve.split(',') {
-            search_builder.add_retrievable_field(attr.to_string());
-        }
-    }
-    if let Some(attributes_to_search_in) = query.attributes_to_search_in {
-        for attr in attributes_to_search_in.split(',') {
-            search_builder.add_retrievable_field(attr.to_string());
-        }
-    }
-    if let Some(attributes_to_crop) = query.attributes_to_crop {
-        let crop_length = query.crop_length.unwrap_or(200);
-        let attributes_to_crop = attributes_to_crop
-            .split(',')
-            .map(|r| (r.to_string(), crop_length))
-            .collect();
-        search_builder.attributes_to_crop(attributes_to_crop);
-    }
-
-    if let Some(attributes_to_highlight) = query.attributes_to_highlight {
-        let attributes_to_highlight = attributes_to_highlight
-            .split(',')
-            .map(ToString::to_string)
-            .collect();
-        search_builder.attributes_to_highlight(attributes_to_highlight);
-    }
-
-    if let Some(filters) = query.filters {
-        search_builder.filters(filters);
-    }
-
-    if let Some(timeout_ms) = query.timeout_ms {
-        search_builder.timeout(Duration::from_millis(timeout_ms));
-    }
-
-    if let Some(matches) = query.matches {
-        if matches {
-            search_builder.get_matches();
-        }
-    }
-
-    let response = match search_builder.search(&reader) {
-        Ok(response) => response,
-        Err(Error::Internal(message)) => return Err(ResponseError::Internal(message)),
-        Err(others) => return Err(ResponseError::bad_request(others)),
-    };
-
-    Ok(tide::response::json(response))
-}
-
-#[derive(Clone, Deserialize)]
-#[serde(rename_all = "camelCase", deny_unknown_fields)]
-struct SearchMultiBody {
-    indexes: HashSet<String>,
-    query: String,
-    offset: Option<usize>,
-    limit: Option<usize>,
-    attributes_to_retrieve: Option<HashSet<String>>,
-    attributes_to_search_in: Option<HashSet<String>>,
-    attributes_to_crop: Option<HashMap<String, usize>>,
-    attributes_to_highlight: Option<HashSet<String>>,
-    filters: Option<String>,
-    timeout_ms: Option<u64>,
-    matches: Option<bool>,
-}
-
-#[derive(Debug, Clone, Serialize)]
-#[serde(rename_all = "camelCase")]
-struct SearchMultiBodyResponse {
-    hits: HashMap<String, Vec<SearchHit>>,
-    offset: usize,
-    hits_per_page: usize,
-    processing_time_ms: usize,
-    query: String,
-}
-
-pub async fn search_multi_index(mut ctx: Context<Data>) -> SResult<Response> {
-    // ctx.is_allowed(DocumentsRead)?;
-    let body = ctx
-        .body_json::<SearchMultiBody>()
-        .await
-        .map_err(ResponseError::bad_request)?;
-
-    let mut index_list = body.clone().indexes;
-
-    for index in index_list.clone() {
-        if index == "*" {
-            index_list = ctx
-                .state()
-                .db
-                .indexes_names()
-                .map_err(ResponseError::internal)?
-                .into_iter()
-                .collect();
-        }
-    }
-
-    let mut offset = 0;
-    let mut count = 20;
-
-    if let Some(body_offset) = body.offset {
-        if let Some(limit) = body.limit {
-            offset = body_offset;
-            count = limit;
-        }
-    }
-
-    let offset = offset;
-    let count = count;
-    let db = &ctx.state().db;
-    let par_body = body.clone();
-    let responses_per_index: Vec<SResult<_>> = index_list
-        .into_par_iter()
-        .map(move |index_name| {
-            let index: Index = db
-                .open_index(&index_name)
-                .ok_or(ResponseError::index_not_found(&index_name))?;
-
-            let mut search_builder = index.new_search(par_body.query.clone());
-
-            search_builder.offset(offset);
-            search_builder.limit(count);
-
-            if let Some(attributes_to_retrieve) = par_body.attributes_to_retrieve.clone() {
-                search_builder.attributes_to_retrieve(attributes_to_retrieve);
-            }
-            if let Some(attributes_to_search_in) = par_body.attributes_to_search_in.clone() {
-                search_builder.attributes_to_search_in(attributes_to_search_in);
-            }
-            if let Some(attributes_to_crop) = par_body.attributes_to_crop.clone() {
-                search_builder.attributes_to_crop(attributes_to_crop);
-            }
-            if let Some(attributes_to_highlight) = par_body.attributes_to_highlight.clone() {
-                search_builder.attributes_to_highlight(attributes_to_highlight);
-            }
-            if let Some(filters) = par_body.filters.clone() {
-                search_builder.filters(filters);
-            }
-            if let Some(timeout_ms) = par_body.timeout_ms {
-                search_builder.timeout(Duration::from_secs(timeout_ms));
-            }
-            if let Some(matches) = par_body.matches {
-                if matches {
-                    search_builder.get_matches();
-                }
-            }
-
-            let env = &db.env;
-            let reader = env.read_txn().map_err(ResponseError::internal)?;
-
-            let response = search_builder
-                .search(&reader)
-                .map_err(ResponseError::internal)?;
-            Ok((index_name, response))
-        })
-        .collect();
-
-    let mut hits_map = HashMap::new();
-
-    let mut max_query_time = 0;
-
-    for response in responses_per_index {
-        if let Ok((index_name, response)) = response {
-            if response.processing_time_ms > max_query_time {
-                max_query_time = response.processing_time_ms;
-            }
-            hits_map.insert(index_name, response.hits);
-        }
-    }
-
-    let response = SearchMultiBodyResponse {
-        hits: hits_map,
-        offset,
-        hits_per_page: count,
-        processing_time_ms: max_query_time,
-        query: body.query,
-    };
-
-    Ok(tide::response::json(response))
-}

meilidb-http/src/routes/setting.rs

@@ -1,93 +0,0 @@
-use std::collections::{HashMap, HashSet};
-
-use http::StatusCode;
-use serde::{Deserialize, Serialize};
-use tide::response::IntoResponse;
-use tide::{Context, Response};
-
-use crate::error::{ResponseError, SResult};
-use crate::helpers::tide::ContextExt;
-use crate::models::token::ACL::*;
-use crate::routes::document::IndexUpdateResponse;
-use crate::Data;
-
-#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase", deny_unknown_fields)]
-pub struct SettingBody {
-    pub stop_words: Option<StopWords>,
-    pub ranking_order: Option<RankingOrder>,
-    pub distinct_field: Option<DistinctField>,
-    pub ranking_rules: Option<RankingRules>,
-}
-
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "lowercase")]
-pub enum RankingOrdering {
-    Asc,
-    Dsc,
-}
-
-pub type StopWords = HashSet<String>;
-pub type RankingOrder = Vec<String>;
-pub type DistinctField = String;
-pub type RankingRules = HashMap<String, RankingOrdering>;
-
-pub async fn get(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(SettingsRead)?;
-    let index = ctx.index()?;
-
-    let env = &ctx.state().db.env;
-    let reader = env.read_txn().map_err(ResponseError::internal)?;
-
-    let settings = match index.main.customs(&reader).unwrap() {
-        Some(bytes) => bincode::deserialize(bytes).unwrap(),
-        None => SettingBody::default(),
-    };
-
-    Ok(tide::response::json(settings))
-}
-
-pub async fn update(mut ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(SettingsWrite)?;
-
-    let settings: SettingBody = ctx.body_json().await.map_err(ResponseError::bad_request)?;
-
-    let index = ctx.index()?;
-
-    let env = &ctx.state().db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-
-    let mut current_settings = match index.main.customs(&writer).unwrap() {
-        Some(bytes) => bincode::deserialize(bytes).unwrap(),
-        None => SettingBody::default(),
-    };
-
-    if let Some(stop_words) = settings.stop_words {
-        current_settings.stop_words = Some(stop_words);
-    }
-
-    if let Some(ranking_order) = settings.ranking_order {
-        current_settings.ranking_order = Some(ranking_order);
-    }
-
-    if let Some(distinct_field) = settings.distinct_field {
-        current_settings.distinct_field = Some(distinct_field);
-    }
-
-    if let Some(ranking_rules) = settings.ranking_rules {
-        current_settings.ranking_rules = Some(ranking_rules);
-    }
-
-    let bytes = bincode::serialize(&current_settings).unwrap();
-
-    let update_id = index
-        .customs_update(&mut writer, bytes)
-        .map_err(ResponseError::internal)?;
-
-    writer.commit().map_err(ResponseError::internal)?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
-}

meilidb-http/src/routes/stop_words.rs

@@ -1,82 +0,0 @@
-use http::StatusCode;
-use tide::response::IntoResponse;
-use tide::{Context, Response};
-
-use crate::error::{ResponseError, SResult};
-use crate::helpers::tide::ContextExt;
-use crate::models::token::ACL::*;
-use crate::routes::document::IndexUpdateResponse;
-use crate::Data;
-
-pub async fn list(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(SettingsRead)?;
-    let index = ctx.index()?;
-
-    let env = &ctx.state().db.env;
-    let reader = env.read_txn().map_err(ResponseError::internal)?;
-
-    let stop_words_fst = index
-        .main
-        .stop_words_fst(&reader)
-        .map_err(ResponseError::internal)?;
-
-    let stop_words = stop_words_fst
-        .unwrap_or_default()
-        .stream()
-        .into_strs()
-        .map_err(ResponseError::internal)?;
-
-    Ok(tide::response::json(stop_words))
-}
-
-pub async fn add(mut ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(SettingsRead)?;
-    let index = ctx.index()?;
-
-    let data: Vec<String> = ctx.body_json().await.map_err(ResponseError::bad_request)?;
-
-    let env = &ctx.state().db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-
-    let mut stop_words_addition = index.stop_words_addition();
-    for stop_word in data {
-        stop_words_addition.add_stop_word(stop_word);
-    }
-
-    let update_id = stop_words_addition
-        .finalize(&mut writer)
-        .map_err(ResponseError::internal)?;
-
-    writer.commit().map_err(ResponseError::internal)?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
-}
-
-pub async fn delete(mut ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(SettingsRead)?;
-    let index = ctx.index()?;
-
-    let data: Vec<String> = ctx.body_json().await.map_err(ResponseError::bad_request)?;
-
-    let env = &ctx.state().db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-
-    let mut stop_words_deletion = index.stop_words_deletion();
-    for stop_word in data {
-        stop_words_deletion.delete_stop_word(stop_word);
-    }
-
-    let update_id = stop_words_deletion
-        .finalize(&mut writer)
-        .map_err(ResponseError::internal)?;
-
-    writer.commit().map_err(ResponseError::internal)?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
-}

meilidb-http/src/routes/synonym.rs

@@ -1,235 +0,0 @@
-use std::collections::HashMap;
-
-use http::StatusCode;
-use serde::{Deserialize, Serialize};
-use tide::response::IntoResponse;
-use tide::{Context, Response};
-
-use crate::error::{ResponseError, SResult};
-use crate::helpers::tide::ContextExt;
-use crate::models::token::ACL::*;
-use crate::routes::document::IndexUpdateResponse;
-use crate::Data;
-
-#[derive(Clone, Serialize, Deserialize)]
-#[serde(untagged)]
-pub enum Synonym {
-    OneWay(SynonymOneWay),
-    MultiWay { synonyms: Vec<String> },
-}
-
-#[derive(Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase", deny_unknown_fields)]
-pub struct SynonymOneWay {
-    pub input: String,
-    pub synonyms: Vec<String>,
-}
-
-pub type Synonyms = Vec<Synonym>;
-
-pub async fn list(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(SettingsRead)?;
-    let index = ctx.index()?;
-
-    let env = &ctx.state().db.env;
-    let reader = env.read_txn().map_err(ResponseError::internal)?;
-
-    let synonyms_fst = index
-        .main
-        .synonyms_fst(&reader)
-        .map_err(ResponseError::internal)?;
-
-    let synonyms_fst = synonyms_fst.unwrap_or_default();
-    let synonyms_list = synonyms_fst.stream().into_strs().unwrap();
-
-    let mut response = HashMap::new();
-
-    let index_synonyms = &index.synonyms;
-
-    for synonym in synonyms_list {
-        let alternative_list = index_synonyms
-            .synonyms(&reader, synonym.as_bytes())
-            .unwrap()
-            .unwrap()
-            .stream()
-            .into_strs()
-            .unwrap();
-        response.insert(synonym, alternative_list);
-    }
-
-    Ok(tide::response::json(response))
-}
-
-pub async fn get(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(SettingsRead)?;
-    let synonym = ctx.url_param("synonym")?;
-    let index = ctx.index()?;
-
-    let env = &ctx.state().db.env;
-    let reader = env.read_txn().map_err(ResponseError::internal)?;
-
-    let synonym_list = index
-        .synonyms
-        .synonyms(&reader, synonym.as_bytes())
-        .unwrap()
-        .unwrap()
-        .stream()
-        .into_strs()
-        .unwrap();
-
-    Ok(tide::response::json(synonym_list))
-}
-
-pub async fn create(mut ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(SettingsWrite)?;
-
-    let data: Synonym = ctx.body_json().await.map_err(ResponseError::bad_request)?;
-
-    let index = ctx.index()?;
-
-    let env = &ctx.state().db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-
-    let mut synonyms_addition = index.synonyms_addition();
-
-    match data.clone() {
-        Synonym::OneWay(content) => {
-            synonyms_addition.add_synonym(content.input, content.synonyms.into_iter())
-        }
-        Synonym::MultiWay { mut synonyms } => {
-            if synonyms.len() > 1 {
-                for _ in 0..synonyms.len() {
-                    let (first, elems) = synonyms.split_first().unwrap();
-                    synonyms_addition.add_synonym(first, elems.iter());
-                    synonyms.rotate_left(1);
-                }
-            }
-        }
-    }
-
-    let update_id = synonyms_addition
-        .finalize(&mut writer)
-        .map_err(ResponseError::internal)?;
-
-    writer.commit().map_err(ResponseError::internal)?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::CREATED)
-        .into_response())
-}
-
-pub async fn update(mut ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(SettingsWrite)?;
-    let synonym = ctx.url_param("synonym")?;
-    let index = ctx.index()?;
-    let data: Vec<String> = ctx.body_json().await.map_err(ResponseError::bad_request)?;
-
-    let env = &ctx.state().db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-
-    let mut synonyms_addition = index.synonyms_addition();
-    synonyms_addition.add_synonym(synonym.clone(), data.clone().into_iter());
-    let update_id = synonyms_addition
-        .finalize(&mut writer)
-        .map_err(ResponseError::internal)?;
-
-    writer.commit().map_err(ResponseError::internal)?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
-}
-
-pub async fn delete(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(SettingsWrite)?;
-    let synonym = ctx.url_param("synonym")?;
-    let index = ctx.index()?;
-
-    let env = &ctx.state().db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-
-    let mut synonyms_deletion = index.synonyms_deletion();
-    synonyms_deletion.delete_all_alternatives_of(synonym);
-    let update_id = synonyms_deletion
-        .finalize(&mut writer)
-        .map_err(ResponseError::internal)?;
-
-    writer.commit().map_err(ResponseError::internal)?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
-}
-
-pub async fn batch_write(mut ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(SettingsWrite)?;
-
-    let data: Synonyms = ctx.body_json().await.map_err(ResponseError::bad_request)?;
-
-    let index = ctx.index()?;
-
-    let env = &ctx.state().db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-
-    let mut synonyms_addition = index.synonyms_addition();
-    for raw in data {
-        match raw {
-            Synonym::OneWay(content) => {
-                synonyms_addition.add_synonym(content.input, content.synonyms.into_iter())
-            }
-            Synonym::MultiWay { mut synonyms } => {
-                if synonyms.len() > 1 {
-                    for _ in 0..synonyms.len() {
-                        let (first, elems) = synonyms.split_first().unwrap();
-                        synonyms_addition.add_synonym(first, elems.iter());
-                        synonyms.rotate_left(1);
-                    }
-                }
-            }
-        }
-    }
-    let update_id = synonyms_addition
-        .finalize(&mut writer)
-        .map_err(ResponseError::internal)?;
-
-    writer.commit().map_err(ResponseError::internal)?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
-}
-
-pub async fn clear(ctx: Context<Data>) -> SResult<Response> {
-    ctx.is_allowed(SettingsWrite)?;
-    let index = ctx.index()?;
-
-    let env = &ctx.state().db.env;
-    let mut writer = env.write_txn().map_err(ResponseError::internal)?;
-
-    let synonyms_fst = index
-        .main
-        .synonyms_fst(&writer)
-        .map_err(ResponseError::internal)?;
-
-    let synonyms_fst = synonyms_fst.unwrap_or_default();
-    let synonyms_list = synonyms_fst.stream().into_strs().unwrap();
-
-    let mut synonyms_deletion = index.synonyms_deletion();
-    for synonym in synonyms_list {
-        synonyms_deletion.delete_all_alternatives_of(synonym);
-    }
-    let update_id = synonyms_deletion
-        .finalize(&mut writer)
-        .map_err(ResponseError::internal)?;
-
-    writer.commit().map_err(ResponseError::internal)?;
-
-    let response_body = IndexUpdateResponse { update_id };
-    Ok(tide::response::json(response_body)
-        .with_status(StatusCode::ACCEPTED)
-        .into_response())
-}

meilidb-schema/Cargo.toml

@@ -1,12 +0,0 @@
-[package]
-name = "meilidb-schema"
-version = "0.6.0"
-authors = ["Kerollmops <renault.cle@gmail.com>"]
-edition = "2018"
-
-[dependencies]
-bincode = "1.1.2"
-indexmap = { version = "1.1.0", features = ["serde-1"] }
-serde = { version = "1.0.91", features = ["derive"] }
-serde_json = { version = "1.0.39", features = ["preserve_order"] }
-toml = { version = "0.5.0", features = ["preserve_order"] }

meilidb-schema/src/lib.rs

@@ -1,450 +0,0 @@
-use std::collections::{BTreeMap, HashMap};
-use std::ops::BitOr;
-use std::sync::Arc;
-use std::{fmt, u16};
-
-use indexmap::IndexMap;
-use serde::{Deserialize, Serialize};
-
-pub const DISPLAYED: SchemaProps = SchemaProps {
-    displayed: true,
-    indexed: false,
-    ranked: false,
-};
-pub const INDEXED: SchemaProps = SchemaProps {
-    displayed: false,
-    indexed: true,
-    ranked: false,
-};
-pub const RANKED: SchemaProps = SchemaProps {
-    displayed: false,
-    indexed: false,
-    ranked: true,
-};
-
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct SchemaProps {
-    #[serde(default)]
-    pub displayed: bool,
-
-    #[serde(default)]
-    pub indexed: bool,
-
-    #[serde(default)]
-    pub ranked: bool,
-}
-
-impl SchemaProps {
-    pub fn is_displayed(self) -> bool {
-        self.displayed
-    }
-
-    pub fn is_indexed(self) -> bool {
-        self.indexed
-    }
-
-    pub fn is_ranked(self) -> bool {
-        self.ranked
-    }
-}
-
-impl BitOr for SchemaProps {
-    type Output = Self;
-
-    fn bitor(self, other: Self) -> Self::Output {
-        SchemaProps {
-            displayed: self.displayed | other.displayed,
-            indexed: self.indexed | other.indexed,
-            ranked: self.ranked | other.ranked,
-        }
-    }
-}
-
-#[derive(Serialize, Deserialize)]
-pub struct SchemaBuilder {
-    identifier: String,
-    attributes: IndexMap<String, SchemaProps>,
-}
-
-impl SchemaBuilder {
-    pub fn with_identifier<S: Into<String>>(name: S) -> SchemaBuilder {
-        SchemaBuilder {
-            identifier: name.into(),
-            attributes: IndexMap::new(),
-        }
-    }
-
-    pub fn new_attribute<S: Into<String>>(&mut self, name: S, props: SchemaProps) -> SchemaAttr {
-        let len = self.attributes.len();
-        if self.attributes.insert(name.into(), props).is_some() {
-            panic!("Field already inserted.")
-        }
-        SchemaAttr(len as u16)
-    }
-
-    pub fn build(self) -> Schema {
-        let mut attrs = HashMap::new();
-        let mut props = Vec::new();
-
-        for (i, (name, prop)) in self.attributes.into_iter().enumerate() {
-            attrs.insert(name.clone(), SchemaAttr(i as u16));
-            props.push((name, prop));
-        }
-
-        let identifier = self.identifier;
-        Schema {
-            inner: Arc::new(InnerSchema {
-                identifier,
-                attrs,
-                props,
-            }),
-        }
-    }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct Schema {
-    inner: Arc<InnerSchema>,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-struct InnerSchema {
-    identifier: String,
-    attrs: HashMap<String, SchemaAttr>,
-    props: Vec<(String, SchemaProps)>,
-}
-
-impl Schema {
-    fn to_builder(&self) -> SchemaBuilder {
-        let identifier = self.inner.identifier.clone();
-        let attributes = self.attributes_ordered();
-        SchemaBuilder {
-            identifier,
-            attributes,
-        }
-    }
-
-    fn attributes_ordered(&self) -> IndexMap<String, SchemaProps> {
-        let mut ordered = BTreeMap::new();
-        for (name, attr) in &self.inner.attrs {
-            let (_, props) = self.inner.props[attr.0 as usize];
-            ordered.insert(attr.0, (name, props));
-        }
-
-        let mut attributes = IndexMap::with_capacity(ordered.len());
-        for (_, (name, props)) in ordered {
-            attributes.insert(name.clone(), props);
-        }
-
-        attributes
-    }
-
-    pub fn props(&self, attr: SchemaAttr) -> SchemaProps {
-        let (_, props) = self.inner.props[attr.0 as usize];
-        props
-    }
-
-    pub fn identifier_name(&self) -> &str {
-        &self.inner.identifier
-    }
-
-    pub fn attribute<S: AsRef<str>>(&self, name: S) -> Option<SchemaAttr> {
-        self.inner.attrs.get(name.as_ref()).cloned()
-    }
-
-    pub fn attribute_name(&self, attr: SchemaAttr) -> &str {
-        let (name, _) = &self.inner.props[attr.0 as usize];
-        name
-    }
-
-    pub fn iter<'a>(&'a self) -> impl Iterator<Item = (&str, SchemaAttr, SchemaProps)> + 'a {
-        self.inner.props.iter().map(move |(name, prop)| {
-            let attr = self.inner.attrs.get(name).unwrap();
-            (name.as_str(), *attr, *prop)
-        })
-    }
-}
-
-impl Serialize for Schema {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::ser::Serializer,
-    {
-        self.to_builder().serialize(serializer)
-    }
-}
-
-impl<'de> Deserialize<'de> for Schema {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: serde::de::Deserializer<'de>,
-    {
-        let builder = SchemaBuilder::deserialize(deserializer)?;
-        Ok(builder.build())
-    }
-}
-
-#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Hash)]
-pub struct SchemaAttr(pub u16);
-
-impl SchemaAttr {
-    pub const fn new(value: u16) -> SchemaAttr {
-        SchemaAttr(value)
-    }
-
-    pub const fn min() -> SchemaAttr {
-        SchemaAttr(u16::min_value())
-    }
-
-    pub const fn max() -> SchemaAttr {
-        SchemaAttr(u16::max_value())
-    }
-
-    pub fn next(self) -> Option<SchemaAttr> {
-        self.0.checked_add(1).map(SchemaAttr)
-    }
-
-    pub fn prev(self) -> Option<SchemaAttr> {
-        self.0.checked_sub(1).map(SchemaAttr)
-    }
-}
-
-impl fmt::Display for SchemaAttr {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        self.0.fmt(f)
-    }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum Diff {
-    IdentChange {
-        old: String,
-        new: String,
-    },
-    AttrMove {
-        name: String,
-        old: usize,
-        new: usize,
-    },
-    AttrPropsChange {
-        name: String,
-        old: SchemaProps,
-        new: SchemaProps,
-    },
-    NewAttr {
-        name: String,
-        pos: usize,
-        props: SchemaProps,
-    },
-    RemovedAttr {
-        name: String,
-    },
-}
-
-pub fn diff(old: &Schema, new: &Schema) -> Vec<Diff> {
-    use Diff::{AttrMove, AttrPropsChange, IdentChange, NewAttr, RemovedAttr};
-
-    let mut differences = Vec::new();
-    let old = old.to_builder();
-    let new = new.to_builder();
-
-    // check if the old identifier differs from the new one
-    if old.identifier != new.identifier {
-        let old = old.identifier;
-        let new = new.identifier;
-        differences.push(IdentChange { old, new });
-    }
-
-    // compare all old attributes positions
-    // and properties with the new ones
-    for (pos, (name, props)) in old.attributes.iter().enumerate() {
-        match new.attributes.get_full(name) {
-            Some((npos, _, nprops)) => {
-                if pos != npos {
-                    let name = name.clone();
-                    differences.push(AttrMove {
-                        name,
-                        old: pos,
-                        new: npos,
-                    });
-                }
-                if props != nprops {
-                    let name = name.clone();
-                    differences.push(AttrPropsChange {
-                        name,
-                        old: *props,
-                        new: *nprops,
-                    });
-                }
-            }
-            None => differences.push(RemovedAttr { name: name.clone() }),
-        }
-    }
-
-    // retrieve all attributes that
-    // were not present in the old schema
-    for (pos, (name, props)) in new.attributes.iter().enumerate() {
-        if !old.attributes.contains_key(name) {
-            let name = name.clone();
-            differences.push(NewAttr {
-                name,
-                pos,
-                props: *props,
-            });
-        }
-    }
-
-    differences
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::error::Error;
-
-    #[test]
-    fn difference() {
-        use Diff::{AttrMove, AttrPropsChange, IdentChange, NewAttr, RemovedAttr};
-
-        let mut builder = SchemaBuilder::with_identifier("id");
-        builder.new_attribute("alpha", DISPLAYED);
-        builder.new_attribute("beta", DISPLAYED | INDEXED);
-        builder.new_attribute("gamma", INDEXED);
-        builder.new_attribute("omega", INDEXED);
-        let old = builder.build();
-
-        let mut builder = SchemaBuilder::with_identifier("kiki");
-        builder.new_attribute("beta", DISPLAYED | INDEXED);
-        builder.new_attribute("alpha", DISPLAYED | INDEXED);
-        builder.new_attribute("delta", RANKED);
-        builder.new_attribute("gamma", DISPLAYED);
-        let new = builder.build();
-
-        let differences = diff(&old, &new);
-        let expected = &[
-            IdentChange {
-                old: format!("id"),
-                new: format!("kiki"),
-            },
-            AttrMove {
-                name: format!("alpha"),
-                old: 0,
-                new: 1,
-            },
-            AttrPropsChange {
-                name: format!("alpha"),
-                old: DISPLAYED,
-                new: DISPLAYED | INDEXED,
-            },
-            AttrMove {
-                name: format!("beta"),
-                old: 1,
-                new: 0,
-            },
-            AttrMove {
-                name: format!("gamma"),
-                old: 2,
-                new: 3,
-            },
-            AttrPropsChange {
-                name: format!("gamma"),
-                old: INDEXED,
-                new: DISPLAYED,
-            },
-            RemovedAttr {
-                name: format!("omega"),
-            },
-            NewAttr {
-                name: format!("delta"),
-                pos: 2,
-                props: RANKED,
-            },
-        ];
-
-        assert_eq!(&differences, expected)
-    }
-
-    #[test]
-    fn serialize_deserialize() -> bincode::Result<()> {
-        let mut builder = SchemaBuilder::with_identifier("id");
-        builder.new_attribute("alpha", DISPLAYED);
-        builder.new_attribute("beta", DISPLAYED | INDEXED);
-        builder.new_attribute("gamma", INDEXED);
-        let schema = builder.build();
-
-        let mut buffer = Vec::new();
-        bincode::serialize_into(&mut buffer, &schema)?;
-        let schema2 = bincode::deserialize_from(buffer.as_slice())?;
-
-        assert_eq!(schema, schema2);
-
-        Ok(())
-    }
-
-    #[test]
-    fn serialize_deserialize_toml() -> Result<(), Box<dyn Error>> {
-        let mut builder = SchemaBuilder::with_identifier("id");
-        builder.new_attribute("alpha", DISPLAYED);
-        builder.new_attribute("beta", DISPLAYED | INDEXED);
-        builder.new_attribute("gamma", INDEXED);
-        let schema = builder.build();
-
-        let buffer = toml::to_vec(&schema)?;
-        let schema2 = toml::from_slice(buffer.as_slice())?;
-
-        assert_eq!(schema, schema2);
-
-        let data = r#"
-            identifier = "id"
-
-            [attributes."alpha"]
-            displayed = true
-
-            [attributes."beta"]
-            displayed = true
-            indexed = true
-
-            [attributes."gamma"]
-            indexed = true
-        "#;
-        let schema2 = toml::from_str(data)?;
-        assert_eq!(schema, schema2);
-
-        Ok(())
-    }
-
-    #[test]
-    fn serialize_deserialize_json() -> Result<(), Box<dyn Error>> {
-        let mut builder = SchemaBuilder::with_identifier("id");
-        builder.new_attribute("alpha", DISPLAYED);
-        builder.new_attribute("beta", DISPLAYED | INDEXED);
-        builder.new_attribute("gamma", INDEXED);
-        let schema = builder.build();
-
-        let buffer = serde_json::to_vec(&schema)?;
-        let schema2 = serde_json::from_slice(buffer.as_slice())?;
-
-        assert_eq!(schema, schema2);
-
-        let data = r#"
-            {
-                "identifier": "id",
-                "attributes": {
-                    "alpha": {
-                        "displayed": true
-                    },
-                    "beta": {
-                        "displayed": true,
-                        "indexed": true
-                    },
-                    "gamma": {
-                        "indexed": true
-                    }
-                }
-            }"#;
-        let schema2 = serde_json::from_str(data)?;
-        assert_eq!(schema, schema2);
-
-        Ok(())
-    }
-}

meilidb-tokenizer/Cargo.toml

@@ -1,9 +0,0 @@
-[package]
-name = "meilidb-tokenizer"
-version = "0.6.1"
-authors = ["Kerollmops <renault.cle@gmail.com>"]
-edition = "2018"
-
-[dependencies]
-deunicode = "1.0.0"
-slice-group-by = "0.2.4"

meilisearch-core/Cargo.toml

@@ -0,0 +1,57 @@
+[package]
+name = "meilisearch-core"
+version = "0.11.0"
+license = "MIT"
+authors = ["Kerollmops <clement@meilisearch.com>"]
+edition = "2018"
+
+[dependencies]
+arc-swap = "0.4.5"
+bincode = "1.2.1"
+byteorder = "1.3.4"
+chrono = { version = "0.4.11", features = ["serde"] }
+compact_arena = "0.4.0"
+cow-utils = "0.1.2"
+crossbeam-channel = "0.4.2"
+deunicode = "1.1.0"
+either = "1.5.3"
+env_logger = "0.7.1"
+fst = "0.4.4"
+hashbrown = { version = "0.7.1", features = ["serde"] }
+heed = "0.8.0"
+indexmap = { version = "1.3.2", features = ["serde-1"] }
+intervaltree = "0.2.5"
+itertools = "0.9.0"
+levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }
+log = "0.4.8"
+meilisearch-error = { path = "../meilisearch-error", version = "0.11.0" }
+meilisearch-schema = { path = "../meilisearch-schema", version = "0.11.0" }
+meilisearch-tokenizer = { path = "../meilisearch-tokenizer", version = "0.11.0" }
+meilisearch-types = { path = "../meilisearch-types", version = "0.11.0" }
+once_cell = "1.3.1"
+ordered-float = { version = "1.0.2", features = ["serde"] }
+pest = { git = "https://github.com/MarinPostma/pest.git", tag = "meilisearch-patch1" }
+pest_derive = "2.0"
+regex = "1.3.6"
+sdset = "0.4.0"
+serde = { version = "1.0.105", features = ["derive"] }
+serde_json = { version = "1.0.50", features = ["preserve_order"] }
+slice-group-by = "0.2.6"
+unicase = "2.6.0"
+zerocopy = "0.3.0"
+
+[dev-dependencies]
+assert_matches = "1.3.0"
+criterion = "0.3.1"
+csv = "1.1.3"
+rustyline = { version = "6.0.0", default-features = false }
+structopt = "0.3.12"
+tempfile = "3.1.0"
+termcolor = "1.1.0"
+
+[target.'cfg(unix)'.dev-dependencies]
+jemallocator = "0.3.2"
+
+[[bench]]
+name = "search_benchmark"
+harness = false

meilisearch-core/benches/search_benchmark.rs

@@ -0,0 +1,104 @@
+#[cfg(test)]
+#[macro_use]
+extern crate assert_matches;
+
+use std::sync::mpsc;
+use std::path::Path;
+use std::fs::File;
+use std::io::BufReader;
+use std::iter;
+
+use meilisearch_core::{Database, DatabaseOptions};
+use meilisearch_core::{ProcessedUpdateResult, UpdateStatus};
+use meilisearch_core::settings::{Settings, SettingsUpdate};
+use meilisearch_schema::Schema;
+use serde_json::Value;
+
+use criterion::{criterion_group, criterion_main, Criterion, BenchmarkId};
+
+fn prepare_database(path: &Path) -> Database {
+    let database = Database::open_or_create(path, DatabaseOptions::default()).unwrap();
+    let db = &database;
+
+    let (sender, receiver) = mpsc::sync_channel(100);
+    let update_fn = move |_name: &str, update: ProcessedUpdateResult| {
+        sender.send(update.update_id).unwrap()
+    };
+    let index = database.create_index("bench").unwrap();
+
+    database.set_update_callback(Box::new(update_fn));
+
+    let mut writer = db.main_write_txn().unwrap();
+    index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap();
+    writer.commit().unwrap();
+
+    let settings_update: SettingsUpdate = {
+        let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../datasets/movies/settings.json");
+        let file = File::open(path).unwrap();
+        let reader = BufReader::new(file);
+        let settings: Settings = serde_json::from_reader(reader).unwrap();
+        settings.into_update().unwrap()
+    };
+
+    let mut update_writer = db.update_write_txn().unwrap();
+    let _update_id = index.settings_update(&mut update_writer, settings_update).unwrap();
+    update_writer.commit().unwrap();
+
+    let mut additions = index.documents_addition();
+
+    let json: Value = {
+        let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../datasets/movies/movies.json");
+        let movies_file = File::open(path).expect("find movies");
+        serde_json::from_reader(movies_file).unwrap()
+    };
+
+    let documents = json.as_array().unwrap();
+
+    for document in documents {
+        additions.update_document(document);
+    }
+
+    let mut update_writer = db.update_write_txn().unwrap();
+    let update_id = additions.finalize(&mut update_writer).unwrap();
+    update_writer.commit().unwrap();
+
+    // block until the transaction is processed
+    let _ = receiver.into_iter().find(|id| *id == update_id);
+
+    let update_reader = db.update_read_txn().unwrap();
+    let result = index.update_status(&update_reader, update_id).unwrap();
+    assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
+
+    database
+}
+
+pub fn criterion_benchmark(c: &mut Criterion) {
+    let dir = tempfile::tempdir().unwrap();
+    let database = prepare_database(dir.path());
+
+    let reader = database.main_read_txn().unwrap();
+    let index = database.open_index("bench").unwrap();
+
+    let mut count = 0;
+    let query = "I love paris ";
+
+    let iter = iter::from_fn(|| {
+        count += 1;
+        query.get(0..count)
+    });
+
+    let mut group = c.benchmark_group("searching in movies (19654 docs)");
+    group.sample_size(10);
+
+    for query in iter {
+        let bench_name = BenchmarkId::from_parameter(format!("{:?}", query));
+        group.bench_with_input(bench_name, &query, |b, query| b.iter(|| {
+            let builder = index.query_builder();
+            builder.query(&reader, query, 0..20).unwrap();
+        }));
+    }
+    group.finish();
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

meilisearch-core/examples/from_file.rs

@@ -1,7 +1,7 @@
-use std::collections::btree_map::{BTreeMap, Entry};
 use std::collections::HashSet;
+use std::collections::btree_map::{BTreeMap, Entry};
 use std::error::Error;
-use std::io::Write;
+use std::io::{Read, Write};
 use std::iter::FromIterator;
 use std::path::{Path, PathBuf};
 use std::time::{Duration, Instant};
@@ -12,8 +12,13 @@ use serde::{Deserialize, Serialize};
 use structopt::StructOpt;
 use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
 
-use meilidb_core::{Database, Highlight, ProcessedUpdateResult};
-use meilidb_schema::SchemaAttr;
+use meilisearch_core::{Database, DatabaseOptions, Highlight, ProcessedUpdateResult};
+use meilisearch_core::settings::Settings;
+use meilisearch_schema::FieldId;
+
+#[cfg(target_os = "linux")]
+#[global_allocator]
+static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
 
 #[derive(Debug, StructOpt)]
 struct IndexCommand {
@@ -22,15 +27,15 @@ struct IndexCommand {
     database_path: PathBuf,
 
     #[structopt(long, default_value = "default")]
-    index_name: String,
+    index_uid: String,
 
-    /// The csv file to index.
+    /// The csv file path to index, you can also use `-` to specify the standard input.
     #[structopt(parse(from_os_str))]
     csv_data_path: PathBuf,
 
-    /// The path to the schema.
+    /// The path to the settings.
     #[structopt(long, parse(from_os_str))]
-    schema: PathBuf,
+    settings: PathBuf,
 
     #[structopt(long)]
     update_group_size: Option<usize>,
@@ -46,7 +51,7 @@ struct SearchCommand {
     database_path: PathBuf,
 
     #[structopt(long, default_value = "default")]
-    index_name: String,
+    index_uid: String,
 
     /// Timeout after which the search will return results.
     #[structopt(long)]
@@ -76,7 +81,7 @@ struct ShowUpdatesCommand {
     database_path: PathBuf,
 
     #[structopt(long, default_value = "default")]
-    index_name: String,
+    index_uid: String,
 }
 
 #[derive(Debug, StructOpt)]
@@ -104,37 +109,32 @@ fn index_command(command: IndexCommand, database: Database) -> Result<(), Box<dy
     let start = Instant::now();
 
     let (sender, receiver) = mpsc::sync_channel(100);
-    let update_fn = move |update: ProcessedUpdateResult| sender.send(update.update_id).unwrap();
-    let index = match database.open_index(&command.index_name) {
+    let update_fn =
+        move |_name: &str, update: ProcessedUpdateResult| sender.send(update.update_id).unwrap();
+    let index = match database.open_index(&command.index_uid) {
         Some(index) => index,
-        None => database.create_index(&command.index_name).unwrap(),
+        None => database.create_index(&command.index_uid).unwrap(),
     };
 
-    let done = database.set_update_callback(&command.index_name, Box::new(update_fn));
-    assert!(done, "could not set the index update function");
+    database.set_update_callback(Box::new(update_fn));
 
-    let env = &database.env;
+    let db = &database;
 
-    let schema = {
-        let string = fs::read_to_string(&command.schema)?;
-        toml::from_str(&string).unwrap()
+    let settings = {
+        let string = fs::read_to_string(&command.settings)?;
+        let settings: Settings = serde_json::from_str(&string).unwrap();
+        settings.into_update().unwrap()
     };
 
-    let mut writer = env.write_txn().unwrap();
-    match index.main.schema(&writer)? {
-        Some(current_schema) => {
-            if current_schema != schema {
-                return Err(meilidb_core::Error::SchemaDiffer.into());
-            }
-            writer.abort();
-        }
-        None => {
-            index.schema_update(&mut writer, schema)?;
-            writer.commit().unwrap();
-        }
-    }
+    db.update_write(|w| index.settings_update(w, settings))?;
+
+    let mut rdr = if command.csv_data_path.as_os_str() == "-" {
+        csv::Reader::from_reader(Box::new(io::stdin()) as Box<dyn Read>)
+    } else {
+        let file = std::fs::File::open(command.csv_data_path)?;
+        csv::Reader::from_reader(Box::new(file) as Box<dyn Read>)
+    };
 
-    let mut rdr = csv::Reader::from_path(command.csv_data_path)?;
     let mut raw_record = csv::StringRecord::new();
     let headers = rdr.headers()?.clone();
 
@@ -173,10 +173,9 @@ fn index_command(command: IndexCommand, database: Database) -> Result<(), Box<dy
 
         println!();
 
-        let mut writer = env.write_txn().unwrap();
+        let update_id = db.update_write(|w| additions.finalize(w))?;
+
         println!("committing update...");
-        let update_id = additions.finalize(&mut writer)?;
-        writer.commit().unwrap();
         max_update_id = max_update_id.max(update_id);
         println!("committed update {}", update_id);
     }
@@ -195,8 +194,9 @@ fn index_command(command: IndexCommand, database: Database) -> Result<(), Box<dy
     );
 
     if let Some(path) = command.compact_to_path {
+        fs::create_dir_all(&path)?;
         let start = Instant::now();
-        let _file = database.copy_and_compact_to_path(&path)?;
+        let _file = database.copy_and_compact_to_path(path.join("data.mdb"))?;
         println!(
             "database compacted in {:.2?} at: {:?}",
             start.elapsed(),
@@ -217,7 +217,11 @@ fn display_highlights(text: &str, ranges: &[usize]) -> io::Result<()> {
             _ => unreachable!(),
         };
         if highlighted {
-            stdout.set_color(ColorSpec::new().set_fg(Some(Color::Yellow)))?;
+            stdout.set_color(
+                ColorSpec::new()
+                    .set_fg(Some(Color::Yellow))
+                    .set_underline(true),
+            )?;
         }
         write!(&mut stdout, "{}", &text[start..end])?;
         stdout.reset()?;
@@ -311,16 +315,16 @@ fn crop_text(
 }
 
 fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<dyn Error>> {
-    let env = &database.env;
+    let db = &database;
     let index = database
-        .open_index(&command.index_name)
+        .open_index(&command.index_uid)
         .expect("Could not find index");
 
-    let reader = env.read_txn().unwrap();
+    let reader = db.main_read_txn().unwrap();
     let schema = index.main.schema(&reader)?;
-    reader.abort();
+    reader.abort().unwrap();
 
-    let schema = schema.ok_or(meilidb_core::Error::SchemaMissing)?;
+    let schema = schema.ok_or(meilisearch_core::Error::SchemaMissing)?;
 
     let fields = command.displayed_fields.iter().map(String::as_str);
     let fields = HashSet::from_iter(fields);
@@ -334,7 +338,7 @@ fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<
             Ok(query) => {
                 let start_total = Instant::now();
 
-                let reader = env.read_txn().unwrap();
+                let reader = db.main_read_txn().unwrap();
                 let ref_index = &index;
                 let ref_reader = &reader;
 
@@ -352,7 +356,7 @@ fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<
                     };
 
                     let attr = schema
-                        .attribute(&filter)
+                        .id(filter)
                         .expect("Could not find filtered attribute");
 
                     builder.with_filter(move |document_id| {
@@ -364,12 +368,12 @@ fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<
                     });
                 }
 
-                let documents = builder.query(ref_reader, &query, 0..command.number_results)?;
+                let result = builder.query(ref_reader, &query, 0..command.number_results)?;
 
                 let mut retrieve_duration = Duration::default();
 
-                let number_of_documents = documents.len();
-                for mut doc in documents {
+                let number_of_documents = result.documents.len();
+                for mut doc in result.documents {
                     doc.highlights
                         .sort_unstable_by_key(|m| (m.char_index, m.char_length));
 
@@ -383,11 +387,11 @@ fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<
                             for (name, text) in document.0 {
                                 print!("{}: ", name);
 
-                                let attr = schema.attribute(&name).unwrap();
+                                let attr = schema.id(&name).unwrap();
                                 let highlights = doc
                                     .highlights
                                     .iter()
-                                    .filter(|m| SchemaAttr::new(m.attribute) == attr)
+                                    .filter(|m| FieldId::new(m.attribute) == attr)
                                     .cloned();
                                 let (text, highlights) =
                                     crop_text(&text, highlights, command.char_context);
@@ -402,8 +406,8 @@ fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<
 
                     let mut matching_attributes = HashSet::new();
                     for highlight in doc.highlights {
-                        let attr = SchemaAttr::new(highlight.attribute);
-                        let name = schema.attribute_name(attr);
+                        let attr = FieldId::new(highlight.attribute);
+                        let name = schema.name(attr);
                         matching_attributes.insert(name);
                     }
 
@@ -439,15 +443,15 @@ fn show_updates_command(
     command: ShowUpdatesCommand,
     database: Database,
 ) -> Result<(), Box<dyn Error>> {
-    let env = &database.env;
+    let db = &database;
     let index = database
-        .open_index(&command.index_name)
+        .open_index(&command.index_uid)
         .expect("Could not find index");
 
-    let reader = env.read_txn().unwrap();
+    let reader = db.update_read_txn().unwrap();
     let updates = index.all_updates_status(&reader)?;
     println!("{:#?}", updates);
-    reader.abort();
+    reader.abort().unwrap();
 
     Ok(())
 }
@@ -456,7 +460,7 @@ fn main() -> Result<(), Box<dyn Error>> {
     env_logger::init();
 
     let opt = Command::from_args();
-    let database = Database::open_or_create(opt.path())?;
+    let database = Database::open_or_create(opt.path(), DatabaseOptions::default())?;
 
     match opt {
         Command::Index(command) => index_command(command, database),

meilisearch-core/src/automaton/dfa.rs

@@ -46,3 +46,8 @@ pub fn build_prefix_dfa(query: &str) -> DFA {
 pub fn build_dfa(query: &str) -> DFA {
     build_dfa_with_setting(query, PrefixSetting::NoPrefix)
 }
+
+pub fn build_exact_dfa(query: &str) -> DFA {
+    let builder = LEVDIST0.get_or_init(|| LevBuilder::new(0, true));
+    builder.build_dfa(query)
+}

meilisearch-core/src/automaton/mod.rs

@@ -0,0 +1,15 @@
+mod dfa;
+
+use meilisearch_tokenizer::is_cjk;
+
+pub use self::dfa::{build_dfa, build_prefix_dfa, build_exact_dfa};
+
+pub fn normalize_str(string: &str) -> String {
+    let mut string = string.to_lowercase();
+
+    if !string.contains(is_cjk) {
+        string = deunicode::deunicode_with_tofu(&string, "");
+    }
+
+    string
+}

meilisearch-core/src/bucket_sort.rs

@@ -0,0 +1,621 @@
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::mem;
+use std::ops::Deref;
+use std::ops::Range;
+use std::rc::Rc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::time::Instant;
+use std::fmt;
+
+use compact_arena::{SmallArena, Idx32, mk_arena};
+use log::debug;
+use meilisearch_types::DocIndex;
+use sdset::{Set, SetBuf, exponential_search, SetOperation, Counter, duo::OpBuilder};
+use slice_group_by::{GroupBy, GroupByMut};
+
+use crate::error::Error;
+use crate::criterion::{Criteria, Context, ContextMut};
+use crate::distinct_map::{BufferedDistinctMap, DistinctMap};
+use crate::raw_document::RawDocument;
+use crate::{database::MainT, reordered_attrs::ReorderedAttrs};
+use crate::{store, Document, DocumentId, MResult};
+use crate::query_tree::{create_query_tree, traverse_query_tree};
+use crate::query_tree::{Operation, QueryResult, QueryKind, QueryId, PostingsKey};
+use crate::query_tree::Context as QTContext;
+
+#[derive(Debug, Default)]
+pub struct SortResult {
+    pub documents: Vec<Document>,
+    pub nb_hits: usize,
+    pub exhaustive_nb_hit: bool,
+    pub facets: Option<HashMap<String, HashMap<String, usize>>>,
+    pub exhaustive_facets_count: Option<bool>,
+}
+
+pub fn bucket_sort<'c, FI>(
+    reader: &heed::RoTxn<MainT>,
+    query: &str,
+    range: Range<usize>,
+    facets_docids: Option<SetBuf<DocumentId>>,
+    facet_count_docids: Option<HashMap<String, HashMap<String, Cow<Set<DocumentId>>>>>,
+    filter: Option<FI>,
+    criteria: Criteria<'c>,
+    searchable_attrs: Option<ReorderedAttrs>,
+    main_store: store::Main,
+    postings_lists_store: store::PostingsLists,
+    documents_fields_counts_store: store::DocumentsFieldsCounts,
+    synonyms_store: store::Synonyms,
+    prefix_documents_cache_store: store::PrefixDocumentsCache,
+    prefix_postings_lists_cache_store: store::PrefixPostingsListsCache,
+) -> MResult<SortResult>
+where
+    FI: Fn(DocumentId) -> bool,
+{
+    // We delegate the filter work to the distinct query builder,
+    // specifying a distinct rule that has no effect.
+    if filter.is_some() {
+        let distinct = |_| None;
+        let distinct_size = 1;
+        return bucket_sort_with_distinct(
+            reader,
+            query,
+            range,
+            facets_docids,
+            facet_count_docids,
+            filter,
+            distinct,
+            distinct_size,
+            criteria,
+            searchable_attrs,
+            main_store,
+            postings_lists_store,
+            documents_fields_counts_store,
+            synonyms_store,
+            prefix_documents_cache_store,
+            prefix_postings_lists_cache_store,
+        );
+    }
+
+    let mut result = SortResult::default();
+
+    let words_set = main_store.words_fst(reader)?;
+    let stop_words = main_store.stop_words_fst(reader)?;
+
+    let context = QTContext {
+        words_set,
+        stop_words,
+        synonyms: synonyms_store,
+        postings_lists: postings_lists_store,
+        prefix_postings_lists: prefix_postings_lists_cache_store,
+    };
+
+    let (operation, mapping) = create_query_tree(reader, &context, query)?;
+    debug!("operation:\n{:?}", operation);
+    debug!("mapping:\n{:?}", mapping);
+
+    fn recurs_operation<'o>(map: &mut HashMap<QueryId, &'o QueryKind>, operation: &'o Operation) {
+        match operation {
+            Operation::And(ops) => ops.iter().for_each(|op| recurs_operation(map, op)),
+            Operation::Or(ops) => ops.iter().for_each(|op| recurs_operation(map, op)),
+            Operation::Query(query) => { map.insert(query.id, &query.kind); },
+        }
+    }
+
+    let mut queries_kinds = HashMap::new();
+    recurs_operation(&mut queries_kinds, &operation);
+
+    let QueryResult { mut docids, queries } = traverse_query_tree(reader, &context, &operation)?;
+    debug!("found {} documents", docids.len());
+    debug!("number of postings {:?}", queries.len());
+
+    if let Some(facets_docids) = facets_docids {
+        let intersection = sdset::duo::OpBuilder::new(docids.as_ref(), facets_docids.as_set())
+            .intersection()
+            .into_set_buf();
+        docids = Cow::Owned(intersection);
+    }
+
+    if let Some(f) = facet_count_docids {
+        // hardcoded value, until approximation optimization
+        result.exhaustive_facets_count = Some(true);
+        result.facets = Some(facet_count(f, &docids));
+    }
+
+    let before = Instant::now();
+    mk_arena!(arena);
+    let mut bare_matches = cleanup_bare_matches(&mut arena, &docids, queries);
+    debug!("matches cleaned in {:.02?}", before.elapsed());
+
+    let before_bucket_sort = Instant::now();
+
+    let before_raw_documents_building = Instant::now();
+    let mut raw_documents = Vec::new();
+    for bare_matches in bare_matches.linear_group_by_key_mut(|sm| sm.document_id) {
+        let raw_document = RawDocument::new(bare_matches, &mut arena, searchable_attrs.as_ref());
+        raw_documents.push(raw_document);
+    }
+    debug!("creating {} candidates documents took {:.02?}",
+        raw_documents.len(),
+        before_raw_documents_building.elapsed(),
+    );
+
+    let before_criterion_loop = Instant::now();
+    let proximity_count = AtomicUsize::new(0);
+
+    let mut groups = vec![raw_documents.as_mut_slice()];
+
+    'criteria: for criterion in criteria.as_ref() {
+        let tmp_groups = mem::replace(&mut groups, Vec::new());
+        let mut documents_seen = 0;
+
+        for mut group in tmp_groups {
+            let before_criterion_preparation = Instant::now();
+
+            let ctx = ContextMut {
+                reader,
+                postings_lists: &mut arena,
+                query_mapping: &mapping,
+                documents_fields_counts_store,
+            };
+
+            criterion.prepare(ctx, &mut group)?;
+            debug!("{:?} preparation took {:.02?}", criterion.name(), before_criterion_preparation.elapsed());
+
+            let ctx = Context {
+                postings_lists: &arena,
+                query_mapping: &mapping,
+            };
+
+            let before_criterion_sort = Instant::now();
+            group.sort_unstable_by(|a, b| criterion.evaluate(&ctx, a, b));
+            debug!("{:?} evaluation took {:.02?}", criterion.name(), before_criterion_sort.elapsed());
+
+            for group in group.binary_group_by_mut(|a, b| criterion.eq(&ctx, a, b)) {
+                debug!("{:?} produced a group of size {}", criterion.name(), group.len());
+
+                documents_seen += group.len();
+                groups.push(group);
+
+                // we have sort enough documents if the last document sorted is after
+                // the end of the requested range, we can continue to the next criterion
+                if documents_seen >= range.end {
+                    continue 'criteria;
+                }
+            }
+        }
+    }
+
+    debug!("criterion loop took {:.02?}", before_criterion_loop.elapsed());
+    debug!("proximity evaluation called {} times", proximity_count.load(Ordering::Relaxed));
+
+    let schema = main_store.schema(reader)?.ok_or(Error::SchemaMissing)?;
+    let iter = raw_documents.into_iter().skip(range.start).take(range.len());
+    let iter = iter.map(|rd| Document::from_raw(rd, &queries_kinds, &arena, searchable_attrs.as_ref(), &schema));
+    let documents = iter.collect();
+
+    debug!("bucket sort took {:.02?}", before_bucket_sort.elapsed());
+
+    result.documents = documents;
+    result.nb_hits = docids.len();
+
+    Ok(result)
+}
+
+pub fn bucket_sort_with_distinct<'c, FI, FD>(
+    reader: &heed::RoTxn<MainT>,
+    query: &str,
+    range: Range<usize>,
+    facets_docids: Option<SetBuf<DocumentId>>,
+    facet_count_docids: Option<HashMap<String, HashMap<String, Cow<Set<DocumentId>>>>>,
+    filter: Option<FI>,
+    distinct: FD,
+    distinct_size: usize,
+    criteria: Criteria<'c>,
+    searchable_attrs: Option<ReorderedAttrs>,
+    main_store: store::Main,
+    postings_lists_store: store::PostingsLists,
+    documents_fields_counts_store: store::DocumentsFieldsCounts,
+    synonyms_store: store::Synonyms,
+    _prefix_documents_cache_store: store::PrefixDocumentsCache,
+    prefix_postings_lists_cache_store: store::PrefixPostingsListsCache,
+) -> MResult<SortResult>
+where
+    FI: Fn(DocumentId) -> bool,
+    FD: Fn(DocumentId) -> Option<u64>,
+{
+    let mut result = SortResult::default();
+
+    let words_set = main_store.words_fst(reader)?;
+    let stop_words = main_store.stop_words_fst(reader)?;
+
+    let context = QTContext {
+        words_set,
+        stop_words,
+        synonyms: synonyms_store,
+        postings_lists: postings_lists_store,
+        prefix_postings_lists: prefix_postings_lists_cache_store,
+    };
+
+    let (operation, mapping) = create_query_tree(reader, &context, query)?;
+    debug!("operation:\n{:?}", operation);
+    debug!("mapping:\n{:?}", mapping);
+
+    fn recurs_operation<'o>(map: &mut HashMap<QueryId, &'o QueryKind>, operation: &'o Operation) {
+        match operation {
+            Operation::And(ops) => ops.iter().for_each(|op| recurs_operation(map, op)),
+            Operation::Or(ops) => ops.iter().for_each(|op| recurs_operation(map, op)),
+            Operation::Query(query) => { map.insert(query.id, &query.kind); },
+        }
+    }
+
+    let mut queries_kinds = HashMap::new();
+    recurs_operation(&mut queries_kinds, &operation);
+
+    let QueryResult { mut docids, queries } = traverse_query_tree(reader, &context, &operation)?;
+    debug!("found {} documents", docids.len());
+    debug!("number of postings {:?}", queries.len());
+
+    if let Some(facets_docids) = facets_docids {
+        let intersection = OpBuilder::new(docids.as_ref(), facets_docids.as_set())
+            .intersection()
+            .into_set_buf();
+        docids = Cow::Owned(intersection);
+    }
+
+    if let Some(f) = facet_count_docids {
+        // hardcoded value, until approximation optimization
+        result.exhaustive_facets_count = Some(true);
+        result.facets = Some(facet_count(f, &docids));
+    }
+
+    let before = Instant::now();
+    mk_arena!(arena);
+    let mut bare_matches = cleanup_bare_matches(&mut arena, &docids, queries);
+    debug!("matches cleaned in {:.02?}", before.elapsed());
+
+    let before_raw_documents_building = Instant::now();
+    let mut raw_documents = Vec::new();
+    for bare_matches in bare_matches.linear_group_by_key_mut(|sm| sm.document_id) {
+        let raw_document = RawDocument::new(bare_matches, &mut arena, searchable_attrs.as_ref());
+        raw_documents.push(raw_document);
+    }
+    debug!("creating {} candidates documents took {:.02?}",
+        raw_documents.len(),
+        before_raw_documents_building.elapsed(),
+    );
+
+    let mut groups = vec![raw_documents.as_mut_slice()];
+    let mut key_cache = HashMap::new();
+
+    let mut filter_map = HashMap::new();
+    // these two variables informs on the current distinct map and
+    // on the raw offset of the start of the group where the
+    // range.start bound is located according to the distinct function
+    let mut distinct_map = DistinctMap::new(distinct_size);
+    let mut distinct_raw_offset = 0;
+
+    'criteria: for criterion in criteria.as_ref() {
+        let tmp_groups = mem::replace(&mut groups, Vec::new());
+        let mut buf_distinct = BufferedDistinctMap::new(&mut distinct_map);
+        let mut documents_seen = 0;
+
+        for mut group in tmp_groups {
+            // if this group does not overlap with the requested range,
+            // push it without sorting and splitting it
+            if documents_seen + group.len() < distinct_raw_offset {
+                documents_seen += group.len();
+                groups.push(group);
+                continue;
+            }
+
+            let ctx = ContextMut {
+                reader,
+                postings_lists: &mut arena,
+                query_mapping: &mapping,
+                documents_fields_counts_store,
+            };
+
+            let before_criterion_preparation = Instant::now();
+            criterion.prepare(ctx, &mut group)?;
+            debug!("{:?} preparation took {:.02?}", criterion.name(), before_criterion_preparation.elapsed());
+
+            let ctx = Context {
+                postings_lists: &arena,
+                query_mapping: &mapping,
+            };
+
+            let before_criterion_sort = Instant::now();
+            group.sort_unstable_by(|a, b| criterion.evaluate(&ctx, a, b));
+            debug!("{:?} evaluation took {:.02?}", criterion.name(), before_criterion_sort.elapsed());
+
+            for group in group.binary_group_by_mut(|a, b| criterion.eq(&ctx, a, b)) {
+                // we must compute the real distinguished len of this sub-group
+                for document in group.iter() {
+                    let filter_accepted = match &filter {
+                        Some(filter) => {
+                            let entry = filter_map.entry(document.id);
+                            *entry.or_insert_with(|| (filter)(document.id))
+                        }
+                        None => true,
+                    };
+
+                    if filter_accepted {
+                        let entry = key_cache.entry(document.id);
+                        let key = entry.or_insert_with(|| (distinct)(document.id).map(Rc::new));
+
+                        match key.clone() {
+                            Some(key) => buf_distinct.register(key),
+                            None => buf_distinct.register_without_key(),
+                        };
+                    }
+
+                    // the requested range end is reached: stop computing distinct
+                    if buf_distinct.len() >= range.end {
+                        break;
+                    }
+                }
+
+                documents_seen += group.len();
+                groups.push(group);
+
+                // if this sub-group does not overlap with the requested range
+                // we must update the distinct map and its start index
+                if buf_distinct.len() < range.start {
+                    buf_distinct.transfert_to_internal();
+                    distinct_raw_offset = documents_seen;
+                }
+
+                // we have sort enough documents if the last document sorted is after
+                // the end of the requested range, we can continue to the next criterion
+                if buf_distinct.len() >= range.end {
+                    continue 'criteria;
+                }
+            }
+        }
+    }
+
+    // once we classified the documents related to the current
+    // automatons we save that as the next valid result
+    let mut seen = BufferedDistinctMap::new(&mut distinct_map);
+    let schema = main_store.schema(reader)?.ok_or(Error::SchemaMissing)?;
+
+    let mut documents = Vec::with_capacity(range.len());
+    for raw_document in raw_documents.into_iter().skip(distinct_raw_offset) {
+        let filter_accepted = match &filter {
+            Some(_) => filter_map.remove(&raw_document.id).unwrap(),
+            None => true,
+        };
+
+        if filter_accepted {
+            let key = key_cache.remove(&raw_document.id).unwrap();
+            let distinct_accepted = match key {
+                Some(key) => seen.register(key),
+                None => seen.register_without_key(),
+            };
+
+            if distinct_accepted && seen.len() > range.start {
+                documents.push(Document::from_raw(raw_document, &queries_kinds, &arena, searchable_attrs.as_ref(), &schema));
+                if documents.len() == range.len() {
+                    break;
+                }
+            }
+        }
+    }
+    result.documents = documents;
+    result.nb_hits = docids.len();
+
+    Ok(result)
+}
+
+fn cleanup_bare_matches<'tag, 'txn>(
+    arena: &mut SmallArena<'tag, PostingsListView<'txn>>,
+    docids: &Set<DocumentId>,
+    queries: HashMap<PostingsKey, Cow<'txn, Set<DocIndex>>>,
+) -> Vec<BareMatch<'tag>>
+{
+    let docidslen = docids.len() as f32;
+    let mut bare_matches = Vec::new();
+
+    for (PostingsKey { query, input, distance, is_exact }, matches) in queries {
+        let postings_list_view = PostingsListView::original(Rc::from(input), Rc::new(matches));
+        let pllen = postings_list_view.len() as f32;
+
+        if docidslen / pllen >= 0.8 {
+            let mut offset = 0;
+            for matches in postings_list_view.linear_group_by_key(|m| m.document_id) {
+                let document_id = matches[0].document_id;
+                if docids.contains(&document_id) {
+                    let range = postings_list_view.range(offset, matches.len());
+                    let posting_list_index = arena.add(range);
+
+                    let bare_match = BareMatch {
+                        document_id,
+                        query_index: query.id,
+                        distance,
+                        is_exact,
+                        postings_list: posting_list_index,
+                    };
+
+                    bare_matches.push(bare_match);
+                }
+
+                offset += matches.len();
+            }
+
+        } else {
+            let mut offset = 0;
+            for id in docids.as_slice() {
+                let di = DocIndex { document_id: *id, ..DocIndex::default() };
+                let pos = exponential_search(&postings_list_view[offset..], &di).unwrap_or_else(|x| x);
+
+                offset += pos;
+
+                let group = postings_list_view[offset..]
+                    .linear_group_by_key(|m| m.document_id)
+                    .next()
+                    .filter(|matches| matches[0].document_id == *id);
+
+                if let Some(matches) = group {
+                    let range = postings_list_view.range(offset, matches.len());
+                    let posting_list_index = arena.add(range);
+
+                    let bare_match = BareMatch {
+                        document_id: *id,
+                        query_index: query.id,
+                        distance,
+                        is_exact,
+                        postings_list: posting_list_index,
+                    };
+
+                    bare_matches.push(bare_match);
+                }
+            }
+        }
+    }
+
+    let before_raw_documents_presort = Instant::now();
+    bare_matches.sort_unstable_by_key(|sm| sm.document_id);
+    debug!("sort by documents ids took {:.02?}", before_raw_documents_presort.elapsed());
+
+    bare_matches
+}
+
+pub struct BareMatch<'tag> {
+    pub document_id: DocumentId,
+    pub query_index: usize,
+    pub distance: u8,
+    pub is_exact: bool,
+    pub postings_list: Idx32<'tag>,
+}
+
+impl fmt::Debug for BareMatch<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("BareMatch")
+            .field("document_id", &self.document_id)
+            .field("query_index", &self.query_index)
+            .field("distance", &self.distance)
+            .field("is_exact", &self.is_exact)
+            .finish()
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct SimpleMatch {
+    pub query_index: usize,
+    pub distance: u8,
+    pub attribute: u16,
+    pub word_index: u16,
+    pub is_exact: bool,
+}
+
+#[derive(Clone)]
+pub enum PostingsListView<'txn> {
+    Original {
+        input: Rc<[u8]>,
+        postings_list: Rc<Cow<'txn, Set<DocIndex>>>,
+        offset: usize,
+        len: usize,
+    },
+    Rewritten {
+        input: Rc<[u8]>,
+        postings_list: SetBuf<DocIndex>,
+    },
+}
+
+impl fmt::Debug for PostingsListView<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("PostingsListView")
+            .field("input", &std::str::from_utf8(&self.input()).unwrap())
+            .field("postings_list", &self.as_ref())
+            .finish()
+    }
+}
+
+impl<'txn> PostingsListView<'txn> {
+    pub fn original(input: Rc<[u8]>, postings_list: Rc<Cow<'txn, Set<DocIndex>>>) -> PostingsListView<'txn> {
+        let len = postings_list.len();
+        PostingsListView::Original { input, postings_list, offset: 0, len }
+    }
+
+    pub fn rewritten(input: Rc<[u8]>, postings_list: SetBuf<DocIndex>) -> PostingsListView<'txn> {
+        PostingsListView::Rewritten { input, postings_list }
+    }
+
+    pub fn rewrite_with(&mut self, postings_list: SetBuf<DocIndex>) {
+        let input = match self {
+            PostingsListView::Original { input, .. } => input.clone(),
+            PostingsListView::Rewritten { input, .. } => input.clone(),
+        };
+        *self = PostingsListView::rewritten(input, postings_list);
+    }
+
+    pub fn len(&self) -> usize {
+        match self {
+            PostingsListView::Original { len, .. } => *len,
+            PostingsListView::Rewritten { postings_list, .. } => postings_list.len(),
+        }
+    }
+
+    pub fn input(&self) -> &[u8] {
+        match self {
+            PostingsListView::Original { ref input, .. } => input,
+            PostingsListView::Rewritten { ref input, .. } => input,
+        }
+    }
+
+    pub fn range(&self, range_offset: usize, range_len: usize) -> PostingsListView<'txn> {
+        match self {
+            PostingsListView::Original { input, postings_list, offset, len } => {
+                assert!(range_offset + range_len <= *len);
+                PostingsListView::Original {
+                    input: input.clone(),
+                    postings_list: postings_list.clone(),
+                    offset: offset + range_offset,
+                    len: range_len,
+                }
+            },
+            PostingsListView::Rewritten { .. } => {
+                panic!("Cannot create a range on a rewritten postings list view");
+            }
+        }
+    }
+}
+
+impl AsRef<Set<DocIndex>> for PostingsListView<'_> {
+    fn as_ref(&self) -> &Set<DocIndex> {
+        self
+    }
+}
+
+impl Deref for PostingsListView<'_> {
+    type Target = Set<DocIndex>;
+
+    fn deref(&self) -> &Set<DocIndex> {
+        match *self {
+            PostingsListView::Original { ref postings_list, offset, len, .. } => {
+                Set::new_unchecked(&postings_list[offset..offset + len])
+            },
+            PostingsListView::Rewritten { ref postings_list, .. } => postings_list,
+        }
+    }
+}
+
+/// For each entry in facet_docids, calculates the number of documents in the intersection with candidate_docids.
+fn facet_count(
+    facet_docids: HashMap<String, HashMap<String, Cow<Set<DocumentId>>>>,
+    candidate_docids: &Set<DocumentId>,
+) -> HashMap<String, HashMap<String, usize>> {
+    let mut facets_counts = HashMap::with_capacity(facet_docids.len());
+    for (key, doc_map) in facet_docids {
+        let mut count_map = HashMap::with_capacity(doc_map.len());
+        for (value, docids) in doc_map {
+            let mut counter = Counter::new();
+            let op = OpBuilder::new(docids.as_ref(), candidate_docids).intersection();
+            SetOperation::<DocumentId>::extend_collection(op, &mut counter);
+            count_map.insert(value, counter.0);
+        }
+        facets_counts.insert(key, count_map);
+    }
+    facets_counts
+}

meilisearch-core/src/criterion/attribute.rs

@@ -0,0 +1,37 @@
+use std::cmp::Ordering;
+use slice_group_by::GroupBy;
+use crate::{RawDocument, MResult};
+use crate::bucket_sort::SimpleMatch;
+use super::{Criterion, Context, ContextMut, prepare_bare_matches};
+
+pub struct Attribute;
+
+impl Criterion for Attribute {
+    fn name(&self) -> &str { "attribute" }
+
+    fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>(
+        &self,
+        ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>,
+        documents: &mut [RawDocument<'r, 'tag>],
+    ) -> MResult<()>
+    {
+        prepare_bare_matches(documents, ctx.postings_lists, ctx.query_mapping);
+        Ok(())
+    }
+
+    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
+        #[inline]
+        fn sum_of_attribute(matches: &[SimpleMatch]) -> usize {
+            let mut sum_of_attribute = 0;
+            for group in matches.linear_group_by_key(|bm| bm.query_index) {
+                sum_of_attribute += group[0].attribute as usize;
+            }
+            sum_of_attribute
+        }
+
+        let lhs = sum_of_attribute(&lhs.processed_matches);
+        let rhs = sum_of_attribute(&rhs.processed_matches);
+
+        lhs.cmp(&rhs)
+    }
+}

meilisearch-core/src/criterion/document_id.rs

@@ -0,0 +1,16 @@
+use std::cmp::Ordering;
+use crate::RawDocument;
+use super::{Criterion, Context};
+
+pub struct DocumentId;
+
+impl Criterion for DocumentId {
+    fn name(&self) -> &str { "stable document id" }
+
+    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
+        let lhs = &lhs.id;
+        let rhs = &rhs.id;
+
+        lhs.cmp(rhs)
+    }
+}

meilisearch-core/src/criterion/exactness.rs

@@ -0,0 +1,78 @@
+use std::cmp::{Ordering, Reverse};
+use std::collections::hash_map::{HashMap, Entry};
+use meilisearch_schema::IndexedPos;
+use slice_group_by::GroupBy;
+use crate::{RawDocument, MResult};
+use crate::bucket_sort::BareMatch;
+use super::{Criterion, Context, ContextMut};
+
+pub struct Exactness;
+
+impl Criterion for Exactness {
+    fn name(&self) -> &str { "exactness" }
+
+    fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>(
+        &self,
+        ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>,
+        documents: &mut [RawDocument<'r, 'tag>],
+    ) -> MResult<()>
+    {
+        let store = ctx.documents_fields_counts_store;
+        let reader = ctx.reader;
+
+        'documents: for doc in documents {
+            doc.bare_matches.sort_unstable_by_key(|bm| (bm.query_index, Reverse(bm.is_exact)));
+
+            // mark the document if we find a "one word field" that matches
+            let mut fields_counts = HashMap::new();
+            for group in doc.bare_matches.linear_group_by_key(|bm| bm.query_index) {
+                for group in group.linear_group_by_key(|bm| bm.is_exact) {
+                    if !group[0].is_exact { break }
+
+                    for bm in group {
+                        for di in ctx.postings_lists[bm.postings_list].as_ref() {
+
+                            let attr = IndexedPos(di.attribute);
+                            let count = match fields_counts.entry(attr) {
+                                Entry::Occupied(entry) => *entry.get(),
+                                Entry::Vacant(entry) => {
+                                    let count = store.document_field_count(reader, doc.id, attr)?;
+                                    *entry.insert(count)
+                                },
+                            };
+
+                            if count == Some(1) {
+                                doc.contains_one_word_field = true;
+                                continue 'documents
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
+        #[inline]
+        fn sum_exact_query_words(matches: &[BareMatch]) -> usize {
+            let mut sum_exact_query_words = 0;
+
+            for group in matches.linear_group_by_key(|bm| bm.query_index) {
+                sum_exact_query_words += group[0].is_exact as usize;
+            }
+
+            sum_exact_query_words
+        }
+
+        // does it contains a "one word field"
+        lhs.contains_one_word_field.cmp(&rhs.contains_one_word_field).reverse()
+        // if not, with document contains the more exact words
+        .then_with(|| {
+            let lhs = sum_exact_query_words(&lhs.bare_matches);
+            let rhs = sum_exact_query_words(&rhs.bare_matches);
+            lhs.cmp(&rhs).reverse()
+        })
+    }
+}