Merge #1904

1904: Update mini-dashboard version to v0.1.5 r=curquiza a=curquiza

Update the mini-dashboard with its latest version (v0.1.5)

Check with `@mdubus,` replaces https://github.com/meilisearch/MeiliSearch/pull/1903

Fixes #1898 

Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>

.dockerignore

@@ -1,5 +1,4 @@
 target
 Dockerfile
 .dockerignore
-.git
 .gitignore

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,30 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**MeiliSearch version:** [e.g. v0.20.0]
+
+**Additional context**
+Additional information that may be relevant to the issue.
+[e.g. architecture, device, OS, browser]

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,10 @@
+contact_links:
+  - name: Feature request
+    url: https://github.com/meilisearch/product/discussions/categories/feedback-feature-proposal
+    about: The feature requests are not managed in this repository, please open a discussion in our dedicated product repository
+  - name: Documentation issue
+    url: https://github.com/meilisearch/documentation/issues/new
+    about: For documentation issues, open an issue or a PR in the documentation repository
+  - name: Support questions & other
+    url: https://github.com/meilisearch/MeiliSearch/discussions/new
+    about: For any other question, open a discussion in this repository

.github/is-latest-release.sh

@@ -0,0 +1,132 @@
+#!/bin/sh
+
+# Checks if the current tag should be the latest (in terms of semver and not of release date).
+# Ex: previous tag -> v0.10.1
+#     new tag -> v0.8.12
+#     The new tag should not be the latest
+#     So it returns "false", the CI should not run for the release v0.8.2
+
+# Used in GHA in publish-docker-latest.yml
+# Returns "true" or "false" (as a string) to be used in the `if` in GHA
+
+# GLOBAL
+GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number]
+
+# FUNCTIONS
+
+# semverParseInto and semverLT from https://github.com/cloudflare/semver_bash/blob/master/semver.sh
+
+# usage: semverParseInto version major minor patch special
+# version: the string version
+# major, minor, patch, special: will be assigned by the function
+semverParseInto() {
+    local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)'
+    #MAJOR
+    eval $2=`echo $1 | sed -e "s#$RE#\1#"`
+    #MINOR
+    eval $3=`echo $1 | sed -e "s#$RE#\2#"`
+    #MINOR
+    eval $4=`echo $1 | sed -e "s#$RE#\3#"`
+    #SPECIAL
+    eval $5=`echo $1 | sed -e "s#$RE#\4#"`
+}
+
+# usage: semverLT version1 version2
+semverLT() {
+    local MAJOR_A=0
+    local MINOR_A=0
+    local PATCH_A=0
+    local SPECIAL_A=0
+
+    local MAJOR_B=0
+    local MINOR_B=0
+    local PATCH_B=0
+    local SPECIAL_B=0
+
+    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
+    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
+
+    if [ $MAJOR_A -lt $MAJOR_B ]; then
+        return 0
+    fi
+    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -lt $MINOR_B ]; then
+        return 0
+    fi
+    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then
+        return 0
+    fi
+    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+        return 1
+    fi
+    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  != "_" ] ; then
+        return 1
+    fi
+    if [ "_$SPECIAL_A"  != "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+        return 0
+    fi
+    if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then
+        return 0
+    fi
+
+    return 1
+}
+
+# Returns the tag of the latest stable release (in terms of semver and not of release date)
+get_latest() {
+    temp_file='temp_file' # temp_file needed because the grep would start before the download is over
+    curl -s 'https://api.github.com/repos/meilisearch/MeiliSearch/releases' > "$temp_file"
+    releases=$(cat "$temp_file" | \
+        grep -E "tag_name|draft|prerelease" \
+        | tr -d ',"' | cut -d ':' -f2 | tr -d ' ')
+        # Returns a list of [tag_name draft_boolean prerelease_boolean ...]
+        # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false...
+
+    i=0
+    latest=""
+    current_tag=""
+    for release_info in $releases; do
+        if [ $i -eq 0 ]; then # Cheking tag_name
+            if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release
+                current_tag=$release_info
+            else
+                current_tag=""
+            fi
+            i=1
+        elif [ $i -eq 1 ]; then # Checking draft boolean
+            if [ "$release_info" = "true" ]; then
+                current_tag=""
+            fi
+            i=2
+        elif [ $i -eq 2 ]; then # Checking prerelease boolean
+            if [ "$release_info" = "true" ]; then
+                current_tag=""
+            fi
+            i=0
+            if [ "$current_tag" != "" ]; then # If the current_tag is valid
+                if [ "$latest" = "" ]; then # If there is no latest yet
+                    latest="$current_tag"
+                else
+                    semverLT $current_tag $latest # Comparing latest and the current tag
+                    if [ $? -eq 1 ]; then
+                        latest="$current_tag"
+                    fi
+                fi
+            fi
+        fi
+    done
+
+    rm -f "$temp_file"
+    echo $latest
+}
+
+# MAIN
+current_tag="$(echo $GITHUB_REF | tr -d 'refs/tags/')"
+latest="$(get_latest)"
+
+if [ "$current_tag" != "$latest" ]; then
+    # The current release tag is not the latest
+    echo "false"
+else
+    # The current release tag is the latest
+    echo "true"
+fi

.github/release-draft-template.yml

@@ -0,0 +1,13 @@
+name-template: 'v$RESOLVED_VERSION'
+tag-template: 'v$RESOLVED_VERSION'
+version-template: '0.21.0-alpha.$PATCH'
+exclude-labels:
+  - 'skip-changelog'
+template: |
+  ## Changes
+
+  $CHANGES
+no-changes-template: 'Changes are coming soon 😎'
+sort-direction: 'ascending'
+version-resolver:
+  default: patch

.github/workflows/README.md

@@ -1,4 +1,4 @@
-# GitHub actions workflow for MeiliDB
+# GitHub Actions Workflow for MeiliSearch
 
 > **Note:**
 
@@ -6,12 +6,14 @@
 
 ## Workflow
 
-- On each pull request, we are triggering `cargo test`.
-- On each tag, we are building:
-    - the tagged docker image
+- On each pull request, we trigger `cargo test`.
+- On each tag, we build:
+    - the tagged Docker image and publish it to Docker Hub
     - the binaries for MacOS, Ubuntu, and Windows
-    - the debian package
-- On each stable release, we are build the latest docker image.
+    - the Debian package
+- On each stable release (`v*.*.*` tag):
+    - we build the `latest` Docker image and publish it to Docker Hub
+    - we publish the binary to Hombrew and Gemfury
 
 ## Problems
 

.github/workflows/coverage.yml

@@ -0,0 +1,33 @@
+---
+on:
+  workflow_dispatch:
+
+name: Execute code coverage
+
+jobs:
+  nightly-coverage:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: clean
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --all-features --no-fail-fast
+        env:
+          CARGO_INCREMENTAL: "0"
+          RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=unwind -Zpanic_abort_tests"
+      - uses: actions-rs/grcov@v0.1
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ${{ steps.coverage.outputs.report }}
+          yml: ./codecov.yml
+          fail_ci_if_error: true

.github/workflows/flaky.yml

@@ -0,0 +1,15 @@
+name: Look for flaky tests
+on:
+  schedule:
+    - cron: "0 12 * * FRI" # every friday at 12:00PM
+
+jobs:
+  flaky:
+    runs-on: ubuntu-18.04
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install cargo-flaky
+      run: cargo install cargo-flaky
+    - name: Run cargo flaky 100 times
+      run: cargo flaky -i 100 --release

.github/workflows/publish-binaries.yml

@@ -1,9 +1,8 @@
-name: Publish binaries to GitHub release
-
 on:
-  push:
-    tags:
-      - '*'
+  release:
+    types: [published]
+
+name: Publish binaries to release
 
 jobs:
   publish:
@@ -11,29 +10,55 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-18.04, macos-latest, windows-latest]
         include:
-          - os: ubuntu-latest
+          - os: ubuntu-18.04
             artifact_name: meilisearch
             asset_name: meilisearch-linux-amd64
           - os: macos-latest
             artifact_name: meilisearch
             asset_name: meilisearch-macos-amd64
           - os: windows-latest
-            artifact_name: meilisearch
-            asset_name: meilisearch-windows-amd64
+            artifact_name: meilisearch.exe
+            asset_name: meilisearch-windows-amd64.exe
 
     steps:
     - uses: hecrj/setup-rust-action@master
       with:
         rust-version: stable
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - name: Build
       run: cargo build --release --locked
     - name: Upload binaries to release
       uses: svenstaro/upload-release-action@v1-release
       with:
-        repo_token: ${{ secrets.GITHUB_TOKEN }}
+        repo_token: ${{ secrets.PUBLISH_TOKEN }}
         file: target/release/${{ matrix.artifact_name }}
         asset_name: ${{ matrix.asset_name }}
         tag: ${{ github.ref }}
+
+  publish-armv8:
+    name: Publish for ARMv8
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: uraimo/run-on-arch-action@v2.1.1
+        id: runcmd
+        with:
+          arch: aarch64 # aka ARMv8
+          distro: ubuntu18.04
+          env: |
+            JEMALLOC_SYS_WITH_LG_PAGE: 16
+          run: |
+            apt update
+            apt install -y curl gcc make
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
+            source $HOME/.cargo/env
+            cargo build --release --locked
+      - name: Upload the binary to release
+        uses: svenstaro/upload-release-action@v1-release
+        with:
+          repo_token: ${{ secrets.PUBLISH_TOKEN }}
+          file: target/release/meilisearch
+          asset_name: meilisearch-linux-armv8
+          tag: ${{ github.ref }}

.github/workflows/publish-crossbuild.yml

@@ -0,0 +1,76 @@
+name: Publish aarch64 binary
+
+on:
+  release:
+    types: [published]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  publish-aarch64:
+    name: Publish to Github
+    runs-on: ${{ matrix.os }}
+    continue-on-error: false
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - build: aarch64
+            os: ubuntu-18.04
+            target: aarch64-unknown-linux-gnu
+            linker: gcc-aarch64-linux-gnu
+            use-cross: true
+            asset_name: meilisearch-linux-aarch64
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Installing Rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+          target: ${{ matrix.target }}
+          override: true
+
+      - name: APT update
+        run: |
+          sudo apt update
+
+      - name: Install target specific tools
+        if: matrix.use-cross
+        run: |
+          sudo apt-get install -y ${{ matrix.linker }}
+
+      - name: Configure target aarch64 GNU
+        if: matrix.target == 'aarch64-unknown-linux-gnu'
+        ## Environment variable is not passed using env:
+        ## LD gold won't work with MUSL
+        # env:
+        #   JEMALLOC_SYS_WITH_LG_PAGE: 16
+        #   RUSTFLAGS: '-Clink-arg=-fuse-ld=gold'
+        run: |
+          echo '[target.aarch64-unknown-linux-gnu]' >> ~/.cargo/config
+          echo 'linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config
+          echo 'JEMALLOC_SYS_WITH_LG_PAGE=16' >> $GITHUB_ENV
+          echo RUSTFLAGS="-Clink-arg=-fuse-ld=gold" >> $GITHUB_ENV
+
+      - name: Cargo build
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          use-cross: ${{ matrix.use-cross }}
+          args: --release --target ${{ matrix.target }}
+
+      - name: List target output files
+        run: ls -lR ./target
+
+      - name: Upload the binary to release
+        uses: svenstaro/upload-release-action@v1-release
+        with:
+          repo_token: ${{ secrets.PUBLISH_TOKEN }}
+          file: target/${{ matrix.target }}/release/meilisearch
+          asset_name: ${{ matrix.asset_name }}
+          tag: ${{ github.ref }}

.github/workflows/publish-deb-brew-pkg.yml

@@ -1,22 +1,20 @@
-name: Publish deb pkg to GitHub release & apt repository
+name: Publish deb pkg to GitHub release & APT repository & Homebrew
 
 on:
-  push:
-    tags:
-      - '*'
+  release:
+    types: [released]
 
 jobs:
-  publish:
+  debian:
     name: Publish debian packagge
-    runs-on: ubuntu-latest
-
+    runs-on: ubuntu-18.04
     steps:
     - uses: hecrj/setup-rust-action@master
       with:
         rust-version: stable
     - name: Install cargo-deb
       run: cargo install cargo-deb
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - name: Build deb package
       run: cargo deb -p meilisearch-http -o target/debian/meilisearch.deb
     - name: Upload debian pkg to release
@@ -28,3 +26,14 @@ jobs:
         tag: ${{ github.ref }}
     - name: Upload debian pkg to apt repository
       run: curl -F package=@target/debian/meilisearch.deb https://${{ secrets.GEMFURY_PUSH_TOKEN }}@push.fury.io/meilisearch/
+
+  homebrew:
+    name: Bump Homebrew formula
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Create PR to Homebrew
+        uses: mislav/bump-homebrew-formula-action@v1
+        with:
+          formula-name: meilisearch
+        env:
+          COMMITTER_TOKEN: ${{ secrets.HOMEBREW_COMMITTER_TOKEN }}

.github/workflows/publish-docker-latest.yml

@@ -1,17 +1,20 @@
 ---
 on:
-  push:
-    tags:
-      - 'v[0-9]+.[0-9]+.[0-9]+'
+  release:
+    types: [released]
 
 name: Publish latest image to Docker Hub
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+      - name: Check if current release is latest
+        run: echo "##[set-output name=is_latest;]$(sh .github/is-latest-release.sh)"
+        id: release
       - name: Publish to Registry
+        if: steps.release.outputs.is_latest == 'true'
         uses: elgohr/Publish-Docker-Github-Action@master
         with:
           name: getmeili/meilisearch

.github/workflows/publish-docker-tag.yml

@@ -8,11 +8,13 @@ name: Publish tagged image to Docker Hub
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - name: Publish to Registry
         uses: elgohr/Publish-Docker-Github-Action@master
+        env:
+          COMMIT_SHA: ${{ github.sha }}
         with:
           name: getmeili/meilisearch
           username: ${{ secrets.DOCKER_USERNAME }}

.github/workflows/release-drafter.yml

@@ -0,0 +1,16 @@
+name: Release Drafter
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  update_release_draft:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: release-drafter/release-drafter@v5
+        with:
+          config-name: release-draft-template.yml
+        env:
+          GITHUB_TOKEN: ${{ secrets.RELEASE_DRAFTER_TOKEN }}

.github/workflows/rust.yml

@@ -0,0 +1,71 @@
+name: Rust
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    # trying and staging branches are for Bors config
+    branches:
+      - trying
+      - staging
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  tests:
+    name: Tests on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, macos-latest, windows-latest]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Cache dependencies
+      uses: Swatinem/rust-cache@v1.3.0
+    - name: Run cargo check without any default features
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: --locked --release --no-default-features
+    - name: Run cargo test
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --locked --release
+
+  clippy:
+    name: Run Clippy
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          components: clippy
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v1.3.0
+      - name: Run cargo clippy
+        uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: --all-targets -- --deny warnings
+
+  fmt:
+    name: Run Rustfmt
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          components: rustfmt
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v1.3.0
+      - name: Run cargo fmt
+        run: cargo fmt --all -- --check

.github/workflows/test.yml

@@ -1,25 +0,0 @@
----
-on: [pull_request]
-
-name: Test binaries with cargo test
-
-jobs:
-  check:
-    name: Test on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-
-    steps:
-    - uses: actions/checkout@v1
-    - uses: actions-rs/toolchain@v1
-      with:
-        profile: minimal
-        toolchain: stable
-        override: true
-    - name: Run cargo test
-      uses: actions-rs/cargo@v1
-      with:
-        command: test
-        args: --locked --release

.gitignore

@@ -1,8 +1,9 @@
 /target
-meilisearch-core/target
 **/*.csv
 **/*.json_lines
 **/*.rs.bk
 /*.mdb
 /query-history.txt
 /data.ms
+/snapshots
+/dumps

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at bonjour@meilisearch.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

CONTRIBUTING.md

@@ -0,0 +1,91 @@
+# Contributing
+
+First, thank you for contributing to MeiliSearch! The goal of this document is to provide everything you need to start contributing to MeiliSearch.
+
+- [Hacktoberfest](#hacktoberfest)
+- [Assumptions](#assumptions)
+- [How to Contribute](#how-to-contribute)
+- [Development Workflow](#development-workflow)
+- [Git Guidelines](#git-guidelines)
+
+## Hacktoberfest
+
+It's [Hacktoberfest month](https://blog.meilisearch.com/contribute-hacktoberfest-2021/)! 🥳
+
+🚀 If your PR gets accepted it will count into your participation to Hacktoberfest!
+
+✅ To be accepted it has either to have been merged, approved or tagged with the `hacktoberfest-accepted` label.
+
+🧐 Don't forget to check the [quality standards](https://hacktoberfest.digitalocean.com/resources/qualitystandards)! Low-quality PRs might get marked as `spam` or `invalid`, and will not count toward your participation in Hacktoberfest.
+
+## Assumptions
+
+1. **You're familiar with [Github](https://github.com) and the [Pull Requests](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests)(PR) workflow.**
+2. **You've read the MeiliSearch [documentation](https://docs.meilisearch.com).**
+3. **You know about the [MeiliSearch community](https://docs.meilisearch.com/learn/what_is_meilisearch/contact.html).
+   Please use this for help.**
+
+## How to Contribute
+
+1. Ensure your change has an issue! Find an
+   [existing issue](https://github.com/meilisearch/meilisearch/issues/) or [open a new issue](https://github.com/meilisearch/meilisearch/issues/new).
+   * This is where you can get a feel if the change will be accepted or not.
+2. Once approved, [fork the MeiliSearch repository](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) in your own Github account.
+3. [Create a new Git branch](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-and-deleting-branches-within-your-repository)
+4. Review the [Development Workflow](#development-workflow) section that describes the steps to maintain the repository.
+5. Make your changes on your branch.
+6. [Submit the branch as a Pull Request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request-from-a-fork) pointing to the `main` branch of the MeiliSearch repository. A maintainer should comment and/or review your Pull Request within a few days. Although depending on the circumstances, it may take longer.
+
+## Development Workflow
+
+### Setup and run MeiliSearch
+
+```bash
+cargo run --release
+```
+
+We recommend using the `--release` flag to test the full performance of MeiliSearch.
+
+### Test
+
+```bash
+cargo test
+```
+
+If you get a "Too many open files" error you might want to increase the open file limit using this command:
+
+```bash
+ulimit -Sn 3000
+```
+
+## Git Guidelines
+
+### Git Branches
+
+All changes must be made in a branch and submitted as PR.
+
+We do not enforce any branch naming style, but please use something descriptive of your changes.
+
+### Git Commits
+
+As minimal requirements, your commit message should:
+- be capitalized
+- not finish by a dot or any other punctuation character (!,?)
+- start with a verb so that we can read your commit message this way: "This commit will ...", where "..." is the commit message.
+  e.g.: "Fix the home page button" or "Add more tests for create_index method"
+
+We don't follow any other convention, but if you want to use one, we recommend [the Chris Beams one](https://chris.beams.io/posts/git-commit/).
+
+### Github Pull Requests
+
+Some notes on GitHub PRs:
+
+- All PRs must be reviewed and approved by at least one maintainer.
+- The PR title should be accurate and descriptive of the changes.
+- [Convert your PR as a draft](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/changing-the-stage-of-a-pull-request) if your changes are a work in progress: no one will review it until you pass your PR as ready for review.<br>
+  The draft PRs are recommended when you want to show that you are working on something and make your work visible.
+- The branch related to the PR must be **up-to-date with `main`** before merging. Fortunately, this project uses [Bors](https://github.com/bors-ng/bors-ng) to automatically enforce this requirement without the PR author having to rebase manually.
+
+<hr>
+
+Thank you again for reading this through, we can not wait to begin to work with you if you made your way through this contributing guide ❤️

Cargo.toml

@@ -1,11 +1,10 @@
 [workspace]
 members = [
-    "meilisearch-core",
     "meilisearch-http",
-    "meilisearch-schema",
-    "meilisearch-tokenizer",
-    "meilisearch-types",
+    "meilisearch-error",
+    "meilisearch-lib",
 ]
+resolver = "2"
 
-[profile.release]
-debug = true
+[patch.crates-io]
+pest = { git = "https://github.com/pest-parser/pest.git", rev = "51fd1d49f1041f7839975664ef71fe15c7dcaf67" }

Cross.toml

@@ -0,0 +1,7 @@
+[build.env]
+passthrough = [
+    "RUST_BACKTRACE",
+    "CARGO_TERM_COLOR",
+    "RUSTFLAGS",
+    "JEMALLOC_SYS_WITH_LG_PAGE"
+]

Dockerfile

@@ -1,27 +1,54 @@
 # Compile
-FROM    alpine:3.10 AS compiler
+FROM    alpine:3.14 AS compiler
 
-RUN     apk update --quiet
-RUN     apk add curl
-RUN     apk add build-base
+RUN     apk update --quiet \
+        && apk add -q --no-cache curl build-base
 
 RUN     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
 
 WORKDIR /meilisearch
 
-COPY    . .
+COPY    Cargo.lock .
+COPY    Cargo.toml .
+
+COPY    meilisearch-error/Cargo.toml meilisearch-error/
+COPY    meilisearch-http/Cargo.toml meilisearch-http/
+COPY    meilisearch-lib/Cargo.toml meilisearch-lib/
 
 ENV     RUSTFLAGS="-C target-feature=-crt-static"
 
+# Create dummy main.rs files for each workspace member to be able to compile all the dependencies
+RUN     find . -type d -name "meilisearch-*" | xargs -I{} sh -c 'mkdir {}/src; echo "fn main() { }" > {}/src/main.rs;'
+# Use `cargo build` instead of `cargo vendor` because we need to not only download but compile dependencies too
+RUN     $HOME/.cargo/bin/cargo build --release
+# Cleanup dummy main.rs files
+RUN     find . -path "*/src/main.rs" -delete
+
+ARG     COMMIT_SHA
+ARG     COMMIT_DATE
+ENV     COMMIT_SHA=${COMMIT_SHA} COMMIT_DATE=${COMMIT_DATE}
+
+COPY    . .
 RUN     $HOME/.cargo/bin/cargo build --release
 
 # Run
-FROM    alpine:3.10
+FROM    alpine:3.14
 
-RUN     apk update --quiet
-RUN     apk add libgcc
+ARG     USER=meili
+ENV     HOME /home/${USER}
+ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
+ENV     MEILI_SERVER_PROVIDER docker
 
+# download runtime deps as root and create ${USER}
+RUN     apk update --quiet \
+        && apk add -q --no-cache libgcc tini curl \
+        && adduser -D ${USER}
+WORKDIR ${HOME}
+USER    ${USER}
+# copy file as ${USER} to ${HOME}
 COPY    --from=compiler /meilisearch/target/release/meilisearch .
 
-ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
+EXPOSE  7700/tcp
+
+ENTRYPOINT ["tini", "--"]
 CMD     ./meilisearch

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019-2020 Meili SAS
+Copyright (c) 2019-2021 Meili SAS
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Procfile

@@ -1 +0,0 @@
-web: ./target/release/meilisearch --http-addr=0.0.0.0:$PORT

README.md

@@ -1,46 +1,85 @@
-# MeiliSearch
+<p align="center">
+  <img src="assets/logo.svg" alt="MeiliSearch" width="200" height="200" />
+</p>
 
-[![Build Status](https://github.com/meilisearch/MeiliSearch/workflows/Cargo%20test/badge.svg)](https://github.com/meilisearch/MeiliSearch/actions)
-[![dependency status](https://deps.rs/repo/github/meilisearch/MeiliSearch/status.svg)](https://deps.rs/repo/github/meilisearch/MeiliSearch)
-[![License](https://img.shields.io/badge/license-MIT-informational)](https://github.com/meilisearch/MeiliSearch/blob/master/LICENSE)
+<h1 align="center">MeiliSearch</h1>
 
-⚡ Ultra relevant and instant full-text search API 🔍
+<h4 align="center">
+  <a href="https://www.meilisearch.com">Website</a> |
+  <a href="https://roadmap.meilisearch.com/tabs/1-under-consideration">Roadmap</a> |
+  <a href="https://blog.meilisearch.com">Blog</a> |
+  <a href="https://fr.linkedin.com/company/meilisearch">LinkedIn</a> |
+  <a href="https://twitter.com/meilisearch">Twitter</a> |
+  <a href="https://docs.meilisearch.com">Documentation</a> |
+  <a href="https://docs.meilisearch.com/faq/">FAQ</a>
+</h4>
 
-MeiliSearch is a powerful, fast, open-source, easy to use, and deploy search engine. The search and indexation are fully customizable and handles features like typo-tolerance, filters, and synonyms.
-For more [details about those features, go to our documentation](https://docs.meilisearch.com/).
+<p align="center">
+  <a href="https://github.com/meilisearch/MeiliSearch/actions"><img src="https://github.com/meilisearch/MeiliSearch/workflows/Cargo%20test/badge.svg" alt="Build Status"></a>
+  <a href="https://deps.rs/repo/github/meilisearch/MeiliSearch"><img src="https://deps.rs/repo/github/meilisearch/MeiliSearch/status.svg" alt="Dependency status"></a>
+  <a href="https://github.com/meilisearch/MeiliSearch/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-informational" alt="License"></a>
+  <a href="https://slack.meilisearch.com"><img src="https://img.shields.io/badge/slack-MeiliSearch-blue.svg?logo=slack" alt="Slack"></a>
+  <a href="https://github.com/meilisearch/MeiliSearch/discussions" alt="Discussions"><img src="https://img.shields.io/badge/github-discussions-red" /></a>
+  <a href="https://app.bors.tech/repositories/26457"><img src="https://bors.tech/images/badge_small.svg" alt="Bors enabled"></a>
+</p>
 
-[![crates.io demo gif](misc/crates-io-demo.gif)](https://crates.meilisearch.com)
-> Meili helps the Rust community find crates on [crates.meilisearch.com](https://crates.meilisearch.com)
+<p align="center">⚡ Lightning Fast, Ultra Relevant, and Typo-Tolerant Search Engine 🔍</p>
 
-## Features
-* Search as-you-type experience (answers < 50ms)
+**MeiliSearch** is a powerful, fast, open-source, easy to use and deploy search engine. Both searching and indexing are highly customizable. Features such as typo-tolerance, filters, and synonyms are provided out-of-the-box.
+For more information about features go to [our documentation](https://docs.meilisearch.com/).
+
+<p align="center">
+  <img src="assets/trumen-fast.gif" alt="Web interface gif" />
+</p>
+
+## ✨ Features
+* Search-as-you-type experience (answers < 50 milliseconds)
 * Full-text search
-* Typo tolerant (understands typos and spelling mistakes)
-* Supports Kanji
-* Supports Synonym
+* Typo tolerant (understands typos and misspelling)
+* Faceted search and filters
+* Supports hanzi (Chinese characters)
+* Supports synonyms
 * Easy to install, deploy, and maintain
-* Whole documents returned
+* Whole documents are returned
 * Highly customizable
-* RESTfull API
+* RESTful API
 
-## Quick Start
+## Getting started
 
 ### Deploy the Server
 
-#### Run it using Docker
-
-```bash
-docker run -it -p 7700:7700 --rm getmeili/meilisearch
-```
-
-#### Installation using Homebrew
+#### Homebrew (Mac OS)
 
 ```bash
 brew update && brew install meilisearch
 meilisearch
 ```
 
-#### Installation using APT
+#### Docker
+
+```bash
+docker run -p 7700:7700 -v "$(pwd)/data.ms:/data.ms" getmeili/meilisearch
+```
+
+#### Announcing a cloud-hosted MeiliSearch
+
+Join the closed beta by filling out this [form](https://meilisearch.typeform.com/to/FtnzvZfh).
+
+#### Try MeiliSearch in our Sandbox
+
+Create a MeiliSearch instance in [MeiliSearch Sandbox](https://sandbox.meilisearch.com/). This instance is free, and will be active for 48 hours.
+
+#### Run on Digital Ocean
+
+[![DigitalOcean Marketplace](assets/do-btn-blue.svg)](https://marketplace.digitalocean.com/apps/meilisearch?action=deploy&refcode=7c67bd97e101)
+
+#### Deploy on Platform.sh
+
+<a href="https://console.platform.sh/projects/create-project?template=https://raw.githubusercontent.com/platformsh/template-builder/master/templates/meilisearch/.platform.template.yaml&utm_content=meilisearch&utm_source=github&utm_medium=button&utm_campaign=deploy_on_platform">
+    <img src="https://platform.sh/images/deploy/lg-blue.svg" alt="Deploy on Platform.sh" width="180px" />
+</a>
+
+#### APT (Debian & Ubuntu)
 
 ```bash
 echo "deb [trusted=yes] https://apt.fury.io/meilisearch/ /" > /etc/apt/sources.list.d/fury.list
@@ -48,20 +87,16 @@ apt update && apt install meilisearch-http
 meilisearch
 ```
 
-#### Download the binary
+#### Download the binary (Linux & Mac OS)
 
 ```bash
 curl -L https://install.meilisearch.com | sh
 ./meilisearch
 ```
 
-#### Run it on heroku
-
-[![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy?template=https://github.com/meilisearch/MeiliSearch)
-
 #### Compile and run it from sources
 
-If you have the Rust toolchain already installed, you can compile from the source
+If you have the latest stable Rust toolchain installed on your local system, clone the repository and change it to your working directory.
 
 ```bash
 git clone https://github.com/meilisearch/MeiliSearch.git
@@ -71,21 +106,13 @@ cargo run --release
 
 ### Create an Index and Upload Some Documents
 
-We provide a movie dataset that you can use for testing purposes.
+Let's create an index! If you need a sample dataset, use [this movie database](https://www.notion.so/meilisearch/A-movies-dataset-to-test-Meili-1cbf7c9cfa4247249c40edfa22d7ca87#b5ae399b81834705ba5420ac70358a65). You can also find it in the `datasets/` directory.
 
 ```bash
 curl -L 'https://bit.ly/2PAcw9l' -o movies.json
 ```
 
-MeiliSearch can serve multiple indexes, with different kinds of documents,
-therefore, it is required to create the index before sending documents to it.
-
-```bash
-curl -i -X POST 'http://127.0.0.1:7700/indexes' --data '{ "name": "Movies", "uid": "movies" }'
-```
-
-Now that the server knows about our brand new index, we can send it data.
-We provided you a small dataset that is available in the `datasets/` directory.
+Now, you're ready to index some data.
 
 ```bash
 curl -i -X POST 'http://127.0.0.1:7700/indexes/movies/documents' \
@@ -97,8 +124,9 @@ curl -i -X POST 'http://127.0.0.1:7700/indexes/movies/documents' \
 
 #### In command line
 
-The search engine is now aware of our documents and can serve those via our HTTP server again.
-The [`jq` command-line tool](https://stedolan.github.io/jq/) can significantly help you read the server responses.
+The search engine is now aware of your documents and can serve those via an HTTP server.
+
+The [`jq` command-line tool](https://stedolan.github.io/jq/) can greatly help you read the server responses.
 
 ```bash
 curl 'http://127.0.0.1:7700/indexes/movies/search?q=botman+robin&limit=2' | jq
@@ -111,85 +139,67 @@ curl 'http://127.0.0.1:7700/indexes/movies/search?q=botman+robin&limit=2' | jq
       "id": "415",
       "title": "Batman & Robin",
       "poster": "https://image.tmdb.org/t/p/w1280/79AYCcxw3kSKbhGpx1LiqaCAbwo.jpg",
-      "overview": "Along with crime-fighting partner Robin and new recruit Batgirl...",
-      "release_date": "1997-06-20",
+      "overview": "Along with crime-fighting partner Robin and new recruit Batgirl, Batman battles the dual threat of frosty genius Mr. Freeze and homicidal horticulturalist Poison Ivy. Freeze plans to put Gotham City on ice, while Ivy tries to drive a wedge between the dynamic duo.",
+      "release_date": 866768400
     },
     {
       "id": "411736",
       "title": "Batman: Return of the Caped Crusaders",
       "poster": "https://image.tmdb.org/t/p/w1280/GW3IyMW5Xgl0cgCN8wu96IlNpD.jpg",
-      "overview": "Adam West and Burt Ward returns to their iconic roles of Batman and Robin...",
-      "release_date": "2016-10-08",
+      "overview": "Adam West and Burt Ward returns to their iconic roles of Batman and Robin. Featuring the voices of Adam West, Burt Ward, and Julie Newmar, the film sees the superheroes going up against classic villains like The Joker, The Riddler, The Penguin and Catwoman, both in Gotham City… and in space.",
+      "release_date": 1475888400
     }
   ],
-  "offset": 0,
+  "nbHits": 8,
+  "exhaustiveNbHits": false,
+  "query": "botman robin",
   "limit": 2,
-  "processingTimeMs": 1,
-  "query": "botman robin"
+  "offset": 0,
+  "processingTimeMs": 2
 }
 ```
 
-#### With the Web Interface
+#### Use the Web Interface
 
-MeiliSearch provides a simple web interface containing a search bar in order to quickly test the instant search experience with a given set of documents.
+We also deliver an **out-of-the-box [web interface](https://github.com/meilisearch/mini-dashboard)** in which you can test MeiliSearch interactively.
 
-This web interface is available in your browser at the root of the server. The default URL is [http://127.0.0.1:7700](http://127.0.0.1:7700).
+You can access the web interface in your web browser at the root of the server. The default URL is [http://127.0.0.1:7700](http://127.0.0.1:7700). All you need to do is open your web browser and enter MeiliSearch’s address to visit it. This will lead you to a web page with a search bar that will allow you to search in the selected index.
 
-### Documentation
+| [See the gif above](#demo)
 
-Now, that you have a running MeiliSearch, you can learn more and tune your search engine using [the documentation](https://docs.meilisearch.com).
+## Documentation
 
-## How it works
-
-MeiliSearch uses [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database) as the internal key-value store. The key-value store allows us to handle updates and queries with small memory and CPU overheads. The whole ranking system is [data oriented](https://github.com/meilisearch/MeiliSearch/issues/82) and provides great performances.
-
-You can [read the deep dive](deep-dive.md) if you want more information on the engine; it describes the whole process of generating updates and handling queries. Also, you can take a look at the [typos and ranking rules](typos-ranking-rules.md) if you want to know the default rules used to sort the documents.
-
-### Technical features
-
-- Provides [6 default ranking criteria](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/criterion/mod.rs#L106-L111) used to [bucket sort](https://en.wikipedia.org/wiki/Bucket_sort) documents
-- Accepts [custom criteria](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/criterion/mod.rs#L20-L29) and can apply them in any custom order
-- Support [ranged queries](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/query_builder.rs#L342), useful for paginating results
-- Can [distinct](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/query_builder.rs#L324-L329) and [filter](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/query_builder.rs#L313-L318) returned documents based on context defined rules
-- Searches for [concatenated](https://github.com/meilisearch/MeiliSearch/pull/164) and [splitted query words](https://github.com/meilisearch/MeiliSearch/pull/232) to improve the search quality.
-- Can store complete documents or only [user schema specified fields](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/datasets/movies/schema.toml)
-- The [default tokenizer](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-tokenizer/src/lib.rs) can index latin and kanji based languages
-- Returns [the matching text areas](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-types/src/lib.rs#L49-L65), useful to highlight matched words in results
-- Accepts query time search config like the [searchable attributes](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/query_builder.rs#L331-L336)
-- Supports [runtime incremental indexing](https://github.com/meilisearch/MeiliSearch/blob/3ea5aa18a209b6973b921542d46a79e1c753c163/meilisearch-core/src/store/mod.rs#L143-L212)
-
-## Performances
-
-With a dataset composed of _100 353_ documents with _352_ attributes each and _3_ of them indexed.
-So more than _300 000_ fields indexed for _35 million_ stored we can handle more than _2.8k req/sec_ with an average response time of _9 ms_ on an Intel i7-7700 (8) @ 4.2GHz.
-
-Requests are made using [wrk](https://github.com/wg/wrk) and scripted to simulate real users' queries.
-
-```
-Running 10s test @ http://localhost:2230
-  2 threads and 25 connections
-  Thread Stats   Avg      Stdev     Max   +/- Stdev
-    Latency     9.52ms    7.61ms  99.25ms   84.58%
-    Req/Sec     1.41k   119.11     1.78k    64.50%
-  28080 requests in 10.01s, 7.42MB read
-Requests/sec:   2806.46
-Transfer/sec:    759.17KB
-```
-
-We also indexed a dataset containing something like _12 millions_ cities names in _24 minutes_ on a machine with _8 cores_, _64 GB of RAM_, and a _300 GB NMVe_ SSD.<br/>
-The resulting database was _16 GB_ and search results were between _30 ms_ and _4 seconds_ for short prefix queries.
-
-### Notes
-
-With Rust 1.32 the allocator has been [changed to use the system allocator](https://blog.rust-lang.org/2019/01/17/Rust-1.32.0.html#jemalloc-is-removed-by-default).
-We have seen much better performances when [using jemalloc as the global allocator](https://github.com/alexcrichton/jemallocator#documentation).
+Now that your MeiliSearch server is up and running, you can learn more about how to tune your search engine in [the documentation](https://docs.meilisearch.com).
 
 ## Contributing
 
-We will be glad if you submit issues and pull requests. You can help to grow this project and start contributing by checking [issues tagged "good-first-issue"](https://github.com/meilisearch/MeiliSearch/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22). It is a good start!
+Hey! We're glad you're thinking about contributing to MeiliSearch! Feel free to pick an [issue labeled as `good first issue`](https://github.com/meilisearch/MeiliSearch/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22), and to ask any question you need. Some points might not be clear and we are available to help you!
 
-### Analytic Events
+Also, we recommend following the [CONTRIBUTING](./CONTRIBUTING.md) to create your PR.
 
-We send events to our Amplitude instance to be aware of the number of people who use MeiliSearch.<br/>
-We only send the platform on which the server runs once by day. No other information is sent.<br/>
-If you do not want us to send events, you can disable these analytics by using the `MEILI_NO_ANALYTICS` env variable.
+## Core engine and tokenizer
+
+The code in this repository is only concerned with managing multiple indexes, handling the update store, and exposing an HTTP API.
+
+Search and indexation are the domain of our core engine, [`milli`](https://github.com/meilisearch/milli), while tokenization is handled by [our `tokenizer` library](https://github.com/meilisearch/tokenizer/).
+## Telemetry
+
+MeiliSearch collects anonymous data regarding general usage.
+This helps us better understand developers' usage of MeiliSearch features.
+
+To find out more on what information we're retrieving, please see our documentation on [Telemetry](https://docs.meilisearch.com/learn/what_is_meilisearch/telemetry.html).
+
+This program is optional, you can disable these analytics by using the `MEILI_NO_ANALYTICS` env variable.
+
+## Feature request
+
+The feature requests are not managed in this repository. Please visit our [dedicated repository](https://github.com/meilisearch/product) to see our work about the MeiliSearch product.
+
+If you have a feature request or any feedback about an existing feature, please open [a discussion](https://github.com/meilisearch/product/discussions).
+Also, feel free to participate in the current discussions, we are looking forward to reading your comments.
+
+## 💌 Contact
+
+Please visit [this page](https://docs.meilisearch.com/learn/what_is_meilisearch/contact.html#contact-us).
+
+MeiliSearch is developed by [Meili](https://www.meilisearch.com), a young company. To know more about us, you can [read our blog](https://blog.meilisearch.com). Any suggestion or feedback is highly appreciated. Thank you for your support!

SECURITY.md

@@ -0,0 +1,33 @@
+# Security
+
+MeiliSearch takes the security of our software products and services seriously.
+
+If you believe you have found a security vulnerability in any MeiliSearch-owned repository, please report it to us as described below.
+
+## Suported versions
+
+As long as we are pre-v1.0, only the latest version of MeiliSearch will be supported with security updates. 
+
+## Reporting security issues
+
+⚠️ Please do not report security vulnerabilities through public GitHub issues. ⚠️ 
+
+Instead, please kindly email us at security@meilisearch.com
+
+Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
+
+- Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+- Full paths of source file(s) related to the manifestation of the issue
+- The location of the affected source code (tag/branch/commit or direct URL)
+- Any special configuration required to reproduce the issue
+- Step-by-step instructions to reproduce the issue
+- Proof-of-concept or exploit code (if possible)
+- Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+You will receive a response from us within 72 hours. If the issue is confirmed, we will release a patch as soon as possible depending on complexity.
+
+## Preferred languages
+
+We prefer all communications to be in English.

app.json

@@ -1,16 +0,0 @@
-{
-  "name": "MeiliSearch",
-  "description": "Ultra relevant, instant and typo-tolerant full-text search API",
-  "keywords": [
-    "search-engine",
-    "instant search",
-    "search API"
-  ],
-  "website": "https://docs.meilisearch.com/",
-  "repository": "https://github.com/meilisearch/MeiliSearch",
-  "buildpacks": [
-    {
-      "url": "https://github.com/emk/heroku-buildpack-rust"
-    }
-  ]
-}

assets/do-btn-blue.svg

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="200px" height="42px" viewBox="0 0 200 42" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 52.5 (67469) - http://www.bohemiancoding.com/sketch -->
+    <title>do-btn-blue</title>
+    <desc>Created with Sketch.</desc>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="Partner-welcome-kit-Copy-3" transform="translate(-651.000000, -762.000000)">
+            <g id="do-btn-blue" transform="translate(651.000000, 763.000000)">
+                <rect id="Rectangle-Copy" fill="#0069FF" x="0" y="0" width="200" height="40" rx="6"></rect>
+                <path d="M45,0 L45,40" id="Line-2" stroke="#FFFFFF" stroke-linecap="square"></path>
+                <g id="DO_Logo_horizontal_blue-Copy" transform="translate(13.000000, 11.000000)" fill="#FFFFFF">
+                    <path d="M10.0098493,20 L10.0098493,16.1262429 C14.12457,16.1262429 17.2897398,12.0548452 15.7269372,7.74627862 C15.1334679,6.14538921 13.8674,4.86072487 12.2650328,4.28756693 C7.952489,2.72620566 3.87733294,5.88845634 3.87733294,9.99938223 C3.87733294,9.99938223 3.87733294,9.99938223 3.87733294,9.99938223 L0,9.99938223 C0,3.45747613 6.3303395,-1.64165309 13.1948014,0.492866119 C16.2017127,1.42177726 18.57559,3.81322933 19.5053586,6.79760341 C21.6418482,13.6754986 16.5577943,20 10.0098493,20 Z" id="XMLID_49_"></path>
+                    <polygon id="XMLID_47_" points="9.56521739 15.6521739 6.08695652 15.6521739 6.08695652 12.173913 6.08695652 12.173913 9.56521739 12.173913 9.56521739 12.173913"></polygon>
+                    <polygon id="XMLID_46_" points="6.08695652 19.1304348 3.47826087 19.1304348 3.47826087 19.1304348 3.47826087 16.5217391 6.08695652 16.5217391"></polygon>
+                    <polygon id="XMLID_45_" points="3.47826087 16.5217391 0.869565217 16.5217391 0.869565217 16.5217391 0.869565217 13.9130435 0.869565217 13.9130435 3.47826087 13.9130435 3.47826087 13.9130435"></polygon>
+                </g>
+                <text id="Create-a-Droplet-Copy" font-family="Sailec-Medium, Sailec" font-size="16" font-weight="400" fill="#FFFFFF">
+                    <tspan x="58" y="26">Create a Droplet</tspan>
+                </text>
+            </g>
+        </g>
+    </g>
+</svg>

assets/logo.svg

@@ -0,0 +1,17 @@
+<svg width="360" height="360" viewBox="0 0 360 360" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g id="logo_main">
+<rect id="Rectangle" x="107.333" y="0.150146" width="274.315" height="274.315" rx="98.8334" transform="rotate(23 107.333 0.150146)" fill="url(#paint0_linear)"/>
+<path id="Rectangle_2" fill-rule="evenodd" clip-rule="evenodd" d="M61.3296 230.199C46.2224 194.608 38.6688 176.813 38.208 160.329C37.5286 136.025 47.0175 112.539 64.3891 95.5282C76.1718 83.9904 93.9669 76.4368 129.557 61.3296C165.147 46.2224 182.943 38.6688 199.427 38.208C223.731 37.5286 247.217 47.0175 264.228 64.3891C275.766 76.1718 283.319 93.9669 298.426 129.557C313.534 165.147 321.087 182.943 321.548 199.427C322.227 223.731 312.738 247.217 295.367 264.228C283.584 275.766 265.789 283.319 230.199 298.426C194.608 313.534 176.813 321.087 160.329 321.548C136.025 322.227 112.539 312.738 95.5282 295.367C83.9903 283.584 76.4368 265.789 61.3296 230.199Z" fill="url(#paint1_linear)"/>
+<path id="m" fill-rule="evenodd" clip-rule="evenodd" d="M219.568 130.748C242.363 130.748 259.263 147.451 259.263 174.569V229.001H227.232V179.678C227.232 166.119 220.747 159.634 210.136 159.634C205.223 159.634 200.311 161.796 195.595 167.494C195.791 169.852 195.988 172.21 195.988 174.569V229.001H164.154V179.678C164.154 166.119 157.472 159.634 147.057 159.634C142.145 159.634 137.429 161.992 132.712 168.084V229.001H100.878V133.695H132.712V139.394C139.197 133.892 145.878 130.748 156.49 130.748C168.477 130.748 178.695 135.267 185.769 143.52C195.791 134.678 205.42 130.748 219.568 130.748Z" fill="white"/>
+</g>
+<defs>
+<linearGradient id="paint0_linear" x1="-13.6248" y1="129.208" x2="244.49" y2="403.522" gradientUnits="userSpaceOnUse">
+<stop stop-color="#E41359"/>
+<stop offset="1" stop-color="#F23C79"/>
+</linearGradient>
+<linearGradient id="paint1_linear" x1="11.0088" y1="111.65" x2="111.65" y2="348.747" gradientUnits="userSpaceOnUse">
+<stop stop-color="#24222F"/>
+<stop offset="1" stop-color="#2B2937"/>
+</linearGradient>
+</defs>
+</svg>

bors.toml

@@ -0,0 +1,10 @@
+status = [
+    'Tests on ubuntu-18.04',
+    'Tests on macos-latest',
+    'Tests on windows-latest',
+    'Run Clippy',
+    'Run Rustfmt'
+]
+pr_status = ['Milestone Check']
+# 3 hours timeout
+timeout-sec = 10800

datasets/movies/README.md

@@ -1 +0,0 @@
-_datas in movies.csv are from https://www.themoviedb.org/_

datasets/movies/settings.json

@@ -1,11 +0,0 @@
-{
-    "primaryKey": "id",
-    "searchableAttributes": ["title", "overview"],
-    "displayedAttributes": [
-        "id",
-        "title",
-        "overview",
-        "release_date",
-        "poster"
-    ]
-}

deep-dive.md

@@ -1,95 +0,0 @@
-# A deep dive in MeiliSearch
-
-On the 15 of May 2019.
-
-MeiliSearch is a full text search engine based on a final state transducer named [fst](https://github.com/BurntSushi/fst) and a key-value store named [sled](https://github.com/spacejam/sled). The goal of a search engine is to store data and to respond to queries as accurate and fast as possible. To achieve this it must save the matching words in an [inverted index](https://en.wikipedia.org/wiki/Inverted_index).
-
-<!-- MarkdownTOC autolink="true" -->
-
-- [Where is the data stored?](#where-is-the-data-stored)
-- [What does the key-value store contains?](#what-does-the-key-value-store-contains)
-    - [The inverted word index](#the-inverted-word-index)
-        - [A final state transducer](#a-final-state-transducer)
-        - [Document indexes](#document-indexes)
-    - [The schema](#the-schema)
-    - [Document attributes](#document-attributes)
-- [How is a request processed?](#how-is-a-request-processed)
-    - [Query lexemes](#query-lexemes)
-    - [Automatons and query index](#automatons-and-query-index)
-    - [Sort by criteria](#sort-by-criteria)
-
-<!-- /MarkdownTOC -->
-
-## Where is the data stored?
-
-MeiliSearch is entirely backed by a key-value store like any good database (i.e. Postgres, MySQL). This brings a great flexibility in the way documents can be stored and updates handled along time.
-
-[sled will brings some](https://github.com/spacejam/sled/tree/434533332a3f485e6d2e467023be0a0b55d3a1af#plans) of the [A.C.I.D. properties](https://en.wikipedia.org/wiki/ACID_(computer_science)) to help us be sure the saved data is consistent.
-
-
-
-## What does the key-value store contains?
-
-It contain the inverted word index, the schema and the documents fields.
-
-### The inverted word index
-
-[The inverted word index](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/words_index.rs) is a sled Tree dedicated to store and give access to all documents that contains a specific word. The information stored under the word is simply a big ordered array of where in the document the word has been found. In other word, a big list of [`DocIndex`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/lib.rs#L35-L51).
-
-#### A final state transducer
-
-_...also abbreviated fst_
-
-This is the first entry point of the engine, you can read more about how it work with the beautiful blog post of @BurntSushi, [Index 1,600,000,000 Keys with Automata and Rust](https://blog.burntsushi.net/transducers/).
-
-To make it short it is a powerful way to store all the words that are present in the indexed documents. You construct it by giving it all the words you want to index. When you want to search in it you can provide any automaton you want, in MeiliSearch [a custom levenshtein automaton](https://github.com/tantivy-search/levenshtein-automata/) is used.
-
-#### Document indexes
-
-The `fst` will only return the words that match with the search automaton but the goal of the search engine is to retrieve all matches in all the documents when a query is made. You want it to return some sort of position in an attribute in a document, an information about where the given word matched.
-
-To make it possible we retrieve all of the `DocIndex` corresponding to all the matching words in the fst, we use the [`WordsIndex`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/words_index.rs#L11-L21) Tree to get the `DocIndexes` corresponding the words.
-
-### The schema
-
-The schema is a data structure that represents which documents attributes should be stored and which should be indexed. It is stored under a the [`MainIndex`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/main_index.rs#L12) Tree and given to MeiliSearch only at the creation of an index.
-
-Each document attribute is associated to a unique 16 bit number named [`SchemaAttr`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/schema.rs#L186).
-
-In the future, this schema type could be given along with updates, the database could be able to handled a new schema and reindex the database according to the new one.
-
-### Document attributes
-
-When the engine handle a query the result that the requester want is a document, not only the [`Matches`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/lib.rs#L62-L88) associated to it, fields of the original document must be returned too.
-
-So MeiliSearch again uses the power of the underlying key-value store and save the documents attributes marked as _STORE_ in the schema. The dedicated Tree for this information is the [`DocumentsIndex`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/documents_index.rs#L11).
-
-When a document field is saved in the key-value store its value is binary encoded using [message pack](https://github.com/3Hren/msgpack-rust), so a document must be serializable using serde.
-
-
-
-## How is a request processed?
-
-Now that we have our inverted index we are able to return results based on a query. In the MeiliSearch universe a query is a simple string containing words.
-
-### Query lexemes
-
-The first step to be able to call the underlying structures is to split the query in words, for that we use a [custom tokenizer](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-tokenizer/src/lib.rs#L82-L84). Note that a tokenizer is specialized for a human language, this is the hard part.
-
-### Automatons and query index
-
-So to query the fst we need an automaton, in MeiliSearch we use a [levenshtein automaton](https://en.wikipedia.org/wiki/Levenshtein_automaton), this automaton is constructed using a string and a maximum distance. According to the [Algolia's blog post](https://blog.algolia.com/inside-the-algolia-engine-part-3-query-processing/#algolia%e2%80%99s-way-of-searching-for-alternatives) we [created the DFAs](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/automaton.rs#L59-L78) with different settings.
-
-Thanks to the power of the fst library [it is possible to union multiple automatons](https://docs.rs/fst/0.3.2/fst/map/struct.OpBuilder.html#method.union) on the same fst set. The `Stream` is able to return all the matching words. We use these words to find the whole list of `DocIndexes` associated.
-
-With all these informations it is possible [to reconstruct a list of all the `DocIndexes` associated](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/query_builder.rs#L103-L130) with the words queried.
-
-### Sort by criteria
-
-Now that we are able to get a big list of [DocIndexes](https://github.com/Kerollmops/MeiliSearch/blob/550dc1e99224e386516877450320f694947332d4/src/lib.rs#L21-L36) it is not enough to sort them by criteria, we need more informations like the levenshtein distance or the fact that a query word match exactly the word stored in the fst. So [we stuff it a little bit](https://github.com/Kerollmops/MeiliSearch/blob/550dc1e99224e386516877450320f694947332d4/src/rank/query_builder.rs#L86-L93), and aggregate all these [Matches](https://github.com/Kerollmops/MeiliSearch/blob/550dc1e99224e386516877450320f694947332d4/src/lib.rs#L47-L74) for each document. This way it will be easy to sort a simple vector of document using a bunch of functions.
-
-With this big list of documents and associated matches [we are able to sort only the part of the slice that we want](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/query_builder.rs#L160-L188) using bucket sorting. [Each criterion](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/criterion/mod.rs#L95-L101) is evaluated on each subslice without copy, thanks to [GroupByMut](https://docs.rs/slice-group-by/0.2.4/slice_group_by/) which, I hope [will soon be merged](https://github.com/rust-lang/rfcs/pull/2477).
-
-Note that it is possible to customize the criteria used by using the `QueryBuilder::with_criteria` constructor, this way you can implement some custom ranking based on the document attributes using the appropriate structure and the [`document` method](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/index.rs#L86).
-
-At this point, MeiliSearch work is over 🎉

download-latest.sh

@@ -6,8 +6,9 @@ GREEN='\033[32m'
 DEFAULT='\033[0m'
 
 # GLOBALS
-GREP_SEMVER_REGEXP='\"v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\"' # i.e. "v[number].[number].[number]"
-BINARY_NAME='meilisearch'
+GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number]
+
+# FUNCTIONS
 
 # semverParseInto and semverLT from https://github.com/cloudflare/semver_bash/blob/master/semver.sh
 
@@ -20,7 +21,7 @@ semverParseInto() {
     eval $2=`echo $1 | sed -e "s#$RE#\1#"`
     #MINOR
     eval $3=`echo $1 | sed -e "s#$RE#\2#"`
-    #MINOR
+    #PATCH
     eval $4=`echo $1 | sed -e "s#$RE#\3#"`
     #SPECIAL
     eval $5=`echo $1 | sed -e "s#$RE#\4#"`
@@ -50,13 +51,13 @@ semverLT() {
     if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then
         return 0
     fi
-    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+    if [ "_$SPECIAL_A"  == '_' ] && [ "_$SPECIAL_B"  == '_' ] ; then
         return 1
     fi
-    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  != "_" ] ; then
+    if [ "_$SPECIAL_A"  == '_' ] && [ "_$SPECIAL_B"  != '_' ] ; then
         return 1
     fi
-    if [ "_$SPECIAL_A"  != "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+    if [ "_$SPECIAL_A"  != '_' ] && [ "_$SPECIAL_B"  == '_' ] ; then
         return 0
     fi
     if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then
@@ -66,8 +67,101 @@ semverLT() {
     return 1
 }
 
+# Get a token from https://github.com/settings/tokens to increasae rate limit (from 60 to 5000), make sure the token scope is set to 'public_repo'
+# Create GITHUB_PAT enviroment variable once you aquired the token to start using it
+# Returns the tag of the latest stable release (in terms of semver and not of release date)
+get_latest() {
+    temp_file='temp_file' # temp_file needed because the grep would start before the download is over
+
+        if [ -z "$GITHUB_PAT" ]; then
+        curl -s 'https://api.github.com/repos/meilisearch/MeiliSearch/releases' > "$temp_file" || return 1
+    else
+        curl -H "Authorization: token $GITHUB_PAT" -s 'https://api.github.com/repos/meilisearch/MeiliSearch/releases' > "$temp_file" || return 1
+    fi
+
+    releases=$(cat "$temp_file" | \
+        grep -E "tag_name|draft|prerelease" \
+        | tr -d ',"' | cut -d ':' -f2 | tr -d ' ')
+        # Returns a list of [tag_name draft_boolean prerelease_boolean ...]
+        # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false...
+
+    i=0
+    latest=''
+    current_tag=''
+    for release_info in $releases; do
+        if [ $i -eq 0 ]; then # Cheking tag_name
+            if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release
+                current_tag=$release_info
+            else
+                current_tag=''
+            fi
+            i=1
+        elif [ $i -eq 1 ]; then # Checking draft boolean
+            if [ "$release_info" = 'true' ]; then
+                current_tag=''
+            fi
+            i=2
+        elif [ $i -eq 2 ]; then # Checking prerelease boolean
+            if [ "$release_info" = 'true' ]; then
+                current_tag=''
+            fi
+            i=0
+            if [ "$current_tag" != '' ]; then # If the current_tag is valid
+                if [ "$latest" = '' ]; then # If there is no latest yet
+                    latest="$current_tag"
+                else
+                    semverLT $current_tag $latest # Comparing latest and the current tag
+                    if [ $? -eq 1 ]; then
+                        latest="$current_tag"
+                    fi
+                fi
+            fi
+        fi
+    done
+
+    rm -f "$temp_file"
+    echo $latest
+}
+
+# Gets the OS by setting the $os variable
+# Returns 0 in case of success, 1 otherwise.
+get_os() {
+    os_name=$(uname -s)
+    case "$os_name" in
+    'Darwin')
+        os='macos'
+        ;;
+    'Linux')
+        os='linux'
+        ;;
+	'MINGW'*)
+        os='windows'
+        ;;
+    *)
+        return 1
+    esac
+    return 0
+}
+
+# Gets the architecture by setting the $archi variable
+# Returns 0 in case of success, 1 otherwise.
+get_archi() {
+    architecture=$(uname -m)
+    case "$architecture" in
+    'x86_64' | 'amd64' | 'arm64')
+        archi='amd64'
+        ;;
+    'aarch64')
+        archi='armv8'
+        ;;
+    *)
+        return 1
+    esac
+    return 0
+}
+
 success_usage() {
-    printf "$GREEN%s\n$DEFAULT" "MeiliSearch binary successfully downloaded as '$BINARY_NAME' file."
+    printf "$GREEN%s\n$DEFAULT" "MeiliSearch $latest binary successfully downloaded as '$binary_name' file."
     echo ''
     echo 'Run it:'
     echo '    $ ./meilisearch'
@@ -76,54 +170,46 @@ success_usage() {
 }
 
 failure_usage() {
-    printf "$RED%s\n$DEFAULT" 'ERROR: MeiliSearch binary is not available for your OS distribution yet.'
+    printf "$RED%s\n$DEFAULT" 'ERROR: MeiliSearch binary is not available for your OS distribution or your architecture yet.'
     echo ''
     echo 'However, you can easily compile the binary from the source files.'
-    echo 'Follow the steps on the docs: https://docs.meilisearch.com/advanced_guides/binary.html#how-to-compile-meilisearch'
+    echo 'Follow the steps at the page ("Source" tab): https://docs.meilisearch.com/learn/getting_started/installation.html'
 }
 
-# OS DETECTION
-echo 'Detecting OS distribution...'
-os_name=$(uname -s)
-if [ "$os_name" != "Darwin" ]; then
-    os_name=$(cat /etc/os-release | grep '^ID=' | tr -d '"' | cut -d '=' -f 2)
+# MAIN
+latest="$(get_latest)"
+
+if [ "$latest" = '' ]; then
+    echo ''
+    echo 'Impossible to get the latest stable version of MeiliSearch.'
+    echo 'Please let us know about this issue: https://github.com/meilisearch/meilisearch-swift/issues/new/choose'
+    exit 1
 fi
-echo "OS distribution detected: $os_name"
-case "$os_name" in
-'Darwin')
-    os='macos'
-    ;;
-'ubuntu' | 'debian')
-    os='linux'
-    ;;
-*)
+
+if ! get_os; then
     failure_usage
     exit 1
+fi
+
+if ! get_archi; then
+    failure_usage
+    exit 1
+fi
+
+echo "Downloading MeiliSearch binary $latest for $os, architecture $archi..."
+case "$os" in
+    'windows')
+        release_file="meilisearch-$os-$archi.exe"
+		binary_name='meilisearch.exe'
+
+        ;;
+	*)
+		release_file="meilisearch-$os-$archi"
+		binary_name='meilisearch'
+
 esac
-
-# GET LATEST VERSION
-tags=$(curl -s 'https://api.github.com/repos/meilisearch/MeiliSearch/tags' \
-    | grep "$GREP_SEMVER_REGEXP" \
-    | grep 'name' \
-    | tr -d '"' | tr -d ',' | cut -d 'v' -f 2)
-
-latest=""
-for tag in $tags; do
-    if [ "$latest" = "" ]; then
-        latest="$tag"
-    else
-        semverLT $tag $latest
-        if [ $? -eq 1 ]; then
-            latest="$tag"
-        fi
-    fi
-done
-
-# DOWNLOAD THE LATEST
-echo "Downloading MeiliSearch binary v$latest for $os..."
-release_file="meilisearch-$os-amd64"
-link="https://github.com/meilisearch/MeiliSearch/releases/download/v$latest/$release_file"
+link="https://github.com/meilisearch/MeiliSearch/releases/download/$latest/$release_file"
 curl -OL "$link"
-mv "$release_file" "$BINARY_NAME"
-chmod 744 "$BINARY_NAME"
+mv "$release_file" "$binary_name"
+chmod 744 "$binary_name"
 success_usage

meilisearch-core/Cargo.toml

@@ -1,50 +0,0 @@
-[package]
-name = "meilisearch-core"
-version = "0.9.0"
-license = "MIT"
-authors = ["Kerollmops <clement@meilisearch.com>"]
-edition = "2018"
-
-[dependencies]
-arc-swap = "0.4.3"
-bincode = "1.1.4"
-byteorder = "1.3.2"
-chrono = { version = "0.4.9", features = ["serde"] }
-compact_arena = "0.4.0"
-crossbeam-channel = "0.4.0"
-deunicode = "1.0.0"
-env_logger = "0.7.0"
-fst = { version = "0.3.5", default-features = false }
-hashbrown = { version = "0.6.0", features = ["serde"] }
-heed = "0.6.1"
-indexmap = { version = "1.2.0", features = ["serde-1"] }
-intervaltree = "0.2.5"
-itertools = "0.8.2"
-levenshtein_automata = { version = "0.1.1", features = ["fst_automaton"] }
-log = "0.4.8"
-meilisearch-schema = { path = "../meilisearch-schema", version = "0.9.0" }
-meilisearch-tokenizer = { path = "../meilisearch-tokenizer", version = "0.9.0" }
-meilisearch-types = { path = "../meilisearch-types", version = "0.9.0" }
-once_cell = "1.2.0"
-ordered-float = { version = "1.0.2", features = ["serde"] }
-regex = "1.3.1"
-sdset = "0.3.6"
-serde = { version = "1.0.101", features = ["derive"] }
-serde_json = "1.0.41"
-siphasher = "0.3.1"
-slice-group-by = "0.2.6"
-zerocopy = "0.2.8"
-
-[dev-dependencies]
-assert_matches = "1.3"
-criterion = "0.3"
-csv = "1.0.7"
-jemallocator = "0.3.2"
-rustyline = { version = "5.0.0", default-features = false }
-structopt = "0.3.2"
-tempfile = "3.1.0"
-termcolor = "1.0.4"
-
-[[bench]]
-name = "search_benchmark"
-harness = false

meilisearch-core/benches/search_benchmark.rs

@@ -1,95 +0,0 @@
-#[cfg(test)]
-#[macro_use]
-extern crate assert_matches;
-
-use std::sync::mpsc;
-use std::path::Path;
-use std::fs;
-use std::iter;
-
-use meilisearch_core::Database;
-use meilisearch_core::{ProcessedUpdateResult, UpdateStatus};
-use serde_json::Value;
-
-use criterion::{criterion_group, criterion_main, Criterion, BenchmarkId};
-
-fn prepare_database(path: &Path) -> Database {
-    let database = Database::open_or_create(path).unwrap();
-    let db = &database;
-
-    let (sender, receiver) = mpsc::sync_channel(100);
-    let update_fn = move |_name: &str, update: ProcessedUpdateResult| {
-        sender.send(update.update_id).unwrap()
-    };
-    let index = database.create_index("bench").unwrap();
-
-    database.set_update_callback(Box::new(update_fn));
-
-    let schema = {
-        let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../datasets/movies/schema.toml");
-        let string = fs::read_to_string(path).expect("find schema");
-        toml::from_str(&string).unwrap()
-    };
-
-    let mut update_writer = db.update_write_txn().unwrap();
-    let _update_id = index.schema_update(&mut update_writer, schema).unwrap();
-    update_writer.commit().unwrap();
-
-    let mut additions = index.documents_addition();
-
-    let json: Value = {
-        let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../datasets/movies/movies.json");
-        let movies_file = fs::File::open(path).expect("find movies");
-        serde_json::from_reader(movies_file).unwrap()
-    };
-
-    let documents = json.as_array().unwrap();
-
-    for document in documents {
-        additions.update_document(document);
-    }
-
-    let mut update_writer = db.update_write_txn().unwrap();
-    let update_id = additions.finalize(&mut update_writer).unwrap();
-    update_writer.commit().unwrap();
-
-    // block until the transaction is processed
-    let _ = receiver.into_iter().find(|id| *id == update_id);
-
-    let update_reader = db.update_read_txn().unwrap();
-    let result = index.update_status(&update_reader, update_id).unwrap();
-    assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-
-    database
-}
-
-pub fn criterion_benchmark(c: &mut Criterion) {
-    let dir = tempfile::tempdir().unwrap();
-    let database = prepare_database(dir.path());
-
-    let reader = database.main_read_txn().unwrap();
-    let index = database.open_index("bench").unwrap();
-
-    let mut count = 0;
-    let query = "I love paris ";
-
-    let iter = iter::from_fn(|| {
-        count += 1;
-        query.get(0..count)
-    });
-
-    let mut group = c.benchmark_group("searching in movies (19654 docs)");
-    group.sample_size(10);
-
-    for query in iter {
-        let bench_name = BenchmarkId::from_parameter(format!("{:?}", query));
-        group.bench_with_input(bench_name, &query, |b, query| b.iter(|| {
-            let builder = index.query_builder();
-            builder.query(&reader, query, 0..20).unwrap();
-        }));
-    }
-    group.finish();
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

meilisearch-core/examples/from_file.rs

@@ -1,473 +0,0 @@
-use std::collections::HashSet;
-use std::collections::btree_map::{BTreeMap, Entry};
-use std::error::Error;
-use std::io::{Read, Write};
-use std::iter::FromIterator;
-use std::path::{Path, PathBuf};
-use std::time::{Duration, Instant};
-use std::{fs, io, sync::mpsc};
-
-use rustyline::{Config, Editor};
-use serde::{Deserialize, Serialize};
-use structopt::StructOpt;
-use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
-
-use meilisearch_core::{Database, Highlight, ProcessedUpdateResult};
-use meilisearch_core::settings::Settings;
-use meilisearch_schema::FieldId;
-
-// #[cfg(target_os = "linux")]
-#[global_allocator]
-static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
-
-#[derive(Debug, StructOpt)]
-struct IndexCommand {
-    /// The destination where the database must be created.
-    #[structopt(parse(from_os_str))]
-    database_path: PathBuf,
-
-    #[structopt(long, default_value = "default")]
-    index_uid: String,
-
-    /// The csv file path to index, you can also use `-` to specify the standard input.
-    #[structopt(parse(from_os_str))]
-    csv_data_path: PathBuf,
-
-    /// The path to the settings.
-    #[structopt(long, parse(from_os_str))]
-    settings: PathBuf,
-
-    #[structopt(long)]
-    update_group_size: Option<usize>,
-
-    #[structopt(long, parse(from_os_str))]
-    compact_to_path: Option<PathBuf>,
-}
-
-#[derive(Debug, StructOpt)]
-struct SearchCommand {
-    /// The path of the database to work with.
-    #[structopt(parse(from_os_str))]
-    database_path: PathBuf,
-
-    #[structopt(long, default_value = "default")]
-    index_uid: String,
-
-    /// Timeout after which the search will return results.
-    #[structopt(long)]
-    fetch_timeout_ms: Option<u64>,
-
-    /// The number of returned results
-    #[structopt(short, long, default_value = "10")]
-    number_results: usize,
-
-    /// The number of characters before and after the first match
-    #[structopt(short = "C", long, default_value = "35")]
-    char_context: usize,
-
-    /// A filter string that can be `!adult` or `adult` to
-    /// filter documents on this specfied field
-    #[structopt(short, long)]
-    filter: Option<String>,
-
-    /// Fields that must be displayed.
-    displayed_fields: Vec<String>,
-}
-
-#[derive(Debug, StructOpt)]
-struct ShowUpdatesCommand {
-    /// The path of the database to work with.
-    #[structopt(parse(from_os_str))]
-    database_path: PathBuf,
-
-    #[structopt(long, default_value = "default")]
-    index_uid: String,
-}
-
-#[derive(Debug, StructOpt)]
-enum Command {
-    Index(IndexCommand),
-    Search(SearchCommand),
-    ShowUpdates(ShowUpdatesCommand),
-}
-
-impl Command {
-    fn path(&self) -> &Path {
-        match self {
-            Command::Index(command) => &command.database_path,
-            Command::Search(command) => &command.database_path,
-            Command::ShowUpdates(command) => &command.database_path,
-        }
-    }
-}
-
-#[derive(Serialize, Deserialize)]
-#[serde(transparent)]
-struct Document(indexmap::IndexMap<String, String>);
-
-fn index_command(command: IndexCommand, database: Database) -> Result<(), Box<dyn Error>> {
-    let start = Instant::now();
-
-    let (sender, receiver) = mpsc::sync_channel(100);
-    let update_fn =
-        move |_name: &str, update: ProcessedUpdateResult| sender.send(update.update_id).unwrap();
-    let index = match database.open_index(&command.index_uid) {
-        Some(index) => index,
-        None => database.create_index(&command.index_uid).unwrap(),
-    };
-
-    database.set_update_callback(Box::new(update_fn));
-
-    let db = &database;
-
-    let settings = {
-        let string = fs::read_to_string(&command.settings)?;
-        let settings: Settings = serde_json::from_str(&string).unwrap();
-        settings.into_update().unwrap()
-    };
-
-    let mut update_writer = db.update_write_txn().unwrap();
-    index.settings_update(&mut update_writer, settings)?;
-    update_writer.commit().unwrap();
-
-    let mut rdr = if command.csv_data_path.as_os_str() == "-" {
-        csv::Reader::from_reader(Box::new(io::stdin()) as Box<dyn Read>)
-    } else {
-        let file = std::fs::File::open(command.csv_data_path)?;
-        csv::Reader::from_reader(Box::new(file) as Box<dyn Read>)
-    };
-
-    let mut raw_record = csv::StringRecord::new();
-    let headers = rdr.headers()?.clone();
-
-    let mut max_update_id = 0;
-    let mut i = 0;
-    let mut end_of_file = false;
-
-    while !end_of_file {
-        let mut additions = index.documents_addition();
-
-        loop {
-            end_of_file = !rdr.read_record(&mut raw_record)?;
-            if end_of_file {
-                break;
-            }
-
-            let document: Document = match raw_record.deserialize(Some(&headers)) {
-                Ok(document) => document,
-                Err(e) => {
-                    eprintln!("{:?}", e);
-                    continue;
-                }
-            };
-
-            additions.update_document(document);
-
-            print!("\rindexing document {}", i);
-            i += 1;
-
-            if let Some(group_size) = command.update_group_size {
-                if i % group_size == 0 {
-                    break;
-                }
-            }
-        }
-
-        println!();
-
-        let mut update_writer = db.update_write_txn().unwrap();
-        println!("committing update...");
-        let update_id = additions.finalize(&mut update_writer)?;
-        update_writer.commit().unwrap();
-        max_update_id = max_update_id.max(update_id);
-        println!("committed update {}", update_id);
-    }
-
-    println!("Waiting for update {}", max_update_id);
-    for id in receiver {
-        if id == max_update_id {
-            break;
-        }
-    }
-
-    println!(
-        "database created in {:.2?} at: {:?}",
-        start.elapsed(),
-        command.database_path
-    );
-
-    if let Some(path) = command.compact_to_path {
-        fs::create_dir_all(&path)?;
-        let start = Instant::now();
-        let _file = database.copy_and_compact_to_path(path.join("data.mdb"))?;
-        println!(
-            "database compacted in {:.2?} at: {:?}",
-            start.elapsed(),
-            path
-        );
-    }
-
-    Ok(())
-}
-
-fn display_highlights(text: &str, ranges: &[usize]) -> io::Result<()> {
-    let mut stdout = StandardStream::stdout(ColorChoice::Always);
-    let mut highlighted = false;
-
-    for range in ranges.windows(2) {
-        let [start, end] = match range {
-            [start, end] => [*start, *end],
-            _ => unreachable!(),
-        };
-        if highlighted {
-            stdout.set_color(
-                ColorSpec::new()
-                    .set_fg(Some(Color::Yellow))
-                    .set_underline(true),
-            )?;
-        }
-        write!(&mut stdout, "{}", &text[start..end])?;
-        stdout.reset()?;
-        highlighted = !highlighted;
-    }
-
-    Ok(())
-}
-
-fn char_to_byte_range(index: usize, length: usize, text: &str) -> (usize, usize) {
-    let mut byte_index = 0;
-    let mut byte_length = 0;
-
-    for (n, (i, c)) in text.char_indices().enumerate() {
-        if n == index {
-            byte_index = i;
-        }
-
-        if n + 1 == index + length {
-            byte_length = i - byte_index + c.len_utf8();
-            break;
-        }
-    }
-
-    (byte_index, byte_length)
-}
-
-fn create_highlight_areas(text: &str, highlights: &[Highlight]) -> Vec<usize> {
-    let mut byte_indexes = BTreeMap::new();
-
-    for highlight in highlights {
-        let char_index = highlight.char_index as usize;
-        let char_length = highlight.char_length as usize;
-        let (byte_index, byte_length) = char_to_byte_range(char_index, char_length, text);
-
-        match byte_indexes.entry(byte_index) {
-            Entry::Vacant(entry) => {
-                entry.insert(byte_length);
-            }
-            Entry::Occupied(mut entry) => {
-                if *entry.get() < byte_length {
-                    entry.insert(byte_length);
-                }
-            }
-        }
-    }
-
-    let mut title_areas = Vec::new();
-    title_areas.push(0);
-    for (byte_index, length) in byte_indexes {
-        title_areas.push(byte_index);
-        title_areas.push(byte_index + length);
-    }
-    title_areas.push(text.len());
-    title_areas.sort_unstable();
-    title_areas
-}
-
-/// note: matches must have been sorted by `char_index` and `char_length` before being passed.
-///
-/// ```no_run
-/// matches.sort_unstable_by_key(|m| (m.char_index, m.char_length));
-///
-/// let matches = matches.matches.iter().filter(|m| SchemaAttr::new(m.attribute) == attr).cloned();
-///
-/// let (text, matches) = crop_text(&text, matches, 35);
-/// ```
-fn crop_text(
-    text: &str,
-    highlights: impl IntoIterator<Item = Highlight>,
-    context: usize,
-) -> (String, Vec<Highlight>) {
-    let mut highlights = highlights.into_iter().peekable();
-
-    let char_index = highlights
-        .peek()
-        .map(|m| m.char_index as usize)
-        .unwrap_or(0);
-    let start = char_index.saturating_sub(context);
-    let text = text.chars().skip(start).take(context * 2).collect();
-
-    let highlights = highlights
-        .take_while(|m| (m.char_index as usize) + (m.char_length as usize) <= start + (context * 2))
-        .map(|highlight| Highlight {
-            char_index: highlight.char_index - start as u16,
-            ..highlight
-        })
-        .collect();
-
-    (text, highlights)
-}
-
-fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<dyn Error>> {
-    let db = &database;
-    let index = database
-        .open_index(&command.index_uid)
-        .expect("Could not find index");
-
-    let reader = db.main_read_txn().unwrap();
-    let schema = index.main.schema(&reader)?;
-    reader.abort();
-
-    let schema = schema.ok_or(meilisearch_core::Error::SchemaMissing)?;
-
-    let fields = command.displayed_fields.iter().map(String::as_str);
-    let fields = HashSet::from_iter(fields);
-
-    let config = Config::builder().auto_add_history(true).build();
-    let mut readline = Editor::<()>::with_config(config);
-    let _ = readline.load_history("query-history.txt");
-
-    for result in readline.iter("Searching for: ") {
-        match result {
-            Ok(query) => {
-                let start_total = Instant::now();
-
-                let reader = db.main_read_txn().unwrap();
-                let ref_index = &index;
-                let ref_reader = &reader;
-
-                let mut builder = index.query_builder();
-                if let Some(timeout) = command.fetch_timeout_ms {
-                    builder.with_fetch_timeout(Duration::from_millis(timeout));
-                }
-
-                if let Some(ref filter) = command.filter {
-                    let filter = filter.as_str();
-                    let (positive, filter) = if filter.chars().next() == Some('!') {
-                        (false, &filter[1..])
-                    } else {
-                        (true, filter)
-                    };
-
-                    let attr = schema
-                        .id(filter)
-                        .expect("Could not find filtered attribute");
-
-                    builder.with_filter(move |document_id| {
-                        let string: String = ref_index
-                            .document_attribute(ref_reader, document_id, attr)
-                            .unwrap()
-                            .unwrap();
-                        (string == "true") == positive
-                    });
-                }
-
-                let documents = builder.query(ref_reader, &query, 0..command.number_results)?;
-
-                let mut retrieve_duration = Duration::default();
-
-                let number_of_documents = documents.len();
-                for mut doc in documents {
-                    doc.highlights
-                        .sort_unstable_by_key(|m| (m.char_index, m.char_length));
-
-                    let start_retrieve = Instant::now();
-                    let result = index.document::<Document>(&reader, Some(&fields), doc.id);
-                    retrieve_duration += start_retrieve.elapsed();
-
-                    match result {
-                        Ok(Some(document)) => {
-                            println!("raw-id: {:?}", doc.id);
-                            for (name, text) in document.0 {
-                                print!("{}: ", name);
-
-                                let attr = schema.id(&name).unwrap();
-                                let highlights = doc
-                                    .highlights
-                                    .iter()
-                                    .filter(|m| FieldId::new(m.attribute) == attr)
-                                    .cloned();
-                                let (text, highlights) =
-                                    crop_text(&text, highlights, command.char_context);
-                                let areas = create_highlight_areas(&text, &highlights);
-                                display_highlights(&text, &areas)?;
-                                println!();
-                            }
-                        }
-                        Ok(None) => eprintln!("missing document"),
-                        Err(e) => eprintln!("{}", e),
-                    }
-
-                    let mut matching_attributes = HashSet::new();
-                    for highlight in doc.highlights {
-                        let attr = FieldId::new(highlight.attribute);
-                        let name = schema.name(attr);
-                        matching_attributes.insert(name);
-                    }
-
-                    let matching_attributes = Vec::from_iter(matching_attributes);
-                    println!("matching in: {:?}", matching_attributes);
-
-                    println!();
-                }
-
-                eprintln!(
-                    "whole documents fields retrieve took {:.2?}",
-                    retrieve_duration
-                );
-                eprintln!(
-                    "===== Found {} results in {:.2?} =====",
-                    number_of_documents,
-                    start_total.elapsed()
-                );
-            }
-            Err(err) => {
-                println!("Error: {:?}", err);
-                break;
-            }
-        }
-    }
-
-    readline.save_history("query-history.txt").unwrap();
-
-    Ok(())
-}
-
-fn show_updates_command(
-    command: ShowUpdatesCommand,
-    database: Database,
-) -> Result<(), Box<dyn Error>> {
-    let db = &database;
-    let index = database
-        .open_index(&command.index_uid)
-        .expect("Could not find index");
-
-    let reader = db.update_read_txn().unwrap();
-    let updates = index.all_updates_status(&reader)?;
-    println!("{:#?}", updates);
-    reader.abort();
-
-    Ok(())
-}
-
-fn main() -> Result<(), Box<dyn Error>> {
-    env_logger::init();
-
-    let opt = Command::from_args();
-    let database = Database::open_or_create(opt.path())?;
-
-    match opt {
-        Command::Index(command) => index_command(command, database),
-        Command::Search(command) => search_command(command, database),
-        Command::ShowUpdates(command) => show_updates_command(command, database),
-    }
-}

meilisearch-core/src/automaton/dfa.rs

@@ -1,53 +0,0 @@
-use levenshtein_automata::{LevenshteinAutomatonBuilder as LevBuilder, DFA};
-use once_cell::sync::OnceCell;
-
-static LEVDIST0: OnceCell<LevBuilder> = OnceCell::new();
-static LEVDIST1: OnceCell<LevBuilder> = OnceCell::new();
-static LEVDIST2: OnceCell<LevBuilder> = OnceCell::new();
-
-#[derive(Copy, Clone)]
-enum PrefixSetting {
-    Prefix,
-    NoPrefix,
-}
-
-fn build_dfa_with_setting(query: &str, setting: PrefixSetting) -> DFA {
-    use PrefixSetting::{NoPrefix, Prefix};
-
-    match query.len() {
-        0..=4 => {
-            let builder = LEVDIST0.get_or_init(|| LevBuilder::new(0, true));
-            match setting {
-                Prefix => builder.build_prefix_dfa(query),
-                NoPrefix => builder.build_dfa(query),
-            }
-        }
-        5..=8 => {
-            let builder = LEVDIST1.get_or_init(|| LevBuilder::new(1, true));
-            match setting {
-                Prefix => builder.build_prefix_dfa(query),
-                NoPrefix => builder.build_dfa(query),
-            }
-        }
-        _ => {
-            let builder = LEVDIST2.get_or_init(|| LevBuilder::new(2, true));
-            match setting {
-                Prefix => builder.build_prefix_dfa(query),
-                NoPrefix => builder.build_dfa(query),
-            }
-        }
-    }
-}
-
-pub fn build_prefix_dfa(query: &str) -> DFA {
-    build_dfa_with_setting(query, PrefixSetting::Prefix)
-}
-
-pub fn build_dfa(query: &str) -> DFA {
-    build_dfa_with_setting(query, PrefixSetting::NoPrefix)
-}
-
-pub fn build_exact_dfa(query: &str) -> DFA {
-    let builder = LEVDIST0.get_or_init(|| LevBuilder::new(0, true));
-    builder.build_dfa(query)
-}

meilisearch-core/src/automaton/mod.rs

@@ -1,15 +0,0 @@
-mod dfa;
-
-use meilisearch_tokenizer::is_cjk;
-
-pub use self::dfa::{build_dfa, build_prefix_dfa, build_exact_dfa};
-
-pub fn normalize_str(string: &str) -> String {
-    let mut string = string.to_lowercase();
-
-    if !string.contains(is_cjk) {
-        string = deunicode::deunicode_with_tofu(&string, "");
-    }
-
-    string
-}

meilisearch-core/src/bucket_sort.rs

@@ -1,560 +0,0 @@
-use std::borrow::Cow;
-use std::collections::HashMap;
-use std::mem;
-use std::ops::Deref;
-use std::ops::Range;
-use std::rc::Rc;
-use std::sync::atomic::{AtomicUsize, Ordering};
-use std::time::Instant;
-use std::fmt;
-
-use compact_arena::{SmallArena, Idx32, mk_arena};
-use log::debug;
-use meilisearch_types::DocIndex;
-use sdset::{Set, SetBuf, exponential_search};
-use slice_group_by::{GroupBy, GroupByMut};
-
-use crate::error::Error;
-use crate::criterion::{Criteria, Context, ContextMut};
-use crate::distinct_map::{BufferedDistinctMap, DistinctMap};
-use crate::raw_document::RawDocument;
-use crate::{database::MainT, reordered_attrs::ReorderedAttrs};
-use crate::{store, Document, DocumentId, MResult};
-use crate::query_tree::{create_query_tree, traverse_query_tree};
-use crate::query_tree::{Operation, QueryResult, QueryKind, QueryId, PostingsKey};
-use crate::query_tree::Context as QTContext;
-
-pub fn bucket_sort<'c, FI>(
-    reader: &heed::RoTxn<MainT>,
-    query: &str,
-    range: Range<usize>,
-    filter: Option<FI>,
-    criteria: Criteria<'c>,
-    searchable_attrs: Option<ReorderedAttrs>,
-    main_store: store::Main,
-    postings_lists_store: store::PostingsLists,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    synonyms_store: store::Synonyms,
-    prefix_documents_cache_store: store::PrefixDocumentsCache,
-    prefix_postings_lists_cache_store: store::PrefixPostingsListsCache,
-) -> MResult<Vec<Document>>
-where
-    FI: Fn(DocumentId) -> bool,
-{
-    // We delegate the filter work to the distinct query builder,
-    // specifying a distinct rule that has no effect.
-    if filter.is_some() {
-        let distinct = |_| None;
-        let distinct_size = 1;
-        return bucket_sort_with_distinct(
-            reader,
-            query,
-            range,
-            filter,
-            distinct,
-            distinct_size,
-            criteria,
-            searchable_attrs,
-            main_store,
-            postings_lists_store,
-            documents_fields_counts_store,
-            synonyms_store,
-            prefix_documents_cache_store,
-            prefix_postings_lists_cache_store,
-        );
-    }
-
-    let words_set = match unsafe { main_store.static_words_fst(reader)? } {
-        Some(words) => words,
-        None => return Ok(Vec::new()),
-    };
-
-    let stop_words = main_store.stop_words_fst(reader)?.unwrap_or_default();
-
-    let context = QTContext {
-        words_set,
-        stop_words,
-        synonyms: synonyms_store,
-        postings_lists: postings_lists_store,
-        prefix_postings_lists: prefix_postings_lists_cache_store,
-    };
-
-    let (operation, mapping) = create_query_tree(reader, &context, query)?;
-    debug!("operation:\n{:?}", operation);
-    debug!("mapping:\n{:?}", mapping);
-
-    fn recurs_operation<'o>(map: &mut HashMap<QueryId, &'o QueryKind>, operation: &'o Operation) {
-        match operation {
-            Operation::And(ops) => ops.iter().for_each(|op| recurs_operation(map, op)),
-            Operation::Or(ops) => ops.iter().for_each(|op| recurs_operation(map, op)),
-            Operation::Query(query) => { map.insert(query.id, &query.kind); },
-        }
-    }
-
-    let mut queries_kinds = HashMap::new();
-    recurs_operation(&mut queries_kinds, &operation);
-
-    let QueryResult { docids, queries } = traverse_query_tree(reader, &context, &operation)?;
-    debug!("found {} documents", docids.len());
-    debug!("number of postings {:?}", queries.len());
-
-    let before = Instant::now();
-    mk_arena!(arena);
-    let mut bare_matches = cleanup_bare_matches(&mut arena, &docids, queries);
-    debug!("matches cleaned in {:.02?}", before.elapsed());
-
-    let before_bucket_sort = Instant::now();
-
-    let before_raw_documents_building = Instant::now();
-    let mut raw_documents = Vec::new();
-    for bare_matches in bare_matches.linear_group_by_key_mut(|sm| sm.document_id) {
-        let raw_document = RawDocument::new(bare_matches, &mut arena, searchable_attrs.as_ref());
-        raw_documents.push(raw_document);
-    }
-    debug!("creating {} candidates documents took {:.02?}",
-        raw_documents.len(),
-        before_raw_documents_building.elapsed(),
-    );
-
-    let before_criterion_loop = Instant::now();
-    let proximity_count = AtomicUsize::new(0);
-
-    let mut groups = vec![raw_documents.as_mut_slice()];
-
-    'criteria: for criterion in criteria.as_ref() {
-        let tmp_groups = mem::replace(&mut groups, Vec::new());
-        let mut documents_seen = 0;
-
-        for mut group in tmp_groups {
-            let before_criterion_preparation = Instant::now();
-
-            let ctx = ContextMut {
-                reader,
-                postings_lists: &mut arena,
-                query_mapping: &mapping,
-                documents_fields_counts_store,
-            };
-
-            criterion.prepare(ctx, &mut group)?;
-            debug!("{:?} preparation took {:.02?}", criterion.name(), before_criterion_preparation.elapsed());
-
-            let ctx = Context {
-                postings_lists: &arena,
-                query_mapping: &mapping,
-            };
-
-            let before_criterion_sort = Instant::now();
-            group.sort_unstable_by(|a, b| criterion.evaluate(&ctx, a, b));
-            debug!("{:?} evaluation took {:.02?}", criterion.name(), before_criterion_sort.elapsed());
-
-            for group in group.binary_group_by_mut(|a, b| criterion.eq(&ctx, a, b)) {
-                debug!("{:?} produced a group of size {}", criterion.name(), group.len());
-
-                documents_seen += group.len();
-                groups.push(group);
-
-                // we have sort enough documents if the last document sorted is after
-                // the end of the requested range, we can continue to the next criterion
-                if documents_seen >= range.end {
-                    continue 'criteria;
-                }
-            }
-        }
-    }
-
-    debug!("criterion loop took {:.02?}", before_criterion_loop.elapsed());
-    debug!("proximity evaluation called {} times", proximity_count.load(Ordering::Relaxed));
-
-    let schema = main_store.schema(reader)?.ok_or(Error::SchemaMissing)?;
-    let iter = raw_documents.into_iter().skip(range.start).take(range.len());
-    let iter = iter.map(|rd| Document::from_raw(rd, &queries_kinds, &arena, searchable_attrs.as_ref(), &schema));
-    let documents = iter.collect();
-
-    debug!("bucket sort took {:.02?}", before_bucket_sort.elapsed());
-
-    Ok(documents)
-}
-
-pub fn bucket_sort_with_distinct<'c, FI, FD>(
-    reader: &heed::RoTxn<MainT>,
-    query: &str,
-    range: Range<usize>,
-    filter: Option<FI>,
-    distinct: FD,
-    distinct_size: usize,
-    criteria: Criteria<'c>,
-    searchable_attrs: Option<ReorderedAttrs>,
-    main_store: store::Main,
-    postings_lists_store: store::PostingsLists,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    synonyms_store: store::Synonyms,
-    _prefix_documents_cache_store: store::PrefixDocumentsCache,
-    prefix_postings_lists_cache_store: store::PrefixPostingsListsCache,
-) -> MResult<Vec<Document>>
-where
-    FI: Fn(DocumentId) -> bool,
-    FD: Fn(DocumentId) -> Option<u64>,
-{
-    let words_set = match unsafe { main_store.static_words_fst(reader)? } {
-        Some(words) => words,
-        None => return Ok(Vec::new()),
-    };
-
-    let stop_words = main_store.stop_words_fst(reader)?.unwrap_or_default();
-
-    let context = QTContext {
-        words_set,
-        stop_words,
-        synonyms: synonyms_store,
-        postings_lists: postings_lists_store,
-        prefix_postings_lists: prefix_postings_lists_cache_store,
-    };
-
-    let (operation, mapping) = create_query_tree(reader, &context, query)?;
-    debug!("operation:\n{:?}", operation);
-    debug!("mapping:\n{:?}", mapping);
-
-    fn recurs_operation<'o>(map: &mut HashMap<QueryId, &'o QueryKind>, operation: &'o Operation) {
-        match operation {
-            Operation::And(ops) => ops.iter().for_each(|op| recurs_operation(map, op)),
-            Operation::Or(ops) => ops.iter().for_each(|op| recurs_operation(map, op)),
-            Operation::Query(query) => { map.insert(query.id, &query.kind); },
-        }
-    }
-
-    let mut queries_kinds = HashMap::new();
-    recurs_operation(&mut queries_kinds, &operation);
-
-    let QueryResult { docids, queries } = traverse_query_tree(reader, &context, &operation)?;
-    debug!("found {} documents", docids.len());
-    debug!("number of postings {:?}", queries.len());
-
-    let before = Instant::now();
-    mk_arena!(arena);
-    let mut bare_matches = cleanup_bare_matches(&mut arena, &docids, queries);
-    debug!("matches cleaned in {:.02?}", before.elapsed());
-
-    let before_raw_documents_building = Instant::now();
-    let mut raw_documents = Vec::new();
-    for bare_matches in bare_matches.linear_group_by_key_mut(|sm| sm.document_id) {
-        let raw_document = RawDocument::new(bare_matches, &mut arena, searchable_attrs.as_ref());
-        raw_documents.push(raw_document);
-    }
-    debug!("creating {} candidates documents took {:.02?}",
-        raw_documents.len(),
-        before_raw_documents_building.elapsed(),
-    );
-
-    let mut groups = vec![raw_documents.as_mut_slice()];
-    let mut key_cache = HashMap::new();
-
-    let mut filter_map = HashMap::new();
-    // these two variables informs on the current distinct map and
-    // on the raw offset of the start of the group where the
-    // range.start bound is located according to the distinct function
-    let mut distinct_map = DistinctMap::new(distinct_size);
-    let mut distinct_raw_offset = 0;
-
-    'criteria: for criterion in criteria.as_ref() {
-        let tmp_groups = mem::replace(&mut groups, Vec::new());
-        let mut buf_distinct = BufferedDistinctMap::new(&mut distinct_map);
-        let mut documents_seen = 0;
-
-        for mut group in tmp_groups {
-            // if this group does not overlap with the requested range,
-            // push it without sorting and splitting it
-            if documents_seen + group.len() < distinct_raw_offset {
-                documents_seen += group.len();
-                groups.push(group);
-                continue;
-            }
-
-            let ctx = ContextMut {
-                reader,
-                postings_lists: &mut arena,
-                query_mapping: &mapping,
-                documents_fields_counts_store,
-            };
-
-            let before_criterion_preparation = Instant::now();
-            criterion.prepare(ctx, &mut group)?;
-            debug!("{:?} preparation took {:.02?}", criterion.name(), before_criterion_preparation.elapsed());
-
-            let ctx = Context {
-                postings_lists: &arena,
-                query_mapping: &mapping,
-            };
-
-            let before_criterion_sort = Instant::now();
-            group.sort_unstable_by(|a, b| criterion.evaluate(&ctx, a, b));
-            debug!("{:?} evaluation took {:.02?}", criterion.name(), before_criterion_sort.elapsed());
-
-            for group in group.binary_group_by_mut(|a, b| criterion.eq(&ctx, a, b)) {
-                // we must compute the real distinguished len of this sub-group
-                for document in group.iter() {
-                    let filter_accepted = match &filter {
-                        Some(filter) => {
-                            let entry = filter_map.entry(document.id);
-                            *entry.or_insert_with(|| (filter)(document.id))
-                        }
-                        None => true,
-                    };
-
-                    if filter_accepted {
-                        let entry = key_cache.entry(document.id);
-                        let key = entry.or_insert_with(|| (distinct)(document.id).map(Rc::new));
-
-                        match key.clone() {
-                            Some(key) => buf_distinct.register(key),
-                            None => buf_distinct.register_without_key(),
-                        };
-                    }
-
-                    // the requested range end is reached: stop computing distinct
-                    if buf_distinct.len() >= range.end {
-                        break;
-                    }
-                }
-
-                documents_seen += group.len();
-                groups.push(group);
-
-                // if this sub-group does not overlap with the requested range
-                // we must update the distinct map and its start index
-                if buf_distinct.len() < range.start {
-                    buf_distinct.transfert_to_internal();
-                    distinct_raw_offset = documents_seen;
-                }
-
-                // we have sort enough documents if the last document sorted is after
-                // the end of the requested range, we can continue to the next criterion
-                if buf_distinct.len() >= range.end {
-                    continue 'criteria;
-                }
-            }
-        }
-    }
-
-    // once we classified the documents related to the current
-    // automatons we save that as the next valid result
-    let mut seen = BufferedDistinctMap::new(&mut distinct_map);
-    let schema = main_store.schema(reader)?.ok_or(Error::SchemaMissing)?;
-
-    let mut documents = Vec::with_capacity(range.len());
-    for raw_document in raw_documents.into_iter().skip(distinct_raw_offset) {
-        let filter_accepted = match &filter {
-            Some(_) => filter_map.remove(&raw_document.id).unwrap(),
-            None => true,
-        };
-
-        if filter_accepted {
-            let key = key_cache.remove(&raw_document.id).unwrap();
-            let distinct_accepted = match key {
-                Some(key) => seen.register(key),
-                None => seen.register_without_key(),
-            };
-
-            if distinct_accepted && seen.len() > range.start {
-                documents.push(Document::from_raw(raw_document, &queries_kinds, &arena, searchable_attrs.as_ref(), &schema));
-                if documents.len() == range.len() {
-                    break;
-                }
-            }
-        }
-    }
-
-    Ok(documents)
-}
-
-fn cleanup_bare_matches<'tag, 'txn>(
-    arena: &mut SmallArena<'tag, PostingsListView<'txn>>,
-    docids: &Set<DocumentId>,
-    queries: HashMap<PostingsKey, Cow<'txn, Set<DocIndex>>>,
-) -> Vec<BareMatch<'tag>>
-{
-    let docidslen = docids.len() as f32;
-    let mut bare_matches = Vec::new();
-
-    for (PostingsKey { query, input, distance, is_exact }, matches) in queries {
-        let postings_list_view = PostingsListView::original(Rc::from(input), Rc::new(matches));
-        let pllen = postings_list_view.len() as f32;
-
-        if docidslen / pllen >= 0.8 {
-            let mut offset = 0;
-            for matches in postings_list_view.linear_group_by_key(|m| m.document_id) {
-                let document_id = matches[0].document_id;
-                if docids.contains(&document_id) {
-                    let range = postings_list_view.range(offset, matches.len());
-                    let posting_list_index = arena.add(range);
-
-                    let bare_match = BareMatch {
-                        document_id,
-                        query_index: query.id,
-                        distance,
-                        is_exact,
-                        postings_list: posting_list_index,
-                    };
-
-                    bare_matches.push(bare_match);
-                }
-
-                offset += matches.len();
-            }
-
-        } else {
-            let mut offset = 0;
-            for id in docids.as_slice() {
-                let di = DocIndex { document_id: *id, ..DocIndex::default() };
-                let pos = exponential_search(&postings_list_view[offset..], &di).unwrap_or_else(|x| x);
-
-                offset += pos;
-
-                let group = postings_list_view[offset..]
-                    .linear_group_by_key(|m| m.document_id)
-                    .next()
-                    .filter(|matches| matches[0].document_id == *id);
-
-                if let Some(matches) = group {
-                    let range = postings_list_view.range(offset, matches.len());
-                    let posting_list_index = arena.add(range);
-
-                    let bare_match = BareMatch {
-                        document_id: *id,
-                        query_index: query.id,
-                        distance,
-                        is_exact,
-                        postings_list: posting_list_index,
-                    };
-
-                    bare_matches.push(bare_match);
-                }
-            }
-        }
-    }
-
-    let before_raw_documents_presort = Instant::now();
-    bare_matches.sort_unstable_by_key(|sm| sm.document_id);
-    debug!("sort by documents ids took {:.02?}", before_raw_documents_presort.elapsed());
-
-    bare_matches
-}
-
-pub struct BareMatch<'tag> {
-    pub document_id: DocumentId,
-    pub query_index: usize,
-    pub distance: u8,
-    pub is_exact: bool,
-    pub postings_list: Idx32<'tag>,
-}
-
-impl fmt::Debug for BareMatch<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("BareMatch")
-            .field("document_id", &self.document_id)
-            .field("query_index", &self.query_index)
-            .field("distance", &self.distance)
-            .field("is_exact", &self.is_exact)
-            .finish()
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-pub struct SimpleMatch {
-    pub query_index: usize,
-    pub distance: u8,
-    pub attribute: u16,
-    pub word_index: u16,
-    pub is_exact: bool,
-}
-
-#[derive(Clone)]
-pub enum PostingsListView<'txn> {
-    Original {
-        input: Rc<[u8]>,
-        postings_list: Rc<Cow<'txn, Set<DocIndex>>>,
-        offset: usize,
-        len: usize,
-    },
-    Rewritten {
-        input: Rc<[u8]>,
-        postings_list: SetBuf<DocIndex>,
-    },
-}
-
-impl fmt::Debug for PostingsListView<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("PostingsListView")
-            .field("input", &std::str::from_utf8(&self.input()).unwrap())
-            .field("postings_list", &self.as_ref())
-            .finish()
-    }
-}
-
-impl<'txn> PostingsListView<'txn> {
-    pub fn original(input: Rc<[u8]>, postings_list: Rc<Cow<'txn, Set<DocIndex>>>) -> PostingsListView<'txn> {
-        let len = postings_list.len();
-        PostingsListView::Original { input, postings_list, offset: 0, len }
-    }
-
-    pub fn rewritten(input: Rc<[u8]>, postings_list: SetBuf<DocIndex>) -> PostingsListView<'txn> {
-        PostingsListView::Rewritten { input, postings_list }
-    }
-
-    pub fn rewrite_with(&mut self, postings_list: SetBuf<DocIndex>) {
-        let input = match self {
-            PostingsListView::Original { input, .. } => input.clone(),
-            PostingsListView::Rewritten { input, .. } => input.clone(),
-        };
-        *self = PostingsListView::rewritten(input, postings_list);
-    }
-
-    pub fn len(&self) -> usize {
-        match self {
-            PostingsListView::Original { len, .. } => *len,
-            PostingsListView::Rewritten { postings_list, .. } => postings_list.len(),
-        }
-    }
-
-    pub fn input(&self) -> &[u8] {
-        match self {
-            PostingsListView::Original { ref input, .. } => input,
-            PostingsListView::Rewritten { ref input, .. } => input,
-        }
-    }
-
-    pub fn range(&self, range_offset: usize, range_len: usize) -> PostingsListView<'txn> {
-        match self {
-            PostingsListView::Original { input, postings_list, offset, len } => {
-                assert!(range_offset + range_len <= *len);
-                PostingsListView::Original {
-                    input: input.clone(),
-                    postings_list: postings_list.clone(),
-                    offset: offset + range_offset,
-                    len: range_len,
-                }
-            },
-            PostingsListView::Rewritten { .. } => {
-                panic!("Cannot create a range on a rewritten postings list view");
-            }
-        }
-    }
-}
-
-impl AsRef<Set<DocIndex>> for PostingsListView<'_> {
-    fn as_ref(&self) -> &Set<DocIndex> {
-        self
-    }
-}
-
-impl Deref for PostingsListView<'_> {
-    type Target = Set<DocIndex>;
-
-    fn deref(&self) -> &Set<DocIndex> {
-        match *self {
-            PostingsListView::Original { ref postings_list, offset, len, .. } => {
-                Set::new_unchecked(&postings_list[offset..offset + len])
-            },
-            PostingsListView::Rewritten { ref postings_list, .. } => postings_list,
-        }
-    }
-}

meilisearch-core/src/criterion/attribute.rs

@@ -1,37 +0,0 @@
-use std::cmp::Ordering;
-use slice_group_by::GroupBy;
-use crate::{RawDocument, MResult};
-use crate::bucket_sort::SimpleMatch;
-use super::{Criterion, Context, ContextMut, prepare_bare_matches};
-
-pub struct Attribute;
-
-impl Criterion for Attribute {
-    fn name(&self) -> &str { "attribute" }
-
-    fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>(
-        &self,
-        ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>,
-        documents: &mut [RawDocument<'r, 'tag>],
-    ) -> MResult<()>
-    {
-        prepare_bare_matches(documents, ctx.postings_lists, ctx.query_mapping);
-        Ok(())
-    }
-
-    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        #[inline]
-        fn sum_of_attribute(matches: &[SimpleMatch]) -> usize {
-            let mut sum_of_attribute = 0;
-            for group in matches.linear_group_by_key(|bm| bm.query_index) {
-                sum_of_attribute += group[0].attribute as usize;
-            }
-            sum_of_attribute
-        }
-
-        let lhs = sum_of_attribute(&lhs.processed_matches);
-        let rhs = sum_of_attribute(&rhs.processed_matches);
-
-        lhs.cmp(&rhs)
-    }
-}

meilisearch-core/src/criterion/document_id.rs

@@ -1,16 +0,0 @@
-use std::cmp::Ordering;
-use crate::RawDocument;
-use super::{Criterion, Context};
-
-pub struct DocumentId;
-
-impl Criterion for DocumentId {
-    fn name(&self) -> &str { "stable document id" }
-
-    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = &lhs.id;
-        let rhs = &rhs.id;
-
-        lhs.cmp(rhs)
-    }
-}

meilisearch-core/src/criterion/exactness.rs

@@ -1,78 +0,0 @@
-use std::cmp::{Ordering, Reverse};
-use std::collections::hash_map::{HashMap, Entry};
-use meilisearch_schema::IndexedPos;
-use slice_group_by::GroupBy;
-use crate::{RawDocument, MResult};
-use crate::bucket_sort::BareMatch;
-use super::{Criterion, Context, ContextMut};
-
-pub struct Exactness;
-
-impl Criterion for Exactness {
-    fn name(&self) -> &str { "exactness" }
-
-    fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>(
-        &self,
-        ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>,
-        documents: &mut [RawDocument<'r, 'tag>],
-    ) -> MResult<()>
-    {
-        let store = ctx.documents_fields_counts_store;
-        let reader = ctx.reader;
-
-        'documents: for doc in documents {
-            doc.bare_matches.sort_unstable_by_key(|bm| (bm.query_index, Reverse(bm.is_exact)));
-
-            // mark the document if we find a "one word field" that matches
-            let mut fields_counts = HashMap::new();
-            for group in doc.bare_matches.linear_group_by_key(|bm| bm.query_index) {
-                for group in group.linear_group_by_key(|bm| bm.is_exact) {
-                    if !group[0].is_exact { break }
-
-                    for bm in group {
-                        for di in ctx.postings_lists[bm.postings_list].as_ref() {
-
-                            let attr = IndexedPos(di.attribute);
-                            let count = match fields_counts.entry(attr) {
-                                Entry::Occupied(entry) => *entry.get(),
-                                Entry::Vacant(entry) => {
-                                    let count = store.document_field_count(reader, doc.id, attr)?;
-                                    *entry.insert(count)
-                                },
-                            };
-
-                            if count == Some(1) {
-                                doc.contains_one_word_field = true;
-                                continue 'documents
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        Ok(())
-    }
-
-    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        #[inline]
-        fn sum_exact_query_words(matches: &[BareMatch]) -> usize {
-            let mut sum_exact_query_words = 0;
-
-            for group in matches.linear_group_by_key(|bm| bm.query_index) {
-                sum_exact_query_words += group[0].is_exact as usize;
-            }
-
-            sum_exact_query_words
-        }
-
-        // does it contains a "one word field"
-        lhs.contains_one_word_field.cmp(&rhs.contains_one_word_field).reverse()
-        // if not, with document contains the more exact words
-        .then_with(|| {
-            let lhs = sum_exact_query_words(&lhs.bare_matches);
-            let rhs = sum_exact_query_words(&rhs.bare_matches);
-            lhs.cmp(&rhs).reverse()
-        })
-    }
-}

meilisearch-core/src/criterion/mod.rs

@@ -1,291 +0,0 @@
-use std::cmp::{self, Ordering};
-use std::collections::HashMap;
-use std::ops::Range;
-
-use compact_arena::SmallArena;
-use sdset::SetBuf;
-use slice_group_by::GroupBy;
-
-use crate::bucket_sort::{SimpleMatch, PostingsListView};
-use crate::database::MainT;
-use crate::query_tree::QueryId;
-use crate::{store, RawDocument, MResult};
-
-mod typo;
-mod words;
-mod proximity;
-mod attribute;
-mod words_position;
-mod exactness;
-mod document_id;
-mod sort_by_attr;
-
-pub use self::typo::Typo;
-pub use self::words::Words;
-pub use self::proximity::Proximity;
-pub use self::attribute::Attribute;
-pub use self::words_position::WordsPosition;
-pub use self::exactness::Exactness;
-pub use self::document_id::DocumentId;
-pub use self::sort_by_attr::SortByAttr;
-
-pub trait Criterion {
-    fn name(&self) -> &str;
-
-    fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>(
-        &self,
-        _ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>,
-        _documents: &mut [RawDocument<'r, 'tag>],
-    ) -> MResult<()>
-    {
-        Ok(())
-    }
-
-    fn evaluate<'p, 'tag, 'txn, 'q, 'r>(
-        &self,
-        ctx: &Context<'p, 'tag, 'txn, 'q>,
-        lhs: &RawDocument<'r, 'tag>,
-        rhs: &RawDocument<'r, 'tag>,
-    ) -> Ordering;
-
-    #[inline]
-    fn eq<'p, 'tag, 'txn, 'q, 'r>(
-        &self,
-        ctx: &Context<'p, 'tag, 'txn, 'q>,
-        lhs: &RawDocument<'r, 'tag>,
-        rhs: &RawDocument<'r, 'tag>,
-    ) -> bool
-    {
-        self.evaluate(ctx, lhs, rhs) == Ordering::Equal
-    }
-}
-
-pub struct ContextMut<'h, 'p, 'tag, 'txn, 'q> {
-    pub reader: &'h heed::RoTxn<MainT>,
-    pub postings_lists: &'p mut SmallArena<'tag, PostingsListView<'txn>>,
-    pub query_mapping: &'q HashMap<QueryId, Range<usize>>,
-    pub documents_fields_counts_store: store::DocumentsFieldsCounts,
-}
-
-pub struct Context<'p, 'tag, 'txn, 'q> {
-    pub postings_lists: &'p SmallArena<'tag, PostingsListView<'txn>>,
-    pub query_mapping: &'q HashMap<QueryId, Range<usize>>,
-}
-
-#[derive(Default)]
-pub struct CriteriaBuilder<'a> {
-    inner: Vec<Box<dyn Criterion + 'a>>,
-}
-
-impl<'a> CriteriaBuilder<'a> {
-    pub fn new() -> CriteriaBuilder<'a> {
-        CriteriaBuilder { inner: Vec::new() }
-    }
-
-    pub fn with_capacity(capacity: usize) -> CriteriaBuilder<'a> {
-        CriteriaBuilder {
-            inner: Vec::with_capacity(capacity),
-        }
-    }
-
-    pub fn reserve(&mut self, additional: usize) {
-        self.inner.reserve(additional)
-    }
-
-    pub fn add<C: 'a>(mut self, criterion: C) -> CriteriaBuilder<'a>
-    where
-        C: Criterion,
-    {
-        self.push(criterion);
-        self
-    }
-
-    pub fn push<C: 'a>(&mut self, criterion: C)
-    where
-        C: Criterion,
-    {
-        self.inner.push(Box::new(criterion));
-    }
-
-    pub fn build(self) -> Criteria<'a> {
-        Criteria { inner: self.inner }
-    }
-}
-
-pub struct Criteria<'a> {
-    inner: Vec<Box<dyn Criterion + 'a>>,
-}
-
-impl<'a> Default for Criteria<'a> {
-    fn default() -> Self {
-        CriteriaBuilder::with_capacity(7)
-            .add(Typo)
-            .add(Words)
-            .add(Proximity)
-            .add(Attribute)
-            .add(WordsPosition)
-            .add(Exactness)
-            .add(DocumentId)
-            .build()
-    }
-}
-
-impl<'a> AsRef<[Box<dyn Criterion + 'a>]> for Criteria<'a> {
-    fn as_ref(&self) -> &[Box<dyn Criterion + 'a>] {
-        &self.inner
-    }
-}
-
-fn prepare_query_distances<'a, 'tag, 'txn>(
-    documents: &mut [RawDocument<'a, 'tag>],
-    query_mapping: &HashMap<QueryId, Range<usize>>,
-    postings_lists: &SmallArena<'tag, PostingsListView<'txn>>,
-) {
-    for document in documents {
-        if !document.processed_distances.is_empty() { continue }
-
-        let mut processed = Vec::new();
-        for m in document.bare_matches.iter() {
-            if postings_lists[m.postings_list].is_empty() { continue }
-
-            let range = query_mapping[&(m.query_index as usize)].clone();
-            let new_len = cmp::max(range.end as usize, processed.len());
-            processed.resize(new_len, None);
-
-            for index in range {
-                let index = index as usize;
-                processed[index] = match processed[index] {
-                    Some(distance) if distance > m.distance => Some(m.distance),
-                    Some(distance) => Some(distance),
-                    None => Some(m.distance),
-                };
-            }
-        }
-
-        document.processed_distances = processed;
-    }
-}
-
-fn prepare_bare_matches<'a, 'tag, 'txn>(
-    documents: &mut [RawDocument<'a, 'tag>],
-    postings_lists: &mut SmallArena<'tag, PostingsListView<'txn>>,
-    query_mapping: &HashMap<QueryId, Range<usize>>,
-) {
-    for document in documents {
-        if !document.processed_matches.is_empty() { continue }
-
-        let mut processed = Vec::new();
-        for m in document.bare_matches.iter() {
-            let postings_list = &postings_lists[m.postings_list];
-            processed.reserve(postings_list.len());
-            for di in postings_list.as_ref() {
-                let simple_match = SimpleMatch {
-                    query_index: m.query_index,
-                    distance: m.distance,
-                    attribute: di.attribute,
-                    word_index: di.word_index,
-                    is_exact: m.is_exact,
-                };
-                processed.push(simple_match);
-            }
-        }
-
-        let processed = multiword_rewrite_matches(&mut processed, query_mapping);
-        document.processed_matches = processed.into_vec();
-    }
-}
-
-fn multiword_rewrite_matches(
-    matches: &mut [SimpleMatch],
-    query_mapping: &HashMap<QueryId, Range<usize>>,
-) -> SetBuf<SimpleMatch>
-{
-    matches.sort_unstable_by_key(|m| (m.attribute, m.word_index));
-
-    let mut padded_matches = Vec::with_capacity(matches.len());
-
-    // let before_padding = Instant::now();
-    // for each attribute of each document
-    for same_document_attribute in matches.linear_group_by_key(|m| m.attribute) {
-        // padding will only be applied
-        // to word indices in the same attribute
-        let mut padding = 0;
-        let mut iter = same_document_attribute.linear_group_by_key(|m| m.word_index);
-
-        // for each match at the same position
-        // in this document attribute
-        while let Some(same_word_index) = iter.next() {
-            // find the biggest padding
-            let mut biggest = 0;
-            for match_ in same_word_index {
-                let mut replacement = query_mapping[&(match_.query_index as usize)].clone();
-                let replacement_len = replacement.len();
-                let nexts = iter.remainder().linear_group_by_key(|m| m.word_index);
-
-                if let Some(query_index) = replacement.next() {
-                    let word_index = match_.word_index + padding as u16;
-                    let match_ = SimpleMatch { query_index, word_index, ..*match_ };
-                    padded_matches.push(match_);
-                }
-
-                let mut found = false;
-
-                // look ahead and if there already is a match
-                // corresponding to this padding word, abort the padding
-                'padding: for (x, next_group) in nexts.enumerate() {
-                    for (i, query_index) in replacement.clone().enumerate().skip(x) {
-                        let word_index = match_.word_index + padding as u16 + (i + 1) as u16;
-                        let padmatch = SimpleMatch { query_index, word_index, ..*match_ };
-
-                        for nmatch_ in next_group {
-                            let mut rep = query_mapping[&(nmatch_.query_index as usize)].clone();
-                            let query_index = rep.next().unwrap();
-                            if query_index == padmatch.query_index {
-                                if !found {
-                                    // if we find a corresponding padding for the
-                                    // first time we must push preceding paddings
-                                    for (i, query_index) in replacement.clone().enumerate().take(i) {
-                                        let word_index = match_.word_index + padding as u16 + (i + 1) as u16;
-                                        let match_ = SimpleMatch { query_index, word_index, ..*match_ };
-                                        padded_matches.push(match_);
-                                        biggest = biggest.max(i + 1);
-                                    }
-                                }
-
-                                padded_matches.push(padmatch);
-                                found = true;
-                                continue 'padding;
-                            }
-                        }
-                    }
-
-                    // if we do not find a corresponding padding in the
-                    // next groups so stop here and pad what was found
-                    break;
-                }
-
-                if !found {
-                    // if no padding was found in the following matches
-                    // we must insert the entire padding
-                    for (i, query_index) in replacement.enumerate() {
-                        let word_index = match_.word_index + padding as u16 + (i + 1) as u16;
-                        let match_ = SimpleMatch { query_index, word_index, ..*match_ };
-                        padded_matches.push(match_);
-                    }
-
-                    biggest = biggest.max(replacement_len - 1);
-                }
-            }
-
-            padding += biggest;
-        }
-    }
-
-    // debug!("padding matches took {:.02?}", before_padding.elapsed());
-
-    // With this check we can see that the loop above takes something
-    // like 43% of the search time even when no rewrite is needed.
-    // assert_eq!(before_matches, padded_matches);
-
-    SetBuf::from_dirty(padded_matches)
-}

meilisearch-core/src/criterion/proximity.rs

@@ -1,68 +0,0 @@
-use std::cmp::{self, Ordering};
-use slice_group_by::GroupBy;
-use crate::bucket_sort::{SimpleMatch};
-use crate::{RawDocument, MResult};
-use super::{Criterion, Context, ContextMut, prepare_bare_matches};
-
-const MAX_DISTANCE: u16 = 8;
-
-pub struct Proximity;
-
-impl Criterion for Proximity {
-    fn name(&self) -> &str { "proximity" }
-
-    fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>(
-        &self,
-        ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>,
-        documents: &mut [RawDocument<'r, 'tag>],
-    ) -> MResult<()>
-    {
-        prepare_bare_matches(documents, ctx.postings_lists, ctx.query_mapping);
-        Ok(())
-    }
-
-    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        fn index_proximity(lhs: u16, rhs: u16) -> u16 {
-            if lhs < rhs {
-                cmp::min(rhs - lhs, MAX_DISTANCE)
-            } else {
-                cmp::min(lhs - rhs, MAX_DISTANCE) + 1
-            }
-        }
-
-        fn attribute_proximity(lhs: SimpleMatch, rhs: SimpleMatch) -> u16 {
-            if lhs.attribute != rhs.attribute { MAX_DISTANCE }
-            else { index_proximity(lhs.word_index, rhs.word_index) }
-        }
-
-        fn min_proximity(lhs: &[SimpleMatch], rhs: &[SimpleMatch]) -> u16 {
-            let mut min_prox = u16::max_value();
-            for a in lhs {
-                for b in rhs {
-                    let prox = attribute_proximity(*a, *b);
-                    min_prox = cmp::min(min_prox, prox);
-                }
-            }
-            min_prox
-        }
-
-        fn matches_proximity(matches: &[SimpleMatch],) -> u16 {
-            let mut proximity = 0;
-            let mut iter = matches.linear_group_by_key(|m| m.query_index);
-
-            // iterate over groups by windows of size 2
-            let mut last = iter.next();
-            while let (Some(lhs), Some(rhs)) = (last, iter.next()) {
-                proximity += min_proximity(lhs, rhs);
-                last = Some(rhs);
-            }
-
-            proximity
-        }
-
-        let lhs = matches_proximity(&lhs.processed_matches);
-        let rhs = matches_proximity(&rhs.processed_matches);
-
-        lhs.cmp(&rhs)
-    }
-}

meilisearch-core/src/criterion/sort_by_attr.rs

@@ -1,129 +0,0 @@
-use std::cmp::Ordering;
-use std::error::Error;
-use std::fmt;
-use meilisearch_schema::{Schema, FieldId};
-use crate::{RankedMap, RawDocument};
-use super::{Criterion, Context};
-
-/// An helper struct that permit to sort documents by
-/// some of their stored attributes.
-///
-/// # Note
-///
-/// If a document cannot be deserialized it will be considered [`None`][].
-///
-/// Deserialized documents are compared like `Some(doc0).cmp(&Some(doc1))`,
-/// so you must check the [`Ord`] of `Option` implementation.
-///
-/// [`None`]: https://doc.rust-lang.org/std/option/enum.Option.html#variant.None
-/// [`Ord`]: https://doc.rust-lang.org/std/option/enum.Option.html#impl-Ord
-///
-/// # Example
-///
-/// ```ignore
-/// use serde_derive::Deserialize;
-/// use meilisearch::rank::criterion::*;
-///
-/// let custom_ranking = SortByAttr::lower_is_better(&ranked_map, &schema, "published_at")?;
-///
-/// let builder = CriteriaBuilder::with_capacity(8)
-///        .add(Typo)
-///        .add(Words)
-///        .add(Proximity)
-///        .add(Attribute)
-///        .add(WordsPosition)
-///        .add(Exactness)
-///        .add(custom_ranking)
-///        .add(DocumentId);
-///
-/// let criterion = builder.build();
-///
-/// ```
-pub struct SortByAttr<'a> {
-    ranked_map: &'a RankedMap,
-    field_id: FieldId,
-    reversed: bool,
-}
-
-impl<'a> SortByAttr<'a> {
-    pub fn lower_is_better(
-        ranked_map: &'a RankedMap,
-        schema: &Schema,
-        attr_name: &str,
-    ) -> Result<SortByAttr<'a>, SortByAttrError> {
-        SortByAttr::new(ranked_map, schema, attr_name, false)
-    }
-
-    pub fn higher_is_better(
-        ranked_map: &'a RankedMap,
-        schema: &Schema,
-        attr_name: &str,
-    ) -> Result<SortByAttr<'a>, SortByAttrError> {
-        SortByAttr::new(ranked_map, schema, attr_name, true)
-    }
-
-    fn new(
-        ranked_map: &'a RankedMap,
-        schema: &Schema,
-        attr_name: &str,
-        reversed: bool,
-    ) -> Result<SortByAttr<'a>, SortByAttrError> {
-        let field_id = match schema.id(attr_name) {
-            Some(field_id) => field_id,
-            None => return Err(SortByAttrError::AttributeNotFound),
-        };
-
-        if !schema.is_ranked(field_id) {
-            return Err(SortByAttrError::AttributeNotRegisteredForRanking);
-        }
-
-        Ok(SortByAttr {
-            ranked_map,
-            field_id,
-            reversed,
-        })
-    }
-}
-
-impl Criterion for SortByAttr<'_> {
-    fn name(&self) -> &str {
-        "sort by attribute"
-    }
-
-    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = self.ranked_map.get(lhs.id, self.field_id);
-        let rhs = self.ranked_map.get(rhs.id, self.field_id);
-
-        match (lhs, rhs) {
-            (Some(lhs), Some(rhs)) => {
-                let order = lhs.cmp(&rhs);
-                if self.reversed {
-                    order.reverse()
-                } else {
-                    order
-                }
-            }
-            (None, Some(_)) => Ordering::Greater,
-            (Some(_), None) => Ordering::Less,
-            (None, None) => Ordering::Equal,
-        }
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum SortByAttrError {
-    AttributeNotFound,
-    AttributeNotRegisteredForRanking,
-}
-
-impl fmt::Display for SortByAttrError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        use SortByAttrError::*;
-        match self {
-            AttributeNotFound => f.write_str("attribute not found in the schema"),
-            AttributeNotRegisteredForRanking => f.write_str("attribute not registered for ranking"),
-        }
-    }
-}
-
-impl Error for SortByAttrError {}

meilisearch-core/src/criterion/typo.rs

@@ -1,55 +0,0 @@
-use std::cmp::Ordering;
-use crate::{RawDocument, MResult};
-use super::{Criterion, Context, ContextMut, prepare_query_distances};
-
-pub struct Typo;
-
-impl Criterion for Typo {
-    fn name(&self) -> &str { "typo" }
-
-    fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>(
-        &self,
-        ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>,
-        documents: &mut [RawDocument<'r, 'tag>],
-    ) -> MResult<()>
-    {
-        prepare_query_distances(documents, ctx.query_mapping, ctx.postings_lists);
-        Ok(())
-    }
-
-    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        // This function is a wrong logarithmic 10 function.
-        // It is safe to panic on input number higher than 3,
-        // the number of typos is never bigger than that.
-        #[inline]
-        fn custom_log10(n: u8) -> f32 {
-            match n {
-                0 => 0.0,     // log(1)
-                1 => 0.30102, // log(2)
-                2 => 0.47712, // log(3)
-                3 => 0.60205, // log(4)
-                _ => panic!("invalid number"),
-            }
-        }
-
-        #[inline]
-        fn compute_typos(distances: &[Option<u8>]) -> usize {
-            let mut number_words: usize = 0;
-            let mut sum_typos = 0.0;
-
-            for distance in distances {
-                if let Some(distance) = distance {
-                    sum_typos += custom_log10(*distance);
-                    number_words += 1;
-                }
-            }
-
-            (number_words as f32 / (sum_typos + 1.0) * 1000.0) as usize
-        }
-
-        let lhs = compute_typos(&lhs.processed_distances);
-        let rhs = compute_typos(&rhs.processed_distances);
-
-        lhs.cmp(&rhs).reverse()
-    }
-}

meilisearch-core/src/criterion/words.rs

@@ -1,31 +0,0 @@
-use std::cmp::Ordering;
-use crate::{RawDocument, MResult};
-use super::{Criterion, Context, ContextMut, prepare_query_distances};
-
-pub struct Words;
-
-impl Criterion for Words {
-    fn name(&self) -> &str { "words" }
-
-    fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>(
-        &self,
-        ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>,
-        documents: &mut [RawDocument<'r, 'tag>],
-    ) -> MResult<()>
-    {
-        prepare_query_distances(documents, ctx.query_mapping, ctx.postings_lists);
-        Ok(())
-    }
-
-    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        #[inline]
-        fn number_of_query_words(distances: &[Option<u8>]) -> usize {
-            distances.iter().cloned().filter(Option::is_some).count()
-        }
-
-        let lhs = number_of_query_words(&lhs.processed_distances);
-        let rhs = number_of_query_words(&rhs.processed_distances);
-
-        lhs.cmp(&rhs).reverse()
-    }
-}

meilisearch-core/src/criterion/words_position.rs

@@ -1,37 +0,0 @@
-use std::cmp::Ordering;
-use slice_group_by::GroupBy;
-use crate::bucket_sort::SimpleMatch;
-use crate::{RawDocument, MResult};
-use super::{Criterion, Context, ContextMut, prepare_bare_matches};
-
-pub struct WordsPosition;
-
-impl Criterion for WordsPosition {
-    fn name(&self) -> &str { "words position" }
-
-    fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>(
-        &self,
-        ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>,
-        documents: &mut [RawDocument<'r, 'tag>],
-    ) -> MResult<()>
-    {
-        prepare_bare_matches(documents, ctx.postings_lists, ctx.query_mapping);
-        Ok(())
-    }
-
-    fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        #[inline]
-        fn sum_words_position(matches: &[SimpleMatch]) -> usize {
-            let mut sum_words_position = 0;
-            for group in matches.linear_group_by_key(|bm| bm.query_index) {
-                sum_words_position += group[0].word_index as usize;
-            }
-            sum_words_position
-        }
-
-        let lhs = sum_words_position(&lhs.processed_matches);
-        let rhs = sum_words_position(&rhs.processed_matches);
-
-        lhs.cmp(&rhs)
-    }
-}

meilisearch-core/src/distinct_map.rs

@@ -1,103 +0,0 @@
-use hashbrown::HashMap;
-use std::hash::Hash;
-
-pub struct DistinctMap<K> {
-    inner: HashMap<K, usize>,
-    limit: usize,
-    len: usize,
-}
-
-impl<K: Hash + Eq> DistinctMap<K> {
-    pub fn new(limit: usize) -> Self {
-        DistinctMap {
-            inner: HashMap::new(),
-            limit,
-            len: 0,
-        }
-    }
-
-    pub fn len(&self) -> usize {
-        self.len
-    }
-}
-
-pub struct BufferedDistinctMap<'a, K> {
-    internal: &'a mut DistinctMap<K>,
-    inner: HashMap<K, usize>,
-    len: usize,
-}
-
-impl<'a, K: Hash + Eq> BufferedDistinctMap<'a, K> {
-    pub fn new(internal: &'a mut DistinctMap<K>) -> BufferedDistinctMap<'a, K> {
-        BufferedDistinctMap {
-            internal,
-            inner: HashMap::new(),
-            len: 0,
-        }
-    }
-
-    pub fn register(&mut self, key: K) -> bool {
-        let internal_seen = self.internal.inner.get(&key).unwrap_or(&0);
-        let inner_seen = self.inner.entry(key).or_insert(0);
-        let seen = *internal_seen + *inner_seen;
-
-        if seen < self.internal.limit {
-            *inner_seen += 1;
-            self.len += 1;
-            true
-        } else {
-            false
-        }
-    }
-
-    pub fn register_without_key(&mut self) -> bool {
-        self.len += 1;
-        true
-    }
-
-    pub fn transfert_to_internal(&mut self) {
-        for (k, v) in self.inner.drain() {
-            let value = self.internal.inner.entry(k).or_insert(0);
-            *value += v;
-        }
-
-        self.internal.len += self.len;
-        self.len = 0;
-    }
-
-    pub fn len(&self) -> usize {
-        self.internal.len() + self.len
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn easy_distinct_map() {
-        let mut map = DistinctMap::new(2);
-        let mut buffered = BufferedDistinctMap::new(&mut map);
-
-        for x in &[1, 1, 1, 2, 3, 4, 5, 6, 6, 6, 6, 6] {
-            buffered.register(x);
-        }
-        buffered.transfert_to_internal();
-        assert_eq!(map.len(), 8);
-
-        let mut map = DistinctMap::new(2);
-        let mut buffered = BufferedDistinctMap::new(&mut map);
-        assert_eq!(buffered.register(1), true);
-        assert_eq!(buffered.register(1), true);
-        assert_eq!(buffered.register(1), false);
-        assert_eq!(buffered.register(1), false);
-
-        assert_eq!(buffered.register(2), true);
-        assert_eq!(buffered.register(3), true);
-        assert_eq!(buffered.register(2), true);
-        assert_eq!(buffered.register(2), false);
-
-        buffered.transfert_to_internal();
-        assert_eq!(map.len(), 5);
-    }
-}

meilisearch-core/src/error.rs

@@ -1,131 +0,0 @@
-use crate::serde::{DeserializerError, SerializerError};
-use serde_json::Error as SerdeJsonError;
-use std::{error, fmt, io};
-
-pub use heed::Error as HeedError;
-pub use fst::Error as FstError;
-pub use bincode::Error as BincodeError;
-
-pub type MResult<T> = Result<T, Error>;
-
-#[derive(Debug)]
-pub enum Error {
-    Io(io::Error),
-    IndexAlreadyExists,
-    MissingPrimaryKey,
-    SchemaMissing,
-    WordIndexMissing,
-    MissingDocumentId,
-    MaxFieldsLimitExceeded,
-    Schema(meilisearch_schema::Error),
-    Zlmdb(heed::Error),
-    Fst(fst::Error),
-    SerdeJson(SerdeJsonError),
-    Bincode(bincode::Error),
-    Serializer(SerializerError),
-    Deserializer(DeserializerError),
-    UnsupportedOperation(UnsupportedOperation),
-}
-
-impl From<io::Error> for Error {
-    fn from(error: io::Error) -> Error {
-        Error::Io(error)
-    }
-}
-
-impl From<meilisearch_schema::Error> for Error {
-    fn from(error: meilisearch_schema::Error) -> Error {
-        Error::Schema(error)
-    }
-}
-
-impl From<HeedError> for Error {
-    fn from(error: HeedError) -> Error {
-        Error::Zlmdb(error)
-    }
-}
-
-impl From<FstError> for Error {
-    fn from(error: FstError) -> Error {
-        Error::Fst(error)
-    }
-}
-
-impl From<SerdeJsonError> for Error {
-    fn from(error: SerdeJsonError) -> Error {
-        Error::SerdeJson(error)
-    }
-}
-
-impl From<BincodeError> for Error {
-    fn from(error: BincodeError) -> Error {
-        Error::Bincode(error)
-    }
-}
-
-impl From<SerializerError> for Error {
-    fn from(error: SerializerError) -> Error {
-        Error::Serializer(error)
-    }
-}
-
-impl From<DeserializerError> for Error {
-    fn from(error: DeserializerError) -> Error {
-        Error::Deserializer(error)
-    }
-}
-
-impl From<UnsupportedOperation> for Error {
-    fn from(op: UnsupportedOperation) -> Error {
-        Error::UnsupportedOperation(op)
-    }
-}
-
-impl fmt::Display for Error {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        use self::Error::*;
-        match self {
-            Io(e) => write!(f, "{}", e),
-            IndexAlreadyExists => write!(f, "index already exists"),
-            MissingPrimaryKey => write!(f, "schema cannot be built without a primary key"),
-            SchemaMissing => write!(f, "this index does not have a schema"),
-            WordIndexMissing => write!(f, "this index does not have a word index"),
-            MissingDocumentId => write!(f, "document id is missing"),
-            MaxFieldsLimitExceeded => write!(f, "maximum number of fields in a document exceeded"),
-            Schema(e) => write!(f, "schema error; {}", e),
-            Zlmdb(e) => write!(f, "heed error; {}", e),
-            Fst(e) => write!(f, "fst error; {}", e),
-            SerdeJson(e) => write!(f, "serde json error; {}", e),
-            Bincode(e) => write!(f, "bincode error; {}", e),
-            Serializer(e) => write!(f, "serializer error; {}", e),
-            Deserializer(e) => write!(f, "deserializer error; {}", e),
-            UnsupportedOperation(op) => write!(f, "unsupported operation; {}", op),
-        }
-    }
-}
-
-impl error::Error for Error {}
-
-#[derive(Debug)]
-pub enum UnsupportedOperation {
-    SchemaAlreadyExists,
-    CannotUpdateSchemaPrimaryKey,
-    CannotReorderSchemaAttribute,
-    CanOnlyIntroduceNewSchemaAttributesAtEnd,
-    CannotRemoveSchemaAttribute,
-}
-
-impl fmt::Display for UnsupportedOperation {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        use self::UnsupportedOperation::*;
-        match self {
-            SchemaAlreadyExists => write!(f, "Cannot update index which already have a schema"),
-            CannotUpdateSchemaPrimaryKey => write!(f, "Cannot update the primary key of a schema"),
-            CannotReorderSchemaAttribute => write!(f, "Cannot reorder the attributes of a schema"),
-            CanOnlyIntroduceNewSchemaAttributesAtEnd => {
-                write!(f, "Can only introduce new attributes at end of a schema")
-            }
-            CannotRemoveSchemaAttribute => write!(f, "Cannot remove attributes from a schema"),
-        }
-    }
-}

meilisearch-core/src/levenshtein.rs

@@ -1,134 +0,0 @@
-use std::cmp::min;
-use std::collections::BTreeMap;
-use std::ops::{Index, IndexMut};
-
-// A simple wrapper around vec so we can get contiguous but index it like it's 2D array.
-struct N2Array<T> {
-    y_size: usize,
-    buf: Vec<T>,
-}
-
-impl<T: Clone> N2Array<T> {
-    fn new(x: usize, y: usize, value: T) -> N2Array<T> {
-        N2Array {
-            y_size: y,
-            buf: vec![value; x * y],
-        }
-    }
-}
-
-impl<T> Index<(usize, usize)> for N2Array<T> {
-    type Output = T;
-
-    #[inline]
-    fn index(&self, (x, y): (usize, usize)) -> &T {
-        &self.buf[(x * self.y_size) + y]
-    }
-}
-
-impl<T> IndexMut<(usize, usize)> for N2Array<T> {
-    #[inline]
-    fn index_mut(&mut self, (x, y): (usize, usize)) -> &mut T {
-        &mut self.buf[(x * self.y_size) + y]
-    }
-}
-
-pub fn prefix_damerau_levenshtein(source: &[u8], target: &[u8]) -> (u32, usize) {
-    let (n, m) = (source.len(), target.len());
-
-    assert!(
-        n <= m,
-        "the source string must be shorter than the target one"
-    );
-
-    if n == 0 {
-        return (m as u32, 0);
-    }
-    if m == 0 {
-        return (n as u32, 0);
-    }
-
-    if n == m && source == target {
-        return (0, m);
-    }
-
-    let inf = n + m;
-    let mut matrix = N2Array::new(n + 2, m + 2, 0);
-
-    matrix[(0, 0)] = inf;
-    for i in 0..n + 1 {
-        matrix[(i + 1, 0)] = inf;
-        matrix[(i + 1, 1)] = i;
-    }
-    for j in 0..m + 1 {
-        matrix[(0, j + 1)] = inf;
-        matrix[(1, j + 1)] = j;
-    }
-
-    let mut last_row = BTreeMap::new();
-
-    for (row, char_s) in source.iter().enumerate() {
-        let mut last_match_col = 0;
-        let row = row + 1;
-
-        for (col, char_t) in target.iter().enumerate() {
-            let col = col + 1;
-            let last_match_row = *last_row.get(&char_t).unwrap_or(&0);
-            let cost = if char_s == char_t { 0 } else { 1 };
-
-            let dist_add = matrix[(row, col + 1)] + 1;
-            let dist_del = matrix[(row + 1, col)] + 1;
-            let dist_sub = matrix[(row, col)] + cost;
-            let dist_trans = matrix[(last_match_row, last_match_col)]
-                + (row - last_match_row - 1)
-                + 1
-                + (col - last_match_col - 1);
-
-            let dist = min(min(dist_add, dist_del), min(dist_sub, dist_trans));
-
-            matrix[(row + 1, col + 1)] = dist;
-
-            if cost == 0 {
-                last_match_col = col;
-            }
-        }
-
-        last_row.insert(char_s, row);
-    }
-
-    let mut minimum = (u32::max_value(), 0);
-
-    for x in n..=m {
-        let dist = matrix[(n + 1, x + 1)] as u32;
-        if dist < minimum.0 {
-            minimum = (dist, x)
-        }
-    }
-
-    minimum
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn matched_length() {
-        let query = "Levenste";
-        let text = "Levenshtein";
-
-        let (dist, length) = prefix_damerau_levenshtein(query.as_bytes(), text.as_bytes());
-        assert_eq!(dist, 1);
-        assert_eq!(&text[..length], "Levenshte");
-    }
-
-    #[test]
-    #[should_panic]
-    fn matched_length_panic() {
-        let query = "Levenshtein";
-        let text = "Levenste";
-
-        // this function will panic if source if longer than target
-        prefix_damerau_levenshtein(query.as_bytes(), text.as_bytes());
-    }
-}

meilisearch-core/src/lib.rs

@@ -1,191 +0,0 @@
-#[cfg(test)]
-#[macro_use]
-extern crate assert_matches;
-
-mod automaton;
-mod bucket_sort;
-mod database;
-mod distinct_map;
-mod error;
-mod levenshtein;
-mod number;
-mod query_builder;
-mod query_tree;
-mod query_words_mapper;
-mod ranked_map;
-mod raw_document;
-mod reordered_attrs;
-mod update;
-pub mod settings;
-pub mod criterion;
-pub mod raw_indexer;
-pub mod serde;
-pub mod store;
-
-pub use self::database::{BoxUpdateFn, Database, MainT, UpdateT};
-pub use self::error::{Error, HeedError, FstError, MResult};
-pub use self::number::{Number, ParseNumberError};
-pub use self::ranked_map::RankedMap;
-pub use self::raw_document::RawDocument;
-pub use self::store::Index;
-pub use self::update::{EnqueuedUpdateResult, ProcessedUpdateResult, UpdateStatus, UpdateType};
-pub use meilisearch_types::{DocIndex, DocumentId, Highlight};
-pub use meilisearch_schema::Schema;
-pub use query_words_mapper::QueryWordsMapper;
-
-use std::convert::TryFrom;
-use std::collections::HashMap;
-use compact_arena::SmallArena;
-use log::{error, trace};
-
-use crate::bucket_sort::PostingsListView;
-use crate::levenshtein::prefix_damerau_levenshtein;
-use crate::query_tree::{QueryId, QueryKind};
-use crate::reordered_attrs::ReorderedAttrs;
-
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
-pub struct Document {
-    pub id: DocumentId,
-    pub highlights: Vec<Highlight>,
-
-    #[cfg(test)]
-    pub matches: Vec<crate::bucket_sort::SimpleMatch>,
-}
-
-fn highlights_from_raw_document<'a, 'tag, 'txn>(
-    raw_document: &RawDocument<'a, 'tag>,
-    queries_kinds: &HashMap<QueryId, &QueryKind>,
-    arena: &SmallArena<'tag, PostingsListView<'txn>>,
-    searchable_attrs: Option<&ReorderedAttrs>,
-    schema: &Schema,
-) -> Vec<Highlight>
-{
-    let mut highlights = Vec::new();
-
-    for bm in raw_document.bare_matches.iter() {
-        let postings_list = &arena[bm.postings_list];
-        let input = postings_list.input();
-        let kind = &queries_kinds.get(&bm.query_index);
-
-        for di in postings_list.iter() {
-            let covered_area = match kind {
-                Some(QueryKind::NonTolerant(query)) | Some(QueryKind::Tolerant(query)) => {
-                    let len = if query.len() > input.len() {
-                        input.len()
-                    } else {
-                        prefix_damerau_levenshtein(query.as_bytes(), input).1
-                    };
-                    u16::try_from(len).unwrap_or(u16::max_value())
-                },
-                _ => di.char_length,
-            };
-
-            let attribute = searchable_attrs
-                .and_then(|sa| sa.reverse(di.attribute))
-                .unwrap_or(di.attribute);
-
-            let attribute = match schema.indexed_pos_to_field_id(attribute) {
-                Some(field_id) => field_id.0,
-                None => {
-                    error!("Cannot convert indexed_pos {} to field_id", attribute);
-                    trace!("Schema is compromized; {:?}", schema);
-                    continue
-                }
-            };
-
-            let highlight = Highlight {
-                attribute,
-                char_index: di.char_index,
-                char_length: covered_area,
-            };
-
-            highlights.push(highlight);
-        }
-    }
-
-    highlights
-}
-
-impl Document {
-    #[cfg(not(test))]
-    pub fn from_highlights(id: DocumentId, highlights: &[Highlight]) -> Document {
-        Document { id, highlights: highlights.to_owned() }
-    }
-
-    #[cfg(test)]
-    pub fn from_highlights(id: DocumentId, highlights: &[Highlight]) -> Document {
-        Document { id, highlights: highlights.to_owned(), matches: Vec::new() }
-    }
-
-    #[cfg(not(test))]
-    pub fn from_raw<'a, 'tag, 'txn>(
-        raw_document: RawDocument<'a, 'tag>,
-        queries_kinds: &HashMap<QueryId, &QueryKind>,
-        arena: &SmallArena<'tag, PostingsListView<'txn>>,
-        searchable_attrs: Option<&ReorderedAttrs>,
-        schema: &Schema,
-    ) -> Document
-    {
-        let highlights = highlights_from_raw_document(
-            &raw_document,
-            queries_kinds,
-            arena,
-            searchable_attrs,
-            schema,
-        );
-
-        Document { id: raw_document.id, highlights }
-    }
-
-    #[cfg(test)]
-    pub fn from_raw<'a, 'tag, 'txn>(
-        raw_document: RawDocument<'a, 'tag>,
-        queries_kinds: &HashMap<QueryId, &QueryKind>,
-        arena: &SmallArena<'tag, PostingsListView<'txn>>,
-        searchable_attrs: Option<&ReorderedAttrs>,
-        schema: &Schema,
-    ) -> Document
-    {
-        use crate::bucket_sort::SimpleMatch;
-
-        let highlights = highlights_from_raw_document(
-            &raw_document,
-            queries_kinds,
-            arena,
-            searchable_attrs,
-            schema,
-        );
-
-        let mut matches = Vec::new();
-        for sm in raw_document.processed_matches {
-            let attribute = searchable_attrs
-                .and_then(|sa| sa.reverse(sm.attribute))
-                .unwrap_or(sm.attribute);
-
-            let attribute = match schema.indexed_pos_to_field_id(attribute) {
-                Some(field_id) => field_id.0,
-                None => {
-                    error!("Cannot convert indexed_pos {} to field_id", attribute);
-                    trace!("Schema is compromized; {:?}", schema);
-                    continue
-                }
-            };
-
-            matches.push(SimpleMatch { attribute, ..sm });
-        }
-        matches.sort_unstable();
-
-        Document { id: raw_document.id, highlights, matches }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::mem;
-
-    #[test]
-    fn docindex_mem_size() {
-        assert_eq!(mem::size_of::<DocIndex>(), 16);
-    }
-}

meilisearch-core/src/number.rs

@@ -1,120 +0,0 @@
-use std::cmp::Ordering;
-use std::fmt;
-use std::num::{ParseFloatError, ParseIntError};
-use std::str::FromStr;
-
-use ordered_float::OrderedFloat;
-use serde::{Deserialize, Serialize};
-
-#[derive(Serialize, Deserialize, Debug, Copy, Clone, Hash)]
-pub enum Number {
-    Unsigned(u64),
-    Signed(i64),
-    Float(OrderedFloat<f64>),
-    Null,
-}
-
-impl Default for Number {
-    fn default() -> Self {
-        Self::Null
-    }
-}
-
-impl FromStr for Number {
-    type Err = ParseNumberError;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        let uint_error = match u64::from_str(s) {
-            Ok(unsigned) => return Ok(Number::Unsigned(unsigned)),
-            Err(error) => error,
-        };
-
-        let int_error = match i64::from_str(s) {
-            Ok(signed) => return Ok(Number::Signed(signed)),
-            Err(error) => error,
-        };
-
-        let float_error = match f64::from_str(s) {
-            Ok(float) => return Ok(Number::Float(OrderedFloat(float))),
-            Err(error) => error,
-        };
-
-        Err(ParseNumberError {
-            uint_error,
-            int_error,
-            float_error,
-        })
-    }
-}
-
-impl PartialEq for Number {
-    fn eq(&self, other: &Number) -> bool {
-        self.cmp(other) == Ordering::Equal
-    }
-}
-
-impl Eq for Number {}
-
-impl PartialOrd for Number {
-    fn partial_cmp(&self, other: &Number) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl Ord for Number {
-    fn cmp(&self, other: &Self) -> Ordering {
-        use Number::{Float, Signed, Unsigned, Null};
-
-        match (*self, *other) {
-            (Unsigned(a), Unsigned(b)) => a.cmp(&b),
-            (Unsigned(a), Signed(b)) => {
-                if b < 0 {
-                    Ordering::Greater
-                } else {
-                    a.cmp(&(b as u64))
-                }
-            }
-            (Unsigned(a), Float(b)) => (OrderedFloat(a as f64)).cmp(&b),
-            (Signed(a), Unsigned(b)) => {
-                if a < 0 {
-                    Ordering::Less
-                } else {
-                    (a as u64).cmp(&b)
-                }
-            }
-            (Signed(a), Signed(b)) => a.cmp(&b),
-            (Signed(a), Float(b)) => OrderedFloat(a as f64).cmp(&b),
-            (Float(a), Unsigned(b)) => a.cmp(&OrderedFloat(b as f64)),
-            (Float(a), Signed(b)) => a.cmp(&OrderedFloat(b as f64)),
-            (Float(a), Float(b)) => a.cmp(&b),
-            (Null, Null) => Ordering::Equal,
-            (_, Null) => Ordering::Less,
-            (Null, _) => Ordering::Greater,
-        }
-    }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ParseNumberError {
-    uint_error: ParseIntError,
-    int_error: ParseIntError,
-    float_error: ParseFloatError,
-}
-
-impl fmt::Display for ParseNumberError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        if self.uint_error == self.int_error {
-            write!(
-                f,
-                "can not parse number: {}, {}",
-                self.uint_error, self.float_error
-            )
-        } else {
-            write!(
-                f,
-                "can not parse number: {}, {}, {}",
-                self.uint_error, self.int_error, self.float_error
-            )
-        }
-    }
-}

meilisearch-core/src/query_tree.rs

@@ -1,560 +0,0 @@
-use std::borrow::Cow;
-use std::collections::HashMap;
-use std::hash::{Hash, Hasher};
-use std::ops::Range;
-use std::time::Instant;
-use std::{cmp, fmt, iter::once};
-
-use fst::{IntoStreamer, Streamer};
-use itertools::{EitherOrBoth, merge_join_by};
-use meilisearch_tokenizer::split_query_string;
-use sdset::{Set, SetBuf, SetOperation};
-use log::debug;
-
-use crate::database::MainT;
-use crate::{store, DocumentId, DocIndex, MResult};
-use crate::automaton::{normalize_str, build_dfa, build_prefix_dfa, build_exact_dfa};
-use crate::QueryWordsMapper;
-
-#[derive(Clone, PartialEq, Eq, Hash)]
-pub enum Operation {
-    And(Vec<Operation>),
-    Or(Vec<Operation>),
-    Query(Query),
-}
-
-impl fmt::Debug for Operation {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        fn pprint_tree(f: &mut fmt::Formatter<'_>, op: &Operation, depth: usize) -> fmt::Result {
-            match op {
-                Operation::And(children) => {
-                    writeln!(f, "{:1$}AND", "", depth * 2)?;
-                    children.iter().try_for_each(|c| pprint_tree(f, c, depth + 1))
-                },
-                Operation::Or(children) => {
-                    writeln!(f, "{:1$}OR", "", depth * 2)?;
-                    children.iter().try_for_each(|c| pprint_tree(f, c, depth + 1))
-                },
-                Operation::Query(query) => writeln!(f, "{:2$}{:?}", "", query, depth * 2),
-            }
-        }
-
-        pprint_tree(f, self, 0)
-    }
-}
-
-impl Operation {
-    fn tolerant(id: QueryId, prefix: bool, s: &str) -> Operation {
-        Operation::Query(Query { id, prefix, exact: true, kind: QueryKind::Tolerant(s.to_string()) })
-    }
-
-    fn non_tolerant(id: QueryId, prefix: bool, s: &str) -> Operation {
-        Operation::Query(Query { id, prefix, exact: true, kind: QueryKind::NonTolerant(s.to_string()) })
-    }
-
-    fn phrase2(id: QueryId, prefix: bool, (left, right): (&str, &str)) -> Operation {
-        let kind = QueryKind::Phrase(vec![left.to_owned(), right.to_owned()]);
-        Operation::Query(Query { id, prefix, exact: true, kind })
-    }
-}
-
-pub type QueryId = usize;
-
-#[derive(Clone, Eq)]
-pub struct Query {
-    pub id: QueryId,
-    pub prefix: bool,
-    pub exact: bool,
-    pub kind: QueryKind,
-}
-
-impl PartialEq for Query {
-    fn eq(&self, other: &Self) -> bool {
-        self.prefix == other.prefix && self.kind == other.kind
-    }
-}
-
-impl Hash for Query {
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        self.prefix.hash(state);
-        self.kind.hash(state);
-    }
-}
-
-#[derive(Clone, PartialEq, Eq, Hash)]
-pub enum QueryKind {
-    Tolerant(String),
-    NonTolerant(String),
-    Phrase(Vec<String>),
-}
-
-impl fmt::Debug for Query {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let Query { id, prefix, kind, .. } = self;
-        let prefix = if *prefix { String::from("Prefix") } else { String::default() };
-        match kind {
-            QueryKind::NonTolerant(word) => {
-                f.debug_struct(&(prefix + "NonTolerant")).field("id", &id).field("word", &word).finish()
-            },
-            QueryKind::Tolerant(word) => {
-                f.debug_struct(&(prefix + "Tolerant")).field("id", &id).field("word", &word).finish()
-            },
-            QueryKind::Phrase(words) => {
-                f.debug_struct(&(prefix + "Phrase")).field("id", &id).field("words", &words).finish()
-            },
-        }
-    }
-}
-
-#[derive(Debug, Default)]
-pub struct PostingsList {
-    docids: SetBuf<DocumentId>,
-    matches: SetBuf<DocIndex>,
-}
-
-pub struct Context {
-    pub words_set: fst::Set,
-    pub stop_words: fst::Set,
-    pub synonyms: store::Synonyms,
-    pub postings_lists: store::PostingsLists,
-    pub prefix_postings_lists: store::PrefixPostingsListsCache,
-}
-
-fn split_best_frequency<'a>(reader: &heed::RoTxn<MainT>, ctx: &Context, word: &'a str) -> MResult<Option<(&'a str, &'a str)>> {
-    let chars = word.char_indices().skip(1);
-    let mut best = None;
-
-    for (i, _) in chars {
-        let (left, right) = word.split_at(i);
-
-        let left_freq = ctx.postings_lists
-            .postings_list(reader, left.as_bytes())?
-            .map(|p| p.docids.len())
-            .unwrap_or(0);
-        let right_freq = ctx.postings_lists
-            .postings_list(reader, right.as_bytes())?
-            .map(|p| p.docids.len())
-            .unwrap_or(0);
-
-        let min_freq = cmp::min(left_freq, right_freq);
-        if min_freq != 0 && best.map_or(true, |(old, _, _)| min_freq > old) {
-            best = Some((min_freq, left, right));
-        }
-    }
-
-    Ok(best.map(|(_, l, r)| (l, r)))
-}
-
-fn fetch_synonyms(reader: &heed::RoTxn<MainT>, ctx: &Context, words: &[&str]) -> MResult<Vec<Vec<String>>> {
-    let words = normalize_str(&words.join(" "));
-    let set = ctx.synonyms.synonyms(reader, words.as_bytes())?.unwrap_or_default();
-
-    let mut strings = Vec::new();
-    let mut stream = set.stream();
-    while let Some(input) = stream.next() {
-        if let Ok(input) = std::str::from_utf8(input) {
-            let alts = input.split_ascii_whitespace().map(ToOwned::to_owned).collect();
-            strings.push(alts);
-        }
-    }
-
-    Ok(strings)
-}
-
-fn create_operation<I, F>(iter: I, f: F) -> Operation
-where I: IntoIterator<Item=Operation>,
-      F: Fn(Vec<Operation>) -> Operation,
-{
-    let mut iter = iter.into_iter();
-    match (iter.next(), iter.next()) {
-        (Some(first), None) => first,
-        (first, second) => f(first.into_iter().chain(second).chain(iter).collect()),
-    }
-}
-
-const MAX_NGRAM: usize = 3;
-
-pub fn create_query_tree(
-    reader: &heed::RoTxn<MainT>,
-    ctx: &Context,
-    query: &str,
-) -> MResult<(Operation, HashMap<QueryId, Range<usize>>)>
-{
-    let words = split_query_string(query).map(str::to_lowercase);
-    let words = words.filter(|w| !ctx.stop_words.contains(w));
-    let words: Vec<_> = words.enumerate().collect();
-
-    let mut mapper = QueryWordsMapper::new(words.iter().map(|(_, w)| w));
-
-    fn create_inner(
-        reader: &heed::RoTxn<MainT>,
-        ctx: &Context,
-        mapper: &mut QueryWordsMapper,
-        words: &[(usize, String)],
-    ) -> MResult<Vec<Operation>>
-    {
-        let mut alts = Vec::new();
-
-        for ngram in 1..=MAX_NGRAM {
-            if let Some(group) = words.get(..ngram) {
-                let mut group_ops = Vec::new();
-
-                let tail = &words[ngram..];
-                let is_last = tail.is_empty();
-
-                let mut group_alts = Vec::new();
-                match group {
-                    [(id, word)] => {
-                        let mut idgen = ((id + 1) * 100)..;
-                        let range = (*id)..id+1;
-
-                        let phrase = split_best_frequency(reader, ctx, word)?
-                            .map(|ws| {
-                                let id = idgen.next().unwrap();
-                                idgen.next().unwrap();
-                                mapper.declare(range.clone(), id, &[ws.0, ws.1]);
-                                Operation::phrase2(id, is_last, ws)
-                            });
-
-                        let synonyms = fetch_synonyms(reader, ctx, &[word])?
-                            .into_iter()
-                            .map(|alts| {
-                                let exact = alts.len() == 1;
-                                let id = idgen.next().unwrap();
-                                mapper.declare(range.clone(), id, &alts);
-
-                                let mut idgen = once(id).chain(&mut idgen);
-                                let iter = alts.into_iter().map(|w| {
-                                    let id = idgen.next().unwrap();
-                                    let kind = QueryKind::NonTolerant(w);
-                                    Operation::Query(Query { id, prefix: false, exact, kind })
-                                });
-
-                                create_operation(iter, Operation::And)
-                            });
-
-                        let original = Operation::tolerant(*id, is_last, word);
-
-                        group_alts.push(original);
-                        group_alts.extend(synonyms.chain(phrase));
-                    },
-                    words => {
-                        let id = words[0].0;
-                        let mut idgen = ((id + 1) * 100_usize.pow(ngram as u32))..;
-                        let range = id..id+ngram;
-
-                        let words: Vec<_> = words.iter().map(|(_, s)| s.as_str()).collect();
-
-                        for synonym in fetch_synonyms(reader, ctx, &words)? {
-                            let exact = synonym.len() == 1;
-                            let id = idgen.next().unwrap();
-                            mapper.declare(range.clone(), id, &synonym);
-
-                            let mut idgen = once(id).chain(&mut idgen);
-                            let synonym = synonym.into_iter().map(|s| {
-                                let id = idgen.next().unwrap();
-                                let kind = QueryKind::NonTolerant(s);
-                                Operation::Query(Query { id, prefix: false, exact, kind })
-                            });
-                            group_alts.push(create_operation(synonym, Operation::And));
-                        }
-
-                        let id = idgen.next().unwrap();
-                        let concat = words.concat();
-                        mapper.declare(range.clone(), id, &[&concat]);
-                        group_alts.push(Operation::non_tolerant(id, is_last, &concat));
-                    }
-                }
-
-                group_ops.push(create_operation(group_alts, Operation::Or));
-
-                if !tail.is_empty() {
-                    let tail_ops = create_inner(reader, ctx, mapper, tail)?;
-                    group_ops.push(create_operation(tail_ops, Operation::Or));
-                }
-
-                alts.push(create_operation(group_ops, Operation::And));
-            }
-        }
-
-        Ok(alts)
-    }
-
-    let alternatives = create_inner(reader, ctx, &mut mapper, &words)?;
-    let operation = Operation::Or(alternatives);
-    let mapping = mapper.mapping();
-
-    Ok((operation, mapping))
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct PostingsKey<'o> {
-    pub query: &'o Query,
-    pub input: Vec<u8>,
-    pub distance: u8,
-    pub is_exact: bool,
-}
-
-pub type Postings<'o, 'txn> = HashMap<PostingsKey<'o>, Cow<'txn, Set<DocIndex>>>;
-pub type Cache<'o, 'txn> = HashMap<&'o Operation, Cow<'txn, Set<DocumentId>>>;
-
-pub struct QueryResult<'o, 'txn> {
-    pub docids: Cow<'txn, Set<DocumentId>>,
-    pub queries: Postings<'o, 'txn>,
-}
-
-pub fn traverse_query_tree<'o, 'txn>(
-    reader: &'txn heed::RoTxn<MainT>,
-    ctx: &Context,
-    tree: &'o Operation,
-) -> MResult<QueryResult<'o, 'txn>>
-{
-    fn execute_and<'o, 'txn>(
-        reader: &'txn heed::RoTxn<MainT>,
-        ctx: &Context,
-        cache: &mut Cache<'o, 'txn>,
-        postings: &mut Postings<'o, 'txn>,
-        depth: usize,
-        operations: &'o [Operation],
-    ) -> MResult<Cow<'txn, Set<DocumentId>>>
-    {
-        debug!("{:1$}AND", "", depth * 2);
-
-        let before = Instant::now();
-        let mut results = Vec::new();
-
-        for op in operations {
-            if cache.get(op).is_none() {
-                let docids = match op {
-                    Operation::And(ops) => execute_and(reader, ctx, cache, postings, depth + 1, &ops)?,
-                    Operation::Or(ops) => execute_or(reader, ctx, cache, postings, depth + 1, &ops)?,
-                    Operation::Query(query) => execute_query(reader, ctx, postings, depth + 1, &query)?,
-                };
-                cache.insert(op, docids);
-            }
-        }
-
-        for op in operations {
-            if let Some(docids) = cache.get(op) {
-                results.push(docids.as_ref());
-            }
-        }
-
-        let op = sdset::multi::Intersection::new(results);
-        let docids = op.into_set_buf();
-
-        debug!("{:3$}--- AND fetched {} documents in {:.02?}", "", docids.len(), before.elapsed(), depth * 2);
-
-        Ok(Cow::Owned(docids))
-    }
-
-    fn execute_or<'o, 'txn>(
-        reader: &'txn heed::RoTxn<MainT>,
-        ctx: &Context,
-        cache: &mut Cache<'o, 'txn>,
-        postings: &mut Postings<'o, 'txn>,
-        depth: usize,
-        operations: &'o [Operation],
-    ) -> MResult<Cow<'txn, Set<DocumentId>>>
-    {
-        debug!("{:1$}OR", "", depth * 2);
-
-        let before = Instant::now();
-        let mut results = Vec::new();
-
-        for op in operations {
-            if cache.get(op).is_none() {
-                let docids = match op {
-                    Operation::And(ops) => execute_and(reader, ctx, cache, postings, depth + 1, &ops)?,
-                    Operation::Or(ops) => execute_or(reader, ctx, cache, postings, depth + 1, &ops)?,
-                    Operation::Query(query) => execute_query(reader, ctx, postings, depth + 1, &query)?,
-                };
-                cache.insert(op, docids);
-            }
-        }
-
-        for op in operations {
-            if let Some(docids) = cache.get(op) {
-                results.push(docids.as_ref());
-            }
-        }
-
-        let op = sdset::multi::Union::new(results);
-        let docids = op.into_set_buf();
-
-        debug!("{:3$}--- OR fetched {} documents in {:.02?}", "", docids.len(), before.elapsed(), depth * 2);
-
-        Ok(Cow::Owned(docids))
-    }
-
-    fn execute_query<'o, 'txn>(
-        reader: &'txn heed::RoTxn<MainT>,
-        ctx: &Context,
-        postings: &mut Postings<'o, 'txn>,
-        depth: usize,
-        query: &'o Query,
-    ) -> MResult<Cow<'txn, Set<DocumentId>>>
-    {
-        let before = Instant::now();
-
-        let Query { prefix, kind, exact, .. } = query;
-        let docids: Cow<Set<_>> = match kind {
-            QueryKind::Tolerant(word) => {
-                if *prefix && word.len() <= 2 {
-                    let prefix = {
-                        let mut array = [0; 4];
-                        let bytes = word.as_bytes();
-                        array[..bytes.len()].copy_from_slice(bytes);
-                        array
-                    };
-
-                    // We retrieve the cached postings lists for all
-                    // the words that starts with this short prefix.
-                    let result = ctx.prefix_postings_lists.prefix_postings_list(reader, prefix)?.unwrap_or_default();
-                    let key = PostingsKey { query, input: word.clone().into_bytes(), distance: 0, is_exact: false };
-                    postings.insert(key, result.matches);
-                    let prefix_docids = &result.docids;
-
-                    // We retrieve the exact postings list for the prefix,
-                    // because we must consider these matches as exact.
-                    let result = ctx.postings_lists.postings_list(reader, word.as_bytes())?.unwrap_or_default();
-                    let key = PostingsKey { query, input: word.clone().into_bytes(), distance: 0, is_exact: true };
-                    postings.insert(key, result.matches);
-                    let exact_docids = &result.docids;
-
-                    let before = Instant::now();
-                    let docids = sdset::duo::Union::new(prefix_docids, exact_docids).into_set_buf();
-                    debug!("{:4$}prefix docids ({} and {}) construction took {:.02?}",
-                        "", prefix_docids.len(), exact_docids.len(), before.elapsed(), depth * 2);
-
-                    Cow::Owned(docids)
-
-                } else {
-                    let dfa = if *prefix { build_prefix_dfa(word) } else { build_dfa(word) };
-
-                    let byte = word.as_bytes()[0];
-                    let mut stream = if byte == u8::max_value() {
-                        ctx.words_set.search(&dfa).ge(&[byte]).into_stream()
-                    } else {
-                        ctx.words_set.search(&dfa).ge(&[byte]).lt(&[byte + 1]).into_stream()
-                    };
-
-                    let before = Instant::now();
-                    let mut results = Vec::new();
-                    while let Some(input) = stream.next() {
-                        if let Some(result) = ctx.postings_lists.postings_list(reader, input)? {
-                            let distance = dfa.eval(input).to_u8();
-                            let is_exact = *exact && distance == 0 && input.len() == word.len();
-                            results.push(result.docids);
-                            let key = PostingsKey { query, input: input.to_owned(), distance, is_exact };
-                            postings.insert(key, result.matches);
-                        }
-                    }
-                    debug!("{:3$}docids retrieval ({:?}) took {:.02?}", "", results.len(), before.elapsed(), depth * 2);
-
-                    let before = Instant::now();
-                    let docids = if results.len() > 10 {
-                        let cap = results.iter().map(|dis| dis.len()).sum();
-                        let mut docids = Vec::with_capacity(cap);
-                        for dis in results {
-                            docids.extend_from_slice(&dis);
-                        }
-                        SetBuf::from_dirty(docids)
-                    } else {
-                        let sets = results.iter().map(AsRef::as_ref).collect();
-                        sdset::multi::Union::new(sets).into_set_buf()
-                    };
-                    debug!("{:2$}docids construction took {:.02?}", "", before.elapsed(), depth * 2);
-
-                    Cow::Owned(docids)
-                }
-            },
-            QueryKind::NonTolerant(word) => {
-                // TODO support prefix and non-prefix exact DFA
-                let dfa = build_exact_dfa(word);
-
-                let byte = word.as_bytes()[0];
-                let mut stream = if byte == u8::max_value() {
-                    ctx.words_set.search(&dfa).ge(&[byte]).into_stream()
-                } else {
-                    ctx.words_set.search(&dfa).ge(&[byte]).lt(&[byte + 1]).into_stream()
-                };
-
-                let before = Instant::now();
-                let mut results = Vec::new();
-                while let Some(input) = stream.next() {
-                    if let Some(result) = ctx.postings_lists.postings_list(reader, input)? {
-                        let distance = dfa.eval(input).to_u8();
-                        results.push(result.docids);
-                        let key = PostingsKey { query, input: input.to_owned(), distance, is_exact: *exact };
-                        postings.insert(key, result.matches);
-                    }
-                }
-                debug!("{:3$}docids retrieval ({:?}) took {:.02?}", "", results.len(), before.elapsed(), depth * 2);
-
-                let before = Instant::now();
-                let docids = if results.len() > 10 {
-                    let cap = results.iter().map(|dis| dis.len()).sum();
-                    let mut docids = Vec::with_capacity(cap);
-                    for dis in results {
-                        docids.extend_from_slice(&dis);
-                    }
-                    SetBuf::from_dirty(docids)
-                } else {
-                    let sets = results.iter().map(AsRef::as_ref).collect();
-                    sdset::multi::Union::new(sets).into_set_buf()
-                };
-                debug!("{:2$}docids construction took {:.02?}", "", before.elapsed(), depth * 2);
-
-                Cow::Owned(docids)
-            },
-            QueryKind::Phrase(words) => {
-                // TODO support prefix and non-prefix exact DFA
-                if let [first, second] = words.as_slice() {
-                    let first = ctx.postings_lists.postings_list(reader, first.as_bytes())?.unwrap_or_default();
-                    let second = ctx.postings_lists.postings_list(reader, second.as_bytes())?.unwrap_or_default();
-
-                    let iter = merge_join_by(first.matches.as_slice(), second.matches.as_slice(), |a, b| {
-                        let x = (a.document_id, a.attribute, (a.word_index as u32) + 1);
-                        let y = (b.document_id, b.attribute, b.word_index as u32);
-                        x.cmp(&y)
-                    });
-
-                    let matches: Vec<_> = iter
-                        .filter_map(EitherOrBoth::both)
-                        .flat_map(|(a, b)| once(*a).chain(Some(*b)))
-                        .collect();
-
-                    let before = Instant::now();
-                    let mut docids: Vec<_> = matches.iter().map(|m| m.document_id).collect();
-                    docids.dedup();
-                    let docids = SetBuf::new(docids).unwrap();
-                    debug!("{:2$}docids construction took {:.02?}", "", before.elapsed(), depth * 2);
-
-                    let matches = Cow::Owned(SetBuf::new(matches).unwrap());
-                    let key = PostingsKey { query, input: vec![], distance: 0, is_exact: true };
-                    postings.insert(key, matches);
-
-                    Cow::Owned(docids)
-                } else {
-                    debug!("{:2$}{:?} skipped", "", words, depth * 2);
-                    Cow::default()
-                }
-            },
-        };
-
-        debug!("{:4$}{:?} fetched {:?} documents in {:.02?}", "", query, docids.len(), before.elapsed(), depth * 2);
-        Ok(docids)
-    }
-
-    let mut cache = Cache::new();
-    let mut postings = Postings::new();
-
-    let docids = match tree {
-        Operation::And(ops) => execute_and(reader, ctx, &mut cache, &mut postings, 0, &ops)?,
-        Operation::Or(ops) => execute_or(reader, ctx, &mut cache, &mut postings, 0, &ops)?,
-        Operation::Query(query) => execute_query(reader, ctx, &mut postings, 0, &query)?,
-    };
-
-    Ok(QueryResult { docids, queries: postings })
-}

meilisearch-core/src/query_words_mapper.rs

@@ -1,415 +0,0 @@
-use std::collections::HashMap;
-use std::iter::FromIterator;
-use std::ops::Range;
-use intervaltree::{Element, IntervalTree};
-
-pub type QueryId = usize;
-
-pub struct QueryWordsMapper {
-    originals: Vec<String>,
-    mappings: HashMap<QueryId, (Range<usize>, Vec<String>)>,
-}
-
-impl QueryWordsMapper {
-    pub fn new<I, A>(originals: I) -> QueryWordsMapper
-    where I: IntoIterator<Item = A>,
-          A: ToString,
-    {
-        let originals = originals.into_iter().map(|s| s.to_string()).collect();
-        QueryWordsMapper { originals, mappings: HashMap::new() }
-    }
-
-    pub fn declare<I, A>(&mut self, range: Range<usize>, id: QueryId, replacement: I)
-    where I: IntoIterator<Item = A>,
-          A: ToString,
-    {
-        assert!(range.len() != 0);
-        assert!(self.originals.get(range.clone()).is_some());
-        assert!(id >= self.originals.len());
-
-        let replacement: Vec<_> = replacement.into_iter().map(|s| s.to_string()).collect();
-
-        assert!(!replacement.is_empty());
-
-        // We detect words at the end and at the front of the
-        // replacement that are common with the originals:
-        //
-        //     x a b c d e f g
-        //       ^^^/   \^^^
-        //     a b x c d k j e f
-        //     ^^^           ^^^
-        //
-
-        let left = &self.originals[..range.start];
-        let right = &self.originals[range.end..];
-
-        let common_left = longest_common_prefix(left, &replacement);
-        let common_right = longest_common_prefix(&replacement, right);
-
-        for i in 0..common_left {
-            let range = range.start - common_left + i..range.start - common_left + i + 1;
-            let replacement = vec![replacement[i].clone()];
-            self.mappings.insert(id + i, (range, replacement));
-        }
-
-        {
-            let replacement = replacement[common_left..replacement.len() - common_right].iter().cloned().collect();
-            self.mappings.insert(id + common_left, (range.clone(), replacement));
-        }
-
-        for i in 0..common_right {
-            let id = id + replacement.len() - common_right + i;
-            let range = range.end + i..range.end + i + 1;
-            let replacement = vec![replacement[replacement.len() - common_right + i].clone()];
-            self.mappings.insert(id, (range, replacement));
-        }
-    }
-
-    pub fn mapping(self) -> HashMap<QueryId, Range<usize>> {
-        let mappings = self.mappings.into_iter().map(|(i, (r, v))| (r, (i, v)));
-        let intervals = IntervalTree::from_iter(mappings);
-
-        let mut output = HashMap::new();
-        let mut offset = 0;
-
-        // We map each original word to the biggest number of
-        // associated words.
-        for i in 0..self.originals.len() {
-            let max = intervals.query_point(i)
-                .filter_map(|e| {
-                    if e.range.end - 1 == i {
-                        let len = e.value.1.iter().skip(i - e.range.start).count();
-                        if len != 0 { Some(len) } else { None }
-                    } else { None }
-                })
-                .max()
-                .unwrap_or(1);
-
-            let range = i + offset..i + offset + max;
-            output.insert(i, range);
-            offset += max - 1;
-        }
-
-        // We retrieve the range that each original word
-        // is mapped to and apply it to each of the words.
-        for i in 0..self.originals.len() {
-
-            let iter = intervals.query_point(i).filter(|e| e.range.end - 1 == i);
-            for Element { range, value: (id, words) } in iter {
-
-                // We ask for the complete range mapped to the area we map.
-                let start = output.get(&range.start).map(|r| r.start).unwrap_or(range.start);
-                let end = output.get(&(range.end - 1)).map(|r| r.end).unwrap_or(range.end);
-                let range = start..end;
-
-                // We map each query id to one word until the last,
-                // we map it to the remainings words.
-                let add = range.len() - words.len();
-                for (j, x) in range.take(words.len()).enumerate() {
-                    let add = if j == words.len() - 1 { add } else { 0 }; // is last?
-                    let range = x..x + 1 + add;
-                    output.insert(id + j, range);
-                }
-            }
-        }
-
-        output
-    }
-}
-
-fn longest_common_prefix<T: Eq + std::fmt::Debug>(a: &[T], b: &[T]) -> usize {
-    let mut best = None;
-    for i in (0..a.len()).rev() {
-        let count = a[i..].iter().zip(b).take_while(|(a, b)| a == b).count();
-        best = match best {
-            Some(old) if count > old => Some(count),
-            Some(_) => break,
-            None => Some(count),
-        };
-    }
-    best.unwrap_or(0)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn original_unmodified() {
-        let query = ["new", "york", "city", "subway"];
-        //             0       1       2        3
-        let mut builder = QueryWordsMapper::new(&query);
-
-        // new york = new york city
-        builder.declare(0..2, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // new = new york city
-        builder.declare(0..1, 7, &["new", "york", "city"]);
-        //                    ^      7       8       9
-
-        let mapping = builder.mapping();
-
-        assert_eq!(mapping[&0], 0..1); // new
-        assert_eq!(mapping[&1], 1..2); // york
-        assert_eq!(mapping[&2], 2..3); // city
-        assert_eq!(mapping[&3], 3..4); // subway
-
-        assert_eq!(mapping[&4], 0..1); // new
-        assert_eq!(mapping[&5], 1..2); // york
-        assert_eq!(mapping[&6], 2..3); // city
-
-        assert_eq!(mapping[&7], 0..1); // new
-        assert_eq!(mapping[&8], 1..2); // york
-        assert_eq!(mapping[&9], 2..3); // city
-    }
-
-    #[test]
-    fn original_unmodified2() {
-        let query = ["new", "york", "city", "subway"];
-        //             0       1       2        3
-        let mut builder = QueryWordsMapper::new(&query);
-
-        // city subway = new york city underground train
-        builder.declare(2..4, 4, &["new", "york", "city", "underground", "train"]);
-        //                    ^      4      5       6           7           8
-
-        let mapping = builder.mapping();
-
-        assert_eq!(mapping[&0], 0..1); // new
-        assert_eq!(mapping[&1], 1..2); // york
-        assert_eq!(mapping[&2], 2..3); // city
-        assert_eq!(mapping[&3], 3..5); // subway
-
-        assert_eq!(mapping[&4], 0..1); // new
-        assert_eq!(mapping[&5], 1..2); // york
-        assert_eq!(mapping[&6], 2..3); // city
-        assert_eq!(mapping[&7], 3..4); // underground
-        assert_eq!(mapping[&8], 4..5); // train
-    }
-
-    #[test]
-    fn original_unmodified3() {
-        let query = ["a", "b", "x", "x", "a", "b", "c", "d", "e", "f", "g"];
-        //            0    1    2    3    4    5    6    7    8    9    10
-        let mut builder = QueryWordsMapper::new(&query);
-
-        // c d = a b x c d k j e f
-        builder.declare(6..8, 11, &["a", "b", "x", "c", "d", "k", "j", "e", "f"]);
-        //                    ^^    11   12   13   14   15   16   17   18   19
-
-        let mapping = builder.mapping();
-
-        assert_eq!(mapping[&0],  0..1); // a
-        assert_eq!(mapping[&1],  1..2); // b
-        assert_eq!(mapping[&2],  2..3); // x
-        assert_eq!(mapping[&3],  3..4); // x
-        assert_eq!(mapping[&4],  4..5); // a
-        assert_eq!(mapping[&5],  5..6); // b
-        assert_eq!(mapping[&6],  6..7); // c
-        assert_eq!(mapping[&7],  7..11); // d
-        assert_eq!(mapping[&8],  11..12); // e
-        assert_eq!(mapping[&9],  12..13); // f
-        assert_eq!(mapping[&10], 13..14); // g
-
-        assert_eq!(mapping[&11], 4..5); // a
-        assert_eq!(mapping[&12], 5..6); // b
-        assert_eq!(mapping[&13], 6..7); // x
-        assert_eq!(mapping[&14], 7..8); // c
-        assert_eq!(mapping[&15], 8..9); // d
-        assert_eq!(mapping[&16], 9..10); // k
-        assert_eq!(mapping[&17], 10..11); // j
-        assert_eq!(mapping[&18], 11..12); // e
-        assert_eq!(mapping[&19], 12..13); // f
-    }
-
-    #[test]
-    fn simple_growing() {
-        let query = ["new", "york", "subway"];
-        //             0       1        2
-        let mut builder = QueryWordsMapper::new(&query);
-
-        // new york = new york city
-        builder.declare(0..2, 3, &["new", "york", "city"]);
-        //                    ^      3       4       5
-
-        let mapping = builder.mapping();
-
-        assert_eq!(mapping[&0], 0..1); // new
-        assert_eq!(mapping[&1], 1..3); // york
-        assert_eq!(mapping[&2], 3..4); // subway
-        assert_eq!(mapping[&3], 0..1); // new
-        assert_eq!(mapping[&4], 1..2); // york
-        assert_eq!(mapping[&5], 2..3); // city
-    }
-
-    #[test]
-    fn same_place_growings() {
-        let query = ["NY", "subway"];
-        //             0       1
-        let mut builder = QueryWordsMapper::new(&query);
-
-        // NY = new york
-        builder.declare(0..1, 2, &["new", "york"]);
-        //                    ^      2       3
-
-        // NY = new york city
-        builder.declare(0..1, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // NY = NYC
-        builder.declare(0..1, 7, &["NYC"]);
-        //                    ^      7
-
-        // NY = new york city
-        builder.declare(0..1, 8, &["new", "york", "city"]);
-        //                    ^      8       9      10
-
-        // subway = underground train
-        builder.declare(1..2, 11, &["underground", "train"]);
-        //                    ^          11          12
-
-        let mapping = builder.mapping();
-
-        assert_eq!(mapping[&0], 0..3); // NY
-        assert_eq!(mapping[&1], 3..5); // subway
-        assert_eq!(mapping[&2], 0..1); // new
-        assert_eq!(mapping[&3], 1..3); // york
-        assert_eq!(mapping[&4], 0..1); // new
-        assert_eq!(mapping[&5], 1..2); // york
-        assert_eq!(mapping[&6], 2..3); // city
-        assert_eq!(mapping[&7], 0..3); // NYC
-        assert_eq!(mapping[&8], 0..1); // new
-        assert_eq!(mapping[&9], 1..2); // york
-        assert_eq!(mapping[&10], 2..3); // city
-        assert_eq!(mapping[&11], 3..4); // underground
-        assert_eq!(mapping[&12], 4..5); // train
-    }
-
-    #[test]
-    fn bigger_growing() {
-        let query = ["NYC", "subway"];
-        //             0        1
-        let mut builder = QueryWordsMapper::new(&query);
-
-        // NYC = new york city
-        builder.declare(0..1, 2, &["new", "york", "city"]);
-        //                    ^      2       3       4
-
-        let mapping = builder.mapping();
-
-        assert_eq!(mapping[&0], 0..3); // NYC
-        assert_eq!(mapping[&1], 3..4); // subway
-        assert_eq!(mapping[&2], 0..1); // new
-        assert_eq!(mapping[&3], 1..2); // york
-        assert_eq!(mapping[&4], 2..3); // city
-    }
-
-    #[test]
-    fn middle_query_growing() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryWordsMapper::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        let mapping = builder.mapping();
-
-        assert_eq!(mapping[&0], 0..1); // great
-        assert_eq!(mapping[&1], 1..2); // awesome
-        assert_eq!(mapping[&2], 2..5); // NYC
-        assert_eq!(mapping[&3], 5..6); // subway
-        assert_eq!(mapping[&4], 2..3); // new
-        assert_eq!(mapping[&5], 3..4); // york
-        assert_eq!(mapping[&6], 4..5); // city
-    }
-
-    #[test]
-    fn end_query_growing() {
-        let query = ["NYC", "subway"];
-        //             0        1
-        let mut builder = QueryWordsMapper::new(&query);
-
-        // NYC = new york city
-        builder.declare(1..2, 2, &["underground", "train"]);
-        //                    ^         2            3
-
-        let mapping = builder.mapping();
-
-        assert_eq!(mapping[&0], 0..1); // NYC
-        assert_eq!(mapping[&1], 1..3); // subway
-        assert_eq!(mapping[&2], 1..2); // underground
-        assert_eq!(mapping[&3], 2..3); // train
-    }
-
-    #[test]
-    fn multiple_growings() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryWordsMapper::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // subway = underground train
-        builder.declare(3..4, 7, &["underground", "train"]);
-        //                    ^          7           8
-
-        let mapping = builder.mapping();
-
-        assert_eq!(mapping[&0], 0..1); // great
-        assert_eq!(mapping[&1], 1..2); // awesome
-        assert_eq!(mapping[&2], 2..5); // NYC
-        assert_eq!(mapping[&3], 5..7); // subway
-        assert_eq!(mapping[&4], 2..3); // new
-        assert_eq!(mapping[&5], 3..4); // york
-        assert_eq!(mapping[&6], 4..5); // city
-        assert_eq!(mapping[&7], 5..6); // underground
-        assert_eq!(mapping[&8], 6..7); // train
-    }
-
-    #[test]
-    fn multiple_probable_growings() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryWordsMapper::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // subway = underground train
-        builder.declare(3..4, 7, &["underground", "train"]);
-        //                    ^          7           8
-
-        // great awesome = good
-        builder.declare(0..2, 9, &["good"]);
-        //                    ^       9
-
-        // awesome NYC = NY
-        builder.declare(1..3, 10, &["NY"]);
-        //                    ^^     10
-
-        // NYC subway = metro
-        builder.declare(2..4, 11, &["metro"]);
-        //                    ^^      11
-
-        let mapping = builder.mapping();
-
-        assert_eq!(mapping[&0], 0..1); // great
-        assert_eq!(mapping[&1], 1..2); // awesome
-        assert_eq!(mapping[&2], 2..5); // NYC
-        assert_eq!(mapping[&3], 5..7); // subway
-        assert_eq!(mapping[&4], 2..3); // new
-        assert_eq!(mapping[&5], 3..4); // york
-        assert_eq!(mapping[&6], 4..5); // city
-        assert_eq!(mapping[&7], 5..6); // underground
-        assert_eq!(mapping[&8], 6..7); // train
-        assert_eq!(mapping[&9], 0..2); // good
-        assert_eq!(mapping[&10], 1..5); // NY
-        assert_eq!(mapping[&11], 2..7); // metro
-    }
-}

meilisearch-core/src/ranked_map.rs

@@ -1,41 +0,0 @@
-use std::io::{Read, Write};
-
-use hashbrown::HashMap;
-use meilisearch_schema::FieldId;
-use serde::{Deserialize, Serialize};
-
-use crate::{DocumentId, Number};
-
-#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(transparent)]
-pub struct RankedMap(HashMap<(DocumentId, FieldId), Number>);
-
-impl RankedMap {
-    pub fn len(&self) -> usize {
-        self.0.len()
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.0.is_empty()
-    }
-
-    pub fn insert(&mut self, document: DocumentId, field: FieldId, number: Number) {
-        self.0.insert((document, field), number);
-    }
-
-    pub fn remove(&mut self, document: DocumentId, field: FieldId) {
-        self.0.remove(&(document, field));
-    }
-
-    pub fn get(&self, document: DocumentId, field: FieldId) -> Option<Number> {
-        self.0.get(&(document, field)).cloned()
-    }
-
-    pub fn read_from_bin<R: Read>(reader: R) -> bincode::Result<RankedMap> {
-        bincode::deserialize_from(reader).map(RankedMap)
-    }
-
-    pub fn write_to_bin<W: Write>(&self, writer: W) -> bincode::Result<()> {
-        bincode::serialize_into(writer, &self.0)
-    }
-}

meilisearch-core/src/raw_document.rs

@@ -1,51 +0,0 @@
-use compact_arena::SmallArena;
-use sdset::SetBuf;
-use crate::DocIndex;
-use crate::bucket_sort::{SimpleMatch, BareMatch, PostingsListView};
-use crate::reordered_attrs::ReorderedAttrs;
-
-pub struct RawDocument<'a, 'tag> {
-    pub id: crate::DocumentId,
-    pub bare_matches: &'a mut [BareMatch<'tag>],
-    pub processed_matches: Vec<SimpleMatch>,
-    /// The list of minimum `distance` found
-    pub processed_distances: Vec<Option<u8>>,
-    /// Does this document contains a field
-    /// with one word that is exactly matching
-    pub contains_one_word_field: bool,
-}
-
-impl<'a, 'tag> RawDocument<'a, 'tag> {
-    pub fn new<'txn>(
-        bare_matches: &'a mut [BareMatch<'tag>],
-        postings_lists: &mut SmallArena<'tag, PostingsListView<'txn>>,
-        searchable_attrs: Option<&ReorderedAttrs>,
-    ) -> RawDocument<'a, 'tag>
-    {
-        if let Some(reordered_attrs) = searchable_attrs {
-            for bm in bare_matches.iter() {
-                let postings_list = &postings_lists[bm.postings_list];
-
-                let mut rewritten = Vec::new();
-                for di in postings_list.iter() {
-                    if let Some(attribute) = reordered_attrs.get(di.attribute) {
-                        rewritten.push(DocIndex { attribute, ..*di });
-                    }
-                }
-
-                let new_postings = SetBuf::from_dirty(rewritten);
-                postings_lists[bm.postings_list].rewrite_with(new_postings);
-            }
-        }
-
-        bare_matches.sort_unstable_by_key(|m| m.query_index);
-
-        RawDocument {
-            id: bare_matches[0].document_id,
-            bare_matches,
-            processed_matches: Vec::new(),
-            processed_distances: Vec::new(),
-            contains_one_word_field: false,
-        }
-    }
-}

meilisearch-core/src/raw_indexer.rs

@@ -1,272 +0,0 @@
-use std::collections::{BTreeMap, HashMap};
-use std::convert::TryFrom;
-
-use crate::{DocIndex, DocumentId};
-use deunicode::deunicode_with_tofu;
-use meilisearch_schema::IndexedPos;
-use meilisearch_tokenizer::{is_cjk, SeqTokenizer, Token, Tokenizer};
-use sdset::SetBuf;
-
-const WORD_LENGTH_LIMIT: usize = 80;
-
-type Word = Vec<u8>; // TODO make it be a SmallVec
-
-pub struct RawIndexer {
-    word_limit: usize, // the maximum number of indexed words
-    stop_words: fst::Set,
-    words_doc_indexes: BTreeMap<Word, Vec<DocIndex>>,
-    docs_words: HashMap<DocumentId, Vec<Word>>,
-}
-
-pub struct Indexed {
-    pub words_doc_indexes: BTreeMap<Word, SetBuf<DocIndex>>,
-    pub docs_words: HashMap<DocumentId, fst::Set>,
-}
-
-impl RawIndexer {
-    pub fn new(stop_words: fst::Set) -> RawIndexer {
-        RawIndexer::with_word_limit(stop_words, 1000)
-    }
-
-    pub fn with_word_limit(stop_words: fst::Set, limit: usize) -> RawIndexer {
-        RawIndexer {
-            word_limit: limit,
-            stop_words,
-            words_doc_indexes: BTreeMap::new(),
-            docs_words: HashMap::new(),
-        }
-    }
-
-    pub fn index_text(&mut self, id: DocumentId, indexed_pos: IndexedPos, text: &str) -> usize {
-        let mut number_of_words = 0;
-
-        for token in Tokenizer::new(text) {
-            let must_continue = index_token(
-                token,
-                id,
-                indexed_pos,
-                self.word_limit,
-                &self.stop_words,
-                &mut self.words_doc_indexes,
-                &mut self.docs_words,
-            );
-
-            number_of_words += 1;
-
-            if !must_continue {
-                break;
-            }
-        }
-
-        number_of_words
-    }
-
-    pub fn index_text_seq<'a, I>(&mut self, id: DocumentId, indexed_pos: IndexedPos, iter: I)
-    where
-        I: IntoIterator<Item = &'a str>,
-    {
-        let iter = iter.into_iter();
-        for token in SeqTokenizer::new(iter) {
-            let must_continue = index_token(
-                token,
-                id,
-                indexed_pos,
-                self.word_limit,
-                &self.stop_words,
-                &mut self.words_doc_indexes,
-                &mut self.docs_words,
-            );
-
-            if !must_continue {
-                break;
-            }
-        }
-    }
-
-    pub fn build(self) -> Indexed {
-        let words_doc_indexes = self
-            .words_doc_indexes
-            .into_iter()
-            .map(|(word, indexes)| (word, SetBuf::from_dirty(indexes)))
-            .collect();
-
-        let docs_words = self
-            .docs_words
-            .into_iter()
-            .map(|(id, mut words)| {
-                words.sort_unstable();
-                words.dedup();
-                (id, fst::Set::from_iter(words).unwrap())
-            })
-            .collect();
-
-        Indexed {
-            words_doc_indexes,
-            docs_words,
-        }
-    }
-}
-
-fn index_token(
-    token: Token,
-    id: DocumentId,
-    indexed_pos: IndexedPos,
-    word_limit: usize,
-    stop_words: &fst::Set,
-    words_doc_indexes: &mut BTreeMap<Word, Vec<DocIndex>>,
-    docs_words: &mut HashMap<DocumentId, Vec<Word>>,
-) -> bool {
-    if token.word_index >= word_limit {
-        return false;
-    }
-
-    let lower = token.word.to_lowercase();
-    let token = Token {
-        word: &lower,
-        ..token
-    };
-
-    if !stop_words.contains(&token.word) {
-        match token_to_docindex(id, indexed_pos, token) {
-            Some(docindex) => {
-                let word = Vec::from(token.word);
-
-                if word.len() <= WORD_LENGTH_LIMIT {
-                    words_doc_indexes
-                        .entry(word.clone())
-                        .or_insert_with(Vec::new)
-                        .push(docindex);
-                    docs_words.entry(id).or_insert_with(Vec::new).push(word);
-
-                    if !lower.contains(is_cjk) {
-                        let unidecoded = deunicode_with_tofu(&lower, "");
-                        if unidecoded != lower && !unidecoded.is_empty() {
-                            let word = Vec::from(unidecoded);
-                            if word.len() <= WORD_LENGTH_LIMIT {
-                                words_doc_indexes
-                                    .entry(word.clone())
-                                    .or_insert_with(Vec::new)
-                                    .push(docindex);
-                                docs_words.entry(id).or_insert_with(Vec::new).push(word);
-                            }
-                        }
-                    }
-                }
-            }
-            None => return false,
-        }
-    }
-
-    true
-}
-
-fn token_to_docindex(id: DocumentId, indexed_pos: IndexedPos, token: Token) -> Option<DocIndex> {
-    let word_index = u16::try_from(token.word_index).ok()?;
-    let char_index = u16::try_from(token.char_index).ok()?;
-    let char_length = u16::try_from(token.word.chars().count()).ok()?;
-
-    let docindex = DocIndex {
-        document_id: id,
-        attribute: indexed_pos.0,
-        word_index,
-        char_index,
-        char_length,
-    };
-
-    Some(docindex)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use meilisearch_schema::IndexedPos;
-
-    #[test]
-    fn strange_apostrophe() {
-        let mut indexer = RawIndexer::new(fst::Set::default());
-
-        let docid = DocumentId(0);
-        let indexed_pos = IndexedPos(0);
-        let text = "Zut, l’aspirateur, j’ai oublié de l’éteindre !";
-        indexer.index_text(docid, indexed_pos, text);
-
-        let Indexed {
-            words_doc_indexes, ..
-        } = indexer.build();
-
-        assert!(words_doc_indexes.get(&b"l"[..]).is_some());
-        assert!(words_doc_indexes.get(&b"aspirateur"[..]).is_some());
-        assert!(words_doc_indexes.get(&b"ai"[..]).is_some());
-        assert!(words_doc_indexes.get(&b"eteindre"[..]).is_some());
-        assert!(words_doc_indexes
-            .get(&"éteindre".to_owned().into_bytes())
-            .is_some());
-    }
-
-    #[test]
-    fn strange_apostrophe_in_sequence() {
-        let mut indexer = RawIndexer::new(fst::Set::default());
-
-        let docid = DocumentId(0);
-        let indexed_pos = IndexedPos(0);
-        let text = vec!["Zut, l’aspirateur, j’ai oublié de l’éteindre !"];
-        indexer.index_text_seq(docid, indexed_pos, text);
-
-        let Indexed {
-            words_doc_indexes, ..
-        } = indexer.build();
-
-        assert!(words_doc_indexes.get(&b"l"[..]).is_some());
-        assert!(words_doc_indexes.get(&b"aspirateur"[..]).is_some());
-        assert!(words_doc_indexes.get(&b"ai"[..]).is_some());
-        assert!(words_doc_indexes.get(&b"eteindre"[..]).is_some());
-        assert!(words_doc_indexes
-            .get(&"éteindre".to_owned().into_bytes())
-            .is_some());
-    }
-
-    #[test]
-    fn basic_stop_words() {
-        let stop_words = sdset::SetBuf::from_dirty(vec!["l", "j", "ai", "de"]);
-        let stop_words = fst::Set::from_iter(stop_words).unwrap();
-
-        let mut indexer = RawIndexer::new(stop_words);
-
-        let docid = DocumentId(0);
-        let indexed_pos = IndexedPos(0);
-        let text = "Zut, l’aspirateur, j’ai oublié de l’éteindre !";
-        indexer.index_text(docid, indexed_pos, text);
-
-        let Indexed {
-            words_doc_indexes, ..
-        } = indexer.build();
-
-        assert!(words_doc_indexes.get(&b"l"[..]).is_none());
-        assert!(words_doc_indexes.get(&b"aspirateur"[..]).is_some());
-        assert!(words_doc_indexes.get(&b"j"[..]).is_none());
-        assert!(words_doc_indexes.get(&b"ai"[..]).is_none());
-        assert!(words_doc_indexes.get(&b"de"[..]).is_none());
-        assert!(words_doc_indexes.get(&b"eteindre"[..]).is_some());
-        assert!(words_doc_indexes
-            .get(&"éteindre".to_owned().into_bytes())
-            .is_some());
-    }
-
-    #[test]
-    fn no_empty_unidecode() {
-        let mut indexer = RawIndexer::new(fst::Set::default());
-
-        let docid = DocumentId(0);
-        let indexed_pos = IndexedPos(0);
-        let text = "🇯🇵";
-        indexer.index_text(docid, indexed_pos, text);
-
-        let Indexed {
-            words_doc_indexes, ..
-        } = indexer.build();
-
-        assert!(words_doc_indexes
-            .get(&"🇯🇵".to_owned().into_bytes())
-            .is_some());
-    }
-}

meilisearch-core/src/reordered_attrs.rs

@@ -1,31 +0,0 @@
-use std::cmp;
-
-#[derive(Default, Clone)]
-pub struct ReorderedAttrs {
-    reorders: Vec<Option<u16>>,
-    reverse: Vec<u16>,
-}
-
-impl ReorderedAttrs {
-    pub fn new() -> ReorderedAttrs {
-        ReorderedAttrs { reorders: Vec::new(), reverse: Vec::new() }
-    }
-
-    pub fn insert_attribute(&mut self, attribute: u16) {
-        let new_len = cmp::max(attribute as usize + 1, self.reorders.len());
-        self.reorders.resize(new_len, None);
-        self.reorders[attribute as usize] = Some(self.reverse.len() as u16);
-        self.reverse.push(attribute);
-    }
-
-    pub fn get(&self, attribute: u16) -> Option<u16> {
-        match self.reorders.get(attribute as usize)? {
-            Some(attribute) => Some(*attribute),
-            None => None,
-        }
-    }
-
-    pub fn reverse(&self, attribute: u16) -> Option<u16> {
-        self.reverse.get(attribute as usize).copied()
-    }
-}

meilisearch-core/src/serde/convert_to_number.rs

@@ -1,198 +0,0 @@
-use std::str::FromStr;
-
-use ordered_float::OrderedFloat;
-use serde::ser;
-use serde::Serialize;
-
-use super::SerializerError;
-use crate::Number;
-
-pub struct ConvertToNumber;
-
-impl ser::Serializer for ConvertToNumber {
-    type Ok = Number;
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    fn serialize_bool(self, value: bool) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_char(self, _value: char) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "char" })
-    }
-
-    fn serialize_i8(self, value: i8) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(i64::from(value)))
-    }
-
-    fn serialize_i16(self, value: i16) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(i64::from(value)))
-    }
-
-    fn serialize_i32(self, value: i32) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(i64::from(value)))
-    }
-
-    fn serialize_i64(self, value: i64) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(value))
-    }
-
-    fn serialize_u8(self, value: u8) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_u16(self, value: u16) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_u32(self, value: u32) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_u64(self, value: u64) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(value))
-    }
-
-    fn serialize_f32(self, value: f32) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Float(OrderedFloat(f64::from(value))))
-    }
-
-    fn serialize_f64(self, value: f64) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Float(OrderedFloat(value)))
-    }
-
-    fn serialize_str(self, value: &str) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::from_str(value)?)
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnrankableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnrankableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "map" })
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "struct",
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "struct variant",
-        })
-    }
-}

meilisearch-core/src/serde/convert_to_string.rs

@@ -1,258 +0,0 @@
-use serde::ser;
-use serde::Serialize;
-
-use super::SerializerError;
-
-pub struct ConvertToString;
-
-impl ser::Serializer for ConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = MapConvertToString;
-    type SerializeStruct = StructConvertToString;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    fn serialize_bool(self, _value: bool) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "boolean",
-        })
-    }
-
-    fn serialize_char(self, value: char) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i8(self, value: i8) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i16(self, value: i16) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i32(self, value: i32) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i64(self, value: i64) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u8(self, value: u8) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u16(self, value: u16) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u32(self, value: u32) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u64(self, value: u64) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_f32(self, value: f32) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_f64(self, value: f64) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_str(self, value: &str) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        Ok(MapConvertToString {
-            text: String::new(),
-        })
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Ok(StructConvertToString {
-            text: String::new(),
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct MapConvertToString {
-    text: String,
-}
-
-impl ser::SerializeMap for MapConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = key.serialize(ConvertToString)?;
-        self.text.push_str(&text);
-        self.text.push_str(" ");
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.text.push_str(&text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.text)
-    }
-}
-
-pub struct StructConvertToString {
-    text: String,
-}
-
-impl ser::SerializeStruct for StructConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let value = value.serialize(ConvertToString)?;
-        self.text.push_str(key);
-        self.text.push_str(" ");
-        self.text.push_str(&value);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.text)
-    }
-}

meilisearch-core/src/serde/deserializer.rs

@@ -1,161 +0,0 @@
-use std::collections::HashSet;
-use std::io::Cursor;
-use std::{error::Error, fmt};
-
-use meilisearch_schema::{Schema, FieldId};
-use serde::{de, forward_to_deserialize_any};
-use serde_json::de::IoRead as SerdeJsonIoRead;
-use serde_json::Deserializer as SerdeJsonDeserializer;
-use serde_json::Error as SerdeJsonError;
-
-use crate::database::MainT;
-use crate::store::DocumentsFields;
-use crate::DocumentId;
-
-#[derive(Debug)]
-pub enum DeserializerError {
-    SerdeJson(SerdeJsonError),
-    Zlmdb(heed::Error),
-    Custom(String),
-}
-
-impl de::Error for DeserializerError {
-    fn custom<T: fmt::Display>(msg: T) -> Self {
-        DeserializerError::Custom(msg.to_string())
-    }
-}
-
-impl fmt::Display for DeserializerError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            DeserializerError::SerdeJson(e) => write!(f, "serde json related error: {}", e),
-            DeserializerError::Zlmdb(e) => write!(f, "heed related error: {}", e),
-            DeserializerError::Custom(s) => f.write_str(s),
-        }
-    }
-}
-
-impl Error for DeserializerError {}
-
-impl From<SerdeJsonError> for DeserializerError {
-    fn from(error: SerdeJsonError) -> DeserializerError {
-        DeserializerError::SerdeJson(error)
-    }
-}
-
-impl From<heed::Error> for DeserializerError {
-    fn from(error: heed::Error) -> DeserializerError {
-        DeserializerError::Zlmdb(error)
-    }
-}
-
-pub struct Deserializer<'a> {
-    pub document_id: DocumentId,
-    pub reader: &'a heed::RoTxn<MainT>,
-    pub documents_fields: DocumentsFields,
-    pub schema: &'a Schema,
-    pub fields: Option<&'a HashSet<FieldId>>,
-}
-
-impl<'de, 'a, 'b> de::Deserializer<'de> for &'b mut Deserializer<'a> {
-    type Error = DeserializerError;
-
-    fn deserialize_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>
-    where
-        V: de::Visitor<'de>,
-    {
-        self.deserialize_option(visitor)
-    }
-
-    fn deserialize_option<V>(self, visitor: V) -> Result<V::Value, Self::Error>
-    where
-        V: de::Visitor<'de>,
-    {
-        self.deserialize_map(visitor)
-    }
-
-    fn deserialize_map<V>(self, visitor: V) -> Result<V::Value, Self::Error>
-    where
-        V: de::Visitor<'de>,
-    {
-        let mut error = None;
-
-        let iter = self
-            .documents_fields
-            .document_fields(self.reader, self.document_id)?
-            .filter_map(|result| {
-                let (attr, value) = match result {
-                    Ok(value) => value,
-                    Err(e) => {
-                        error = Some(e);
-                        return None;
-                    }
-                };
-
-                let is_displayed = self.schema.is_displayed(attr);
-                if is_displayed && self.fields.map_or(true, |f| f.contains(&attr)) {
-                    if let Some(attribute_name) = self.schema.name(attr) {
-                        let cursor = Cursor::new(value.to_owned());
-                        let ioread = SerdeJsonIoRead::new(cursor);
-                        let value = Value(SerdeJsonDeserializer::new(ioread));
-
-                        Some((attribute_name, value))
-                    } else {
-                        None
-                    }
-                } else {
-                    None
-                }
-            });
-
-        let mut iter = iter.peekable();
-
-        let result = match iter.peek() {
-            Some(_) => {
-                let map_deserializer = de::value::MapDeserializer::new(iter);
-                visitor
-                    .visit_some(map_deserializer)
-                    .map_err(DeserializerError::from)
-            }
-            None => visitor.visit_none(),
-        };
-
-        match error.take() {
-            Some(error) => Err(error.into()),
-            None => result,
-        }
-    }
-
-    forward_to_deserialize_any! {
-        bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string
-        bytes byte_buf unit unit_struct newtype_struct seq tuple
-        tuple_struct struct enum identifier ignored_any
-    }
-}
-
-struct Value(SerdeJsonDeserializer<SerdeJsonIoRead<Cursor<Vec<u8>>>>);
-
-impl<'de> de::IntoDeserializer<'de, SerdeJsonError> for Value {
-    type Deserializer = Self;
-
-    fn into_deserializer(self) -> Self::Deserializer {
-        self
-    }
-}
-
-impl<'de> de::Deserializer<'de> for Value {
-    type Error = SerdeJsonError;
-
-    fn deserialize_any<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
-    where
-        V: de::Visitor<'de>,
-    {
-        self.0.deserialize_any(visitor)
-    }
-
-    forward_to_deserialize_any! {
-        bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string
-        bytes byte_buf option unit unit_struct newtype_struct seq tuple
-        tuple_struct map struct enum identifier ignored_any
-    }
-}

meilisearch-core/src/serde/extract_document_id.rs

@@ -1,310 +0,0 @@
-use std::hash::{Hash, Hasher};
-
-use crate::DocumentId;
-use serde::{ser, Serialize};
-use serde_json::{Value, Number};
-use siphasher::sip::SipHasher;
-
-use super::{ConvertToString, SerializerError};
-
-pub fn extract_document_id<D>(
-    primary_key: &str,
-    document: &D,
-) -> Result<Option<DocumentId>, SerializerError>
-where
-    D: serde::Serialize,
-{
-    let serializer = ExtractDocumentId { primary_key };
-    document.serialize(serializer)
-}
-
-fn validate_number(value: &Number) -> Option<String> {
-    if value.is_f64() {
-        return None
-    }
-    Some(value.to_string())
-}
-
-fn validate_string(value: &str) -> Option<String> {
-    if value.chars().all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') {
-        Some(value.to_string())
-    } else {
-        None
-    }
-}
-
-pub fn value_to_string(value: &Value) -> Option<String> {
-    match value {
-        Value::Null => None,
-        Value::Bool(_) => None,
-        Value::Number(value) => validate_number(value),
-        Value::String(value) => validate_string(value),
-        Value::Array(_) => None,
-        Value::Object(_) => None,
-    }
-}
-
-pub fn compute_document_id<H: Hash>(t: H) -> DocumentId {
-    let mut s = SipHasher::new();
-    t.hash(&mut s);
-    let hash = s.finish();
-    DocumentId(hash)
-}
-
-struct ExtractDocumentId<'a> {
-    primary_key: &'a str,
-}
-
-impl<'a> ser::Serializer for ExtractDocumentId<'a> {
-    type Ok = Option<DocumentId>;
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = ExtractDocumentIdMapSerializer<'a>;
-    type SerializeStruct = ExtractDocumentIdStructSerializer<'a>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    forward_to_unserializable_type! {
-        bool => serialize_bool,
-        char => serialize_char,
-
-        i8  => serialize_i8,
-        i16 => serialize_i16,
-        i32 => serialize_i32,
-        i64 => serialize_i64,
-
-        u8  => serialize_u8,
-        u16 => serialize_u16,
-        u32 => serialize_u32,
-        u64 => serialize_u64,
-
-        f32 => serialize_f32,
-        f64 => serialize_f64,
-    }
-
-    fn serialize_str(self, _value: &str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "str" })
-    }
-
-    fn serialize_bytes(self, _value: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        let serializer = ExtractDocumentIdMapSerializer {
-            primary_key: self.primary_key,
-            document_id: None,
-            current_key_name: None,
-        };
-
-        Ok(serializer)
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        let serializer = ExtractDocumentIdStructSerializer {
-            primary_key: self.primary_key,
-            document_id: None,
-        };
-
-        Ok(serializer)
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct ExtractDocumentIdMapSerializer<'a> {
-    primary_key: &'a str,
-    document_id: Option<DocumentId>,
-    current_key_name: Option<String>,
-}
-
-impl<'a> ser::SerializeMap for ExtractDocumentIdMapSerializer<'a> {
-    type Ok = Option<DocumentId>;
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-        self.current_key_name = Some(key);
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        let key = self.current_key_name.take().unwrap();
-        self.serialize_entry(&key, value)
-    }
-
-    fn serialize_entry<K: ?Sized, V: ?Sized>(
-        &mut self,
-        key: &K,
-        value: &V,
-    ) -> Result<(), Self::Error>
-    where
-        K: Serialize,
-        V: Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-
-        if self.primary_key == key {
-            let value = serde_json::to_string(value).and_then(|s| serde_json::from_str(&s))?;
-            match value_to_string(&value).map(|s| compute_document_id(&s)) {
-                Some(document_id) => self.document_id = Some(document_id),
-                None => return Err(SerializerError::InvalidDocumentIdType),
-            }
-        }
-
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.document_id)
-    }
-}
-
-pub struct ExtractDocumentIdStructSerializer<'a> {
-    primary_key: &'a str,
-    document_id: Option<DocumentId>,
-}
-
-impl<'a> ser::SerializeStruct for ExtractDocumentIdStructSerializer<'a> {
-    type Ok = Option<DocumentId>;
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        if self.primary_key == key {
-            let value = serde_json::to_string(value).and_then(|s| serde_json::from_str(&s))?;
-            match value_to_string(&value).map(compute_document_id) {
-                Some(document_id) => self.document_id = Some(document_id),
-                None => return Err(SerializerError::InvalidDocumentIdType),
-            }
-        }
-
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.document_id)
-    }
-}

meilisearch-core/src/serde/indexer.rs

@@ -1,362 +0,0 @@
-use meilisearch_schema::IndexedPos;
-use serde::ser;
-use serde::Serialize;
-
-use super::{ConvertToString, SerializerError};
-use crate::raw_indexer::RawIndexer;
-use crate::DocumentId;
-
-pub struct Indexer<'a> {
-    pub pos: IndexedPos,
-    pub indexer: &'a mut RawIndexer,
-    pub document_id: DocumentId,
-}
-
-impl<'a> ser::Serializer for Indexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-    type SerializeSeq = SeqIndexer<'a>;
-    type SerializeTuple = TupleIndexer<'a>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = MapIndexer<'a>;
-    type SerializeStruct = StructIndexer<'a>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    fn serialize_bool(self, _value: bool) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_char(self, value: char) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i8(self, value: i8) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i16(self, value: i16) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i32(self, value: i32) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i64(self, value: i64) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u8(self, value: u8) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u16(self, value: u16) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u32(self, value: u32) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u64(self, value: u64) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_f32(self, value: f32) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_f64(self, value: f64) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_str(self, text: &str) -> Result<Self::Ok, Self::Error> {
-        let number_of_words = self
-            .indexer
-            .index_text(self.document_id, self.pos, text);
-        Ok(Some(number_of_words))
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_some<T: ?Sized>(self, value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        let number_of_words = self
-            .indexer
-            .index_text(self.document_id, self.pos, &text);
-        Ok(Some(number_of_words))
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        Err(SerializerError::UnindexableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        let indexer = SeqIndexer {
-            pos: self.pos,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        let indexer = TupleIndexer {
-            pos: self.pos,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        let indexer = MapIndexer {
-            pos: self.pos,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        let indexer = StructIndexer {
-            pos: self.pos,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct SeqIndexer<'a> {
-    pos: IndexedPos,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeSeq for SeqIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_element<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.pos, texts);
-        Ok(None)
-    }
-}
-
-pub struct MapIndexer<'a> {
-    pos: IndexedPos,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeMap for MapIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = key.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.pos, texts);
-        Ok(None)
-    }
-}
-
-pub struct StructIndexer<'a> {
-    pos: IndexedPos,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeStruct for StructIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let key_text = key.to_owned();
-        let value_text = value.serialize(ConvertToString)?;
-        self.texts.push(key_text);
-        self.texts.push(value_text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.pos, texts);
-        Ok(None)
-    }
-}
-
-pub struct TupleIndexer<'a> {
-    pos: IndexedPos,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeTuple for TupleIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_element<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.pos, texts);
-        Ok(None)
-    }
-}

meilisearch-core/src/serde/mod.rs

@@ -1,112 +0,0 @@
-macro_rules! forward_to_unserializable_type {
-    ($($ty:ident => $se_method:ident,)*) => {
-        $(
-            fn $se_method(self, _v: $ty) -> Result<Self::Ok, Self::Error> {
-                Err(SerializerError::UnserializableType { type_name: "$ty" })
-            }
-        )*
-    }
-}
-
-mod convert_to_number;
-mod convert_to_string;
-mod deserializer;
-mod extract_document_id;
-mod indexer;
-mod serializer;
-
-pub use self::convert_to_number::ConvertToNumber;
-pub use self::convert_to_string::ConvertToString;
-pub use self::deserializer::{Deserializer, DeserializerError};
-pub use self::extract_document_id::{compute_document_id, extract_document_id, value_to_string};
-pub use self::indexer::Indexer;
-pub use self::serializer::{serialize_value, serialize_value_with_id, Serializer};
-
-use std::{error::Error, fmt};
-
-use serde::ser;
-use serde_json::Error as SerdeJsonError;
-use meilisearch_schema::Error as SchemaError;
-
-use crate::ParseNumberError;
-
-#[derive(Debug)]
-pub enum SerializerError {
-    DocumentIdNotFound,
-    InvalidDocumentIdType,
-    Zlmdb(heed::Error),
-    SerdeJson(SerdeJsonError),
-    ParseNumber(ParseNumberError),
-    Schema(SchemaError),
-    UnserializableType { type_name: &'static str },
-    UnindexableType { type_name: &'static str },
-    UnrankableType { type_name: &'static str },
-    Custom(String),
-}
-
-impl ser::Error for SerializerError {
-    fn custom<T: fmt::Display>(msg: T) -> Self {
-        SerializerError::Custom(msg.to_string())
-    }
-}
-
-impl fmt::Display for SerializerError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            SerializerError::DocumentIdNotFound => {
-                f.write_str("serialized document does not have an id according to the schema")
-            }
-            SerializerError::InvalidDocumentIdType => {
-                f.write_str("a document primary key can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_).")
-            }
-            SerializerError::Zlmdb(e) => write!(f, "heed related error: {}", e),
-            SerializerError::SerdeJson(e) => write!(f, "serde json error: {}", e),
-            SerializerError::ParseNumber(e) => {
-                write!(f, "error while trying to parse a number: {}", e)
-            }
-            SerializerError::Schema(e) => write!(f, "impossible to update schema: {}", e),
-            SerializerError::UnserializableType { type_name } => {
-                write!(f, "{} is not a serializable type", type_name)
-            }
-            SerializerError::UnindexableType { type_name } => {
-                write!(f, "{} is not an indexable type", type_name)
-            }
-            SerializerError::UnrankableType { type_name } => {
-                write!(f, "{} types can not be used for ranking", type_name)
-            }
-            SerializerError::Custom(s) => f.write_str(s),
-        }
-    }
-}
-
-impl Error for SerializerError {}
-
-impl From<String> for SerializerError {
-    fn from(value: String) -> SerializerError {
-        SerializerError::Custom(value)
-    }
-}
-
-impl From<SerdeJsonError> for SerializerError {
-    fn from(error: SerdeJsonError) -> SerializerError {
-        SerializerError::SerdeJson(error)
-    }
-}
-
-impl From<heed::Error> for SerializerError {
-    fn from(error: heed::Error) -> SerializerError {
-        SerializerError::Zlmdb(error)
-    }
-}
-
-impl From<ParseNumberError> for SerializerError {
-    fn from(error: ParseNumberError) -> SerializerError {
-        SerializerError::ParseNumber(error)
-    }
-}
-
-impl From<SchemaError> for SerializerError {
-    fn from(error: SchemaError) -> SerializerError {
-        SerializerError::Schema(error)
-   }
-}

meilisearch-core/src/serde/serializer.rs

@@ -1,361 +0,0 @@
-use meilisearch_schema::{Schema, FieldId};
-use serde::ser;
-
-use crate::database::MainT;
-use crate::raw_indexer::RawIndexer;
-use crate::store::{DocumentsFields, DocumentsFieldsCounts};
-use crate::{DocumentId, RankedMap};
-
-use super::{ConvertToNumber, ConvertToString, Indexer, SerializerError};
-
-pub struct Serializer<'a, 'b> {
-    pub txn: &'a mut heed::RwTxn<'b, MainT>,
-    pub schema: &'a mut Schema,
-    pub document_store: DocumentsFields,
-    pub document_fields_counts: DocumentsFieldsCounts,
-    pub indexer: &'a mut RawIndexer,
-    pub ranked_map: &'a mut RankedMap,
-    pub document_id: DocumentId,
-}
-
-impl<'a, 'b> ser::Serializer for Serializer<'a, 'b> {
-    type Ok = ();
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = MapSerializer<'a, 'b>;
-    type SerializeStruct = StructSerializer<'a, 'b>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    forward_to_unserializable_type! {
-        bool => serialize_bool,
-        char => serialize_char,
-
-        i8  => serialize_i8,
-        i16 => serialize_i16,
-        i32 => serialize_i32,
-        i64 => serialize_i64,
-
-        u8  => serialize_u8,
-        u16 => serialize_u16,
-        u32 => serialize_u32,
-        u64 => serialize_u64,
-
-        f32 => serialize_f32,
-        f64 => serialize_f64,
-    }
-
-    fn serialize_str(self, _v: &str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "str" })
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        Ok(MapSerializer {
-            txn: self.txn,
-            schema: self.schema,
-            document_id: self.document_id,
-            document_store: self.document_store,
-            document_fields_counts: self.document_fields_counts,
-            indexer: self.indexer,
-            ranked_map: self.ranked_map,
-            current_key_name: None,
-        })
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Ok(StructSerializer {
-            txn: self.txn,
-            schema: self.schema,
-            document_id: self.document_id,
-            document_store: self.document_store,
-            document_fields_counts: self.document_fields_counts,
-            indexer: self.indexer,
-            ranked_map: self.ranked_map,
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct MapSerializer<'a, 'b> {
-    txn: &'a mut heed::RwTxn<'b, MainT>,
-    schema: &'a mut Schema,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    document_fields_counts: DocumentsFieldsCounts,
-    indexer: &'a mut RawIndexer,
-    ranked_map: &'a mut RankedMap,
-    current_key_name: Option<String>,
-}
-
-impl<'a, 'b> ser::SerializeMap for MapSerializer<'a, 'b> {
-    type Ok = ();
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-        self.current_key_name = Some(key);
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let key = self.current_key_name.take().unwrap();
-        self.serialize_entry(&key, value)
-    }
-
-    fn serialize_entry<K: ?Sized, V: ?Sized>(
-        &mut self,
-        key: &K,
-        value: &V,
-    ) -> Result<(), Self::Error>
-    where
-        K: ser::Serialize,
-        V: ser::Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-        serialize_value(
-            self.txn,
-            key.as_str(),
-            self.schema,
-            self.document_id,
-            self.document_store,
-            self.document_fields_counts,
-            self.indexer,
-            self.ranked_map,
-            value,
-        )
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(())
-    }
-}
-
-pub struct StructSerializer<'a, 'b> {
-    txn: &'a mut heed::RwTxn<'b, MainT>,
-    schema: &'a mut Schema,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    document_fields_counts: DocumentsFieldsCounts,
-    indexer: &'a mut RawIndexer,
-    ranked_map: &'a mut RankedMap,
-}
-
-impl<'a, 'b> ser::SerializeStruct for StructSerializer<'a, 'b> {
-    type Ok = ();
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        serialize_value(
-            self.txn,
-            key,
-            self.schema,
-            self.document_id,
-            self.document_store,
-            self.document_fields_counts,
-            self.indexer,
-            self.ranked_map,
-            value,
-        )
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(())
-    }
-}
-
-pub fn serialize_value<'a, T: ?Sized>(
-    txn: &mut heed::RwTxn<MainT>,
-    attribute: &str,
-    schema: &'a mut Schema,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    documents_fields_counts: DocumentsFieldsCounts,
-    indexer: &mut RawIndexer,
-    ranked_map: &mut RankedMap,
-    value: &T,
-) -> Result<(), SerializerError>
-where
-    T: ser::Serialize,
-{
-    let field_id = schema.insert_and_index(&attribute)?;
-    serialize_value_with_id(
-        txn,
-        field_id,
-        schema,
-        document_id,
-        document_store,
-        documents_fields_counts,
-        indexer,
-        ranked_map,
-        value,
-    )
-}
-
-pub fn serialize_value_with_id<'a, T: ?Sized>(
-    txn: &mut heed::RwTxn<MainT>,
-    field_id: FieldId,
-    schema: &'a Schema,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    documents_fields_counts: DocumentsFieldsCounts,
-    indexer: &mut RawIndexer,
-    ranked_map: &mut RankedMap,
-    value: &T,
-) -> Result<(), SerializerError>
-where
-    T: ser::Serialize,
-{
-    let serialized = serde_json::to_vec(value)?;
-    document_store.put_document_field(txn, document_id, field_id, &serialized)?;
-
-    if let Some(indexed_pos) = schema.is_indexed(field_id) {
-        let indexer = Indexer {
-            pos: *indexed_pos,
-            indexer,
-            document_id,
-        };
-        if let Some(number_of_words) = value.serialize(indexer)? {
-            documents_fields_counts.put_document_field_count(
-                txn,
-                document_id,
-                *indexed_pos,
-                number_of_words as u16,
-            )?;
-        }
-    }
-
-    if schema.is_ranked(field_id) {
-        let number = value.serialize(ConvertToNumber).unwrap_or_default();
-        ranked_map.insert(document_id, field_id, number);
-    }
-
-    Ok(())
-}

meilisearch-core/src/settings.rs

@@ -1,184 +0,0 @@
-use std::collections::{BTreeMap, BTreeSet, HashSet};
-use std::str::FromStr;
-use std::iter::IntoIterator;
-
-use serde::{Deserialize, Deserializer, Serialize};
-use once_cell::sync::Lazy;
-
-use self::RankingRule::*;
-
-pub const DEFAULT_RANKING_RULES: [RankingRule; 6] = [Typo, Words, Proximity, Attribute, WordsPosition, Exactness];
-
-static RANKING_RULE_REGEX: Lazy<regex::Regex> = Lazy::new(|| {
-    let regex = regex::Regex::new(r"(asc|desc)\(([a-zA-Z0-9-_]*)\)").unwrap();
-    regex
-});
-
-#[derive(Default, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase", deny_unknown_fields)]
-pub struct Settings {
-    #[serde(default, deserialize_with = "deserialize_some")]
-    pub ranking_rules: Option<Option<Vec<String>>>,
-    #[serde(default, deserialize_with = "deserialize_some")]
-    pub distinct_attribute: Option<Option<String>>,
-    #[serde(default, deserialize_with = "deserialize_some")]
-    pub searchable_attributes: Option<Option<Vec<String>>>,
-    #[serde(default, deserialize_with = "deserialize_some")]
-    pub displayed_attributes: Option<Option<HashSet<String>>>,
-    #[serde(default, deserialize_with = "deserialize_some")]
-    pub stop_words: Option<Option<BTreeSet<String>>>,
-    #[serde(default, deserialize_with = "deserialize_some")]
-    pub synonyms: Option<Option<BTreeMap<String, Vec<String>>>>,
-    #[serde(default, deserialize_with = "deserialize_some")]
-    pub accept_new_fields: Option<Option<bool>>,
-}
-
-// Any value that is present is considered Some value, including null.
-fn deserialize_some<'de, T, D>(deserializer: D) -> Result<Option<T>, D::Error>
-    where T: Deserialize<'de>,
-          D: Deserializer<'de>
-{
-    Deserialize::deserialize(deserializer).map(Some)
-}
-
-impl Settings {
-    pub fn into_update(&self) -> Result<SettingsUpdate, RankingRuleConversionError> {
-        let settings = self.clone();
-
-        let ranking_rules = match settings.ranking_rules {
-            Some(Some(rules)) => UpdateState::Update(RankingRule::from_iter(rules.iter())?),
-            Some(None) => UpdateState::Clear,
-            None => UpdateState::Nothing,
-        };
-
-        Ok(SettingsUpdate {
-            ranking_rules,
-            distinct_attribute: settings.distinct_attribute.into(),
-            primary_key: UpdateState::Nothing,
-            searchable_attributes: settings.searchable_attributes.into(),
-            displayed_attributes: settings.displayed_attributes.into(),
-            stop_words: settings.stop_words.into(),
-            synonyms: settings.synonyms.into(),
-            accept_new_fields: settings.accept_new_fields.into(),
-        })
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum UpdateState<T> {
-    Update(T),
-    Clear,
-    Nothing,
-}
-
-impl <T> From<Option<Option<T>>> for UpdateState<T> {
-    fn from(opt: Option<Option<T>>) -> UpdateState<T> {
-        match opt {
-            Some(Some(t)) => UpdateState::Update(t),
-            Some(None) => UpdateState::Clear,
-            None => UpdateState::Nothing,
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct RankingRuleConversionError;
-
-impl std::fmt::Display for RankingRuleConversionError {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, "impossible to convert into RankingRule")
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum RankingRule {
-    Typo,
-    Words,
-    Proximity,
-    Attribute,
-    WordsPosition,
-    Exactness,
-    Asc(String),
-    Desc(String),
-}
-
-impl std::fmt::Display for RankingRule {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        match self {
-            RankingRule::Typo => f.write_str("typo"),
-            RankingRule::Words => f.write_str("words"),
-            RankingRule::Proximity => f.write_str("proximity"),
-            RankingRule::Attribute => f.write_str("attribute"),
-            RankingRule::WordsPosition => f.write_str("wordsPosition"),
-            RankingRule::Exactness => f.write_str("exactness"),
-            RankingRule::Asc(field) => write!(f, "asc({})", field),
-            RankingRule::Desc(field) => write!(f, "desc({})", field),
-        }
-    }
-}
-
-impl FromStr for RankingRule {
-    type Err = RankingRuleConversionError;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        let rule = match s {
-            "typo" => RankingRule::Typo,
-            "words" => RankingRule::Words,
-            "proximity" => RankingRule::Proximity,
-            "attribute" => RankingRule::Attribute,
-            "wordsPosition" => RankingRule::WordsPosition,
-            "exactness" => RankingRule::Exactness,
-            _ => {
-                let captures = RANKING_RULE_REGEX.captures(s).ok_or(RankingRuleConversionError)?;
-                match (captures.get(1).map(|m| m.as_str()), captures.get(2)) {
-                    (Some("asc"), Some(field)) => RankingRule::Asc(field.as_str().to_string()),
-                    (Some("desc"), Some(field)) => RankingRule::Desc(field.as_str().to_string()),
-                    _ => return Err(RankingRuleConversionError)
-                }
-            }
-        };
-        Ok(rule)
-    }
-}
-
-impl RankingRule {
-    pub fn field(&self) -> Option<&str> {
-        match self {
-            RankingRule::Asc(field) | RankingRule::Desc(field) => Some(field),
-            _ => None,
-        }
-    }
-
-    pub fn from_iter(rules: impl IntoIterator<Item = impl AsRef<str>>) -> Result<Vec<RankingRule>, RankingRuleConversionError> {
-        rules.into_iter()
-            .map(|s| RankingRule::from_str(s.as_ref()))
-            .collect()
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct SettingsUpdate {
-    pub ranking_rules: UpdateState<Vec<RankingRule>>,
-    pub distinct_attribute: UpdateState<String>,
-    pub primary_key: UpdateState<String>,
-    pub searchable_attributes: UpdateState<Vec<String>>,
-    pub displayed_attributes: UpdateState<HashSet<String>>,
-    pub stop_words: UpdateState<BTreeSet<String>>,
-    pub synonyms: UpdateState<BTreeMap<String, Vec<String>>>,
-    pub accept_new_fields: UpdateState<bool>,
-}
-
-impl Default for SettingsUpdate {
-    fn default() -> Self {
-        Self {
-            ranking_rules: UpdateState::Nothing,
-            distinct_attribute: UpdateState::Nothing,
-            primary_key: UpdateState::Nothing,
-            searchable_attributes: UpdateState::Nothing,
-            displayed_attributes: UpdateState::Nothing,
-            stop_words: UpdateState::Nothing,
-            synonyms: UpdateState::Nothing,
-            accept_new_fields: UpdateState::Nothing,
-        }
-    }
-}

meilisearch-core/src/store/docs_words.rs

@@ -1,50 +0,0 @@
-use super::BEU64;
-use crate::database::MainT;
-use crate::DocumentId;
-use heed::types::{ByteSlice, OwnedType};
-use heed::Result as ZResult;
-use std::sync::Arc;
-
-#[derive(Copy, Clone)]
-pub struct DocsWords {
-    pub(crate) docs_words: heed::Database<OwnedType<BEU64>, ByteSlice>,
-}
-
-impl DocsWords {
-    pub fn put_doc_words(
-        self,
-        writer: &mut heed::RwTxn<MainT>,
-        document_id: DocumentId,
-        words: &fst::Set,
-    ) -> ZResult<()> {
-        let document_id = BEU64::new(document_id.0);
-        let bytes = words.as_fst().as_bytes();
-        self.docs_words.put(writer, &document_id, bytes)
-    }
-
-    pub fn del_doc_words(self, writer: &mut heed::RwTxn<MainT>, document_id: DocumentId) -> ZResult<bool> {
-        let document_id = BEU64::new(document_id.0);
-        self.docs_words.delete(writer, &document_id)
-    }
-
-    pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
-        self.docs_words.clear(writer)
-    }
-
-    pub fn doc_words(
-        self,
-        reader: &heed::RoTxn<MainT>,
-        document_id: DocumentId,
-    ) -> ZResult<Option<fst::Set>> {
-        let document_id = BEU64::new(document_id.0);
-        match self.docs_words.get(reader, &document_id)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
-        }
-    }
-}

meilisearch-core/src/store/documents_fields.rs

@@ -1,79 +0,0 @@
-use heed::types::{ByteSlice, OwnedType};
-use crate::database::MainT;
-use heed::Result as ZResult;
-use meilisearch_schema::FieldId;
-
-use super::DocumentFieldStoredKey;
-use crate::DocumentId;
-
-#[derive(Copy, Clone)]
-pub struct DocumentsFields {
-    pub(crate) documents_fields: heed::Database<OwnedType<DocumentFieldStoredKey>, ByteSlice>,
-}
-
-impl DocumentsFields {
-    pub fn put_document_field(
-        self,
-        writer: &mut heed::RwTxn<MainT>,
-        document_id: DocumentId,
-        field: FieldId,
-        value: &[u8],
-    ) -> ZResult<()> {
-        let key = DocumentFieldStoredKey::new(document_id, field);
-        self.documents_fields.put(writer, &key, value)
-    }
-
-    pub fn del_all_document_fields(
-        self,
-        writer: &mut heed::RwTxn<MainT>,
-        document_id: DocumentId,
-    ) -> ZResult<usize> {
-        let start = DocumentFieldStoredKey::new(document_id, FieldId::min());
-        let end = DocumentFieldStoredKey::new(document_id, FieldId::max());
-        self.documents_fields.delete_range(writer, &(start..=end))
-    }
-
-    pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
-        self.documents_fields.clear(writer)
-    }
-
-    pub fn document_attribute<'txn>(
-        self,
-        reader: &'txn heed::RoTxn<MainT>,
-        document_id: DocumentId,
-        field: FieldId,
-    ) -> ZResult<Option<&'txn [u8]>> {
-        let key = DocumentFieldStoredKey::new(document_id, field);
-        self.documents_fields.get(reader, &key)
-    }
-
-    pub fn document_fields<'txn>(
-        self,
-        reader: &'txn heed::RoTxn<MainT>,
-        document_id: DocumentId,
-    ) -> ZResult<DocumentFieldsIter<'txn>> {
-        let start = DocumentFieldStoredKey::new(document_id, FieldId::min());
-        let end = DocumentFieldStoredKey::new(document_id, FieldId::max());
-        let iter = self.documents_fields.range(reader, &(start..=end))?;
-        Ok(DocumentFieldsIter { iter })
-    }
-}
-
-pub struct DocumentFieldsIter<'txn> {
-    iter: heed::RoRange<'txn, OwnedType<DocumentFieldStoredKey>, ByteSlice>,
-}
-
-impl<'txn> Iterator for DocumentFieldsIter<'txn> {
-    type Item = ZResult<(FieldId, &'txn [u8])>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        match self.iter.next() {
-            Some(Ok((key, bytes))) => {
-                let field_id = FieldId(key.field_id.get());
-                Some(Ok((field_id, bytes)))
-            }
-            Some(Err(e)) => Some(Err(e)),
-            None => None,
-        }
-    }
-}

meilisearch-core/src/store/documents_fields_counts.rs

@@ -1,142 +0,0 @@
-use super::DocumentFieldIndexedKey;
-use crate::database::MainT;
-use crate::DocumentId;
-use heed::types::OwnedType;
-use heed::Result as ZResult;
-use meilisearch_schema::IndexedPos;
-
-#[derive(Copy, Clone)]
-pub struct DocumentsFieldsCounts {
-    pub(crate) documents_fields_counts: heed::Database<OwnedType<DocumentFieldIndexedKey>, OwnedType<u16>>,
-}
-
-impl DocumentsFieldsCounts {
-    pub fn put_document_field_count(
-        self,
-        writer: &mut heed::RwTxn<MainT>,
-        document_id: DocumentId,
-        attribute: IndexedPos,
-        value: u16,
-    ) -> ZResult<()> {
-        let key = DocumentFieldIndexedKey::new(document_id, attribute);
-        self.documents_fields_counts.put(writer, &key, &value)
-    }
-
-    pub fn del_all_document_fields_counts(
-        self,
-        writer: &mut heed::RwTxn<MainT>,
-        document_id: DocumentId,
-    ) -> ZResult<usize> {
-        let start = DocumentFieldIndexedKey::new(document_id, IndexedPos::min());
-        let end = DocumentFieldIndexedKey::new(document_id, IndexedPos::max());
-        self.documents_fields_counts.delete_range(writer, &(start..=end))
-    }
-
-    pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
-        self.documents_fields_counts.clear(writer)
-    }
-
-    pub fn document_field_count(
-        self,
-        reader: &heed::RoTxn<MainT>,
-        document_id: DocumentId,
-        attribute: IndexedPos,
-    ) -> ZResult<Option<u16>> {
-        let key = DocumentFieldIndexedKey::new(document_id, attribute);
-        match self.documents_fields_counts.get(reader, &key)? {
-            Some(count) => Ok(Some(count)),
-            None => Ok(None),
-        }
-    }
-
-    pub fn document_fields_counts<'txn>(
-        self,
-        reader: &'txn heed::RoTxn<MainT>,
-        document_id: DocumentId,
-    ) -> ZResult<DocumentFieldsCountsIter<'txn>> {
-        let start = DocumentFieldIndexedKey::new(document_id, IndexedPos::min());
-        let end = DocumentFieldIndexedKey::new(document_id, IndexedPos::max());
-        let iter = self.documents_fields_counts.range(reader, &(start..=end))?;
-        Ok(DocumentFieldsCountsIter { iter })
-    }
-
-    pub fn documents_ids<'txn>(self, reader: &'txn heed::RoTxn<MainT>) -> ZResult<DocumentsIdsIter<'txn>> {
-        let iter = self.documents_fields_counts.iter(reader)?;
-        Ok(DocumentsIdsIter {
-            last_seen_id: None,
-            iter,
-        })
-    }
-
-    pub fn all_documents_fields_counts<'txn>(
-        self,
-        reader: &'txn heed::RoTxn<MainT>,
-    ) -> ZResult<AllDocumentsFieldsCountsIter<'txn>> {
-        let iter = self.documents_fields_counts.iter(reader)?;
-        Ok(AllDocumentsFieldsCountsIter { iter })
-    }
-}
-
-pub struct DocumentFieldsCountsIter<'txn> {
-    iter: heed::RoRange<'txn, OwnedType<DocumentFieldIndexedKey>, OwnedType<u16>>,
-}
-
-impl Iterator for DocumentFieldsCountsIter<'_> {
-    type Item = ZResult<(IndexedPos, u16)>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        match self.iter.next() {
-            Some(Ok((key, count))) => {
-                let indexed_pos = IndexedPos(key.indexed_pos.get());
-                Some(Ok((indexed_pos, count)))
-            }
-            Some(Err(e)) => Some(Err(e)),
-            None => None,
-        }
-    }
-}
-
-pub struct DocumentsIdsIter<'txn> {
-    last_seen_id: Option<DocumentId>,
-    iter: heed::RoIter<'txn, OwnedType<DocumentFieldIndexedKey>, OwnedType<u16>>,
-}
-
-impl Iterator for DocumentsIdsIter<'_> {
-    type Item = ZResult<DocumentId>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        for result in &mut self.iter {
-            match result {
-                Ok((key, _)) => {
-                    let document_id = DocumentId(key.docid.get());
-                    if Some(document_id) != self.last_seen_id {
-                        self.last_seen_id = Some(document_id);
-                        return Some(Ok(document_id));
-                    }
-                }
-                Err(e) => return Some(Err(e)),
-            }
-        }
-        None
-    }
-}
-
-pub struct AllDocumentsFieldsCountsIter<'txn> {
-    iter: heed::RoIter<'txn, OwnedType<DocumentFieldIndexedKey>, OwnedType<u16>>,
-}
-
-impl Iterator for AllDocumentsFieldsCountsIter<'_> {
-    type Item = ZResult<(DocumentId, IndexedPos, u16)>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        match self.iter.next() {
-            Some(Ok((key, count))) => {
-                let docid = DocumentId(key.docid.get());
-                let indexed_pos = IndexedPos(key.indexed_pos.get());
-                Some(Ok((docid, indexed_pos, count)))
-            }
-            Some(Err(e)) => Some(Err(e)),
-            None => None,
-        }
-    }
-}

meilisearch-core/src/store/main.rs

@@ -1,226 +0,0 @@
-use std::sync::Arc;
-use std::collections::HashMap;
-
-use chrono::{DateTime, Utc};
-use heed::types::{ByteSlice, OwnedType, SerdeBincode, Str};
-use heed::Result as ZResult;
-use meilisearch_schema::Schema;
-
-use crate::database::MainT;
-use crate::RankedMap;
-use crate::settings::RankingRule;
-
-const CREATED_AT_KEY: &str = "created-at";
-const RANKING_RULES_KEY: &str = "ranking-rules";
-const DISTINCT_ATTRIBUTE_KEY: &str = "distinct-attribute";
-const STOP_WORDS_KEY: &str = "stop-words";
-const SYNONYMS_KEY: &str = "synonyms";
-const CUSTOMS_KEY: &str = "customs";
-const FIELDS_FREQUENCY_KEY: &str = "fields-frequency";
-const NAME_KEY: &str = "name";
-const NUMBER_OF_DOCUMENTS_KEY: &str = "number-of-documents";
-const RANKED_MAP_KEY: &str = "ranked-map";
-const SCHEMA_KEY: &str = "schema";
-const UPDATED_AT_KEY: &str = "updated-at";
-const WORDS_KEY: &str = "words";
-
-pub type FreqsMap = HashMap<String, usize>;
-type SerdeFreqsMap = SerdeBincode<FreqsMap>;
-type SerdeDatetime = SerdeBincode<DateTime<Utc>>;
-
-#[derive(Copy, Clone)]
-pub struct Main {
-    pub(crate) main: heed::PolyDatabase,
-}
-
-impl Main {
-    pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
-        self.main.clear(writer)
-    }
-
-    pub fn put_name(self, writer: &mut heed::RwTxn<MainT>, name: &str) -> ZResult<()> {
-        self.main.put::<_, Str, Str>(writer, NAME_KEY, name)
-    }
-
-    pub fn name(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<String>> {
-        Ok(self
-            .main
-            .get::<_, Str, Str>(reader, NAME_KEY)?
-            .map(|name| name.to_owned()))
-    }
-
-    pub fn put_created_at(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
-        self.main
-            .put::<_, Str, SerdeDatetime>(writer, CREATED_AT_KEY, &Utc::now())
-    }
-
-    pub fn created_at(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<DateTime<Utc>>> {
-        self.main.get::<_, Str, SerdeDatetime>(reader, CREATED_AT_KEY)
-    }
-
-    pub fn put_updated_at(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
-        self.main
-            .put::<_, Str, SerdeDatetime>(writer, UPDATED_AT_KEY, &Utc::now())
-    }
-
-    pub fn updated_at(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<DateTime<Utc>>> {
-        self.main.get::<_, Str, SerdeDatetime>(reader, UPDATED_AT_KEY)
-    }
-
-    pub fn put_words_fst(self, writer: &mut heed::RwTxn<MainT>, fst: &fst::Set) -> ZResult<()> {
-        let bytes = fst.as_fst().as_bytes();
-        self.main.put::<_, Str, ByteSlice>(writer, WORDS_KEY, bytes)
-    }
-
-    pub unsafe fn static_words_fst(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<fst::Set>> {
-        match self.main.get::<_, Str, ByteSlice>(reader, WORDS_KEY)? {
-            Some(bytes) => {
-                let bytes: &'static [u8] = std::mem::transmute(bytes);
-                let set = fst::Set::from_static_slice(bytes).unwrap();
-                Ok(Some(set))
-            }
-            None => Ok(None),
-        }
-    }
-
-    pub fn words_fst(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<fst::Set>> {
-        match self.main.get::<_, Str, ByteSlice>(reader, WORDS_KEY)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
-        }
-    }
-
-    pub fn put_schema(self, writer: &mut heed::RwTxn<MainT>, schema: &Schema) -> ZResult<()> {
-        self.main.put::<_, Str, SerdeBincode<Schema>>(writer, SCHEMA_KEY, schema)
-    }
-
-    pub fn schema(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<Schema>> {
-        self.main.get::<_, Str, SerdeBincode<Schema>>(reader, SCHEMA_KEY)
-    }
-
-    pub fn delete_schema(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<bool> {
-        self.main.delete::<_, Str>(writer, SCHEMA_KEY)
-    }
-
-    pub fn put_ranked_map(self, writer: &mut heed::RwTxn<MainT>, ranked_map: &RankedMap) -> ZResult<()> {
-        self.main.put::<_, Str, SerdeBincode<RankedMap>>(writer, RANKED_MAP_KEY, &ranked_map)
-    }
-
-    pub fn ranked_map(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<RankedMap>> {
-        self.main.get::<_, Str, SerdeBincode<RankedMap>>(reader, RANKED_MAP_KEY)
-    }
-
-    pub fn put_synonyms_fst(self, writer: &mut heed::RwTxn<MainT>, fst: &fst::Set) -> ZResult<()> {
-        let bytes = fst.as_fst().as_bytes();
-        self.main.put::<_, Str, ByteSlice>(writer, SYNONYMS_KEY, bytes)
-    }
-
-    pub fn synonyms_fst(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<fst::Set>> {
-        match self.main.get::<_, Str, ByteSlice>(reader, SYNONYMS_KEY)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
-        }
-    }
-
-    pub fn put_stop_words_fst(self, writer: &mut heed::RwTxn<MainT>, fst: &fst::Set) -> ZResult<()> {
-        let bytes = fst.as_fst().as_bytes();
-        self.main.put::<_, Str, ByteSlice>(writer, STOP_WORDS_KEY, bytes)
-    }
-
-    pub fn stop_words_fst(self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<fst::Set>> {
-        match self.main.get::<_, Str, ByteSlice>(reader, STOP_WORDS_KEY)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
-        }
-    }
-
-    pub fn put_number_of_documents<F>(self, writer: &mut heed::RwTxn<MainT>, f: F) -> ZResult<u64>
-    where
-        F: Fn(u64) -> u64,
-    {
-        let new = self.number_of_documents(&*writer).map(f)?;
-        self.main
-            .put::<_, Str, OwnedType<u64>>(writer, NUMBER_OF_DOCUMENTS_KEY, &new)?;
-        Ok(new)
-    }
-
-    pub fn number_of_documents(self, reader: &heed::RoTxn<MainT>) -> ZResult<u64> {
-        match self
-            .main
-            .get::<_, Str, OwnedType<u64>>(reader, NUMBER_OF_DOCUMENTS_KEY)?
-        {
-            Some(value) => Ok(value),
-            None => Ok(0),
-        }
-    }
-
-    pub fn put_fields_frequency(
-        self,
-        writer: &mut heed::RwTxn<MainT>,
-        fields_frequency: &FreqsMap,
-    ) -> ZResult<()> {
-        self.main
-            .put::<_, Str, SerdeFreqsMap>(writer, FIELDS_FREQUENCY_KEY, fields_frequency)
-    }
-
-    pub fn fields_frequency(&self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<FreqsMap>> {
-        match self
-            .main
-            .get::<_, Str, SerdeFreqsMap>(reader, FIELDS_FREQUENCY_KEY)?
-        {
-            Some(freqs) => Ok(Some(freqs)),
-            None => Ok(None),
-        }
-    }
-
-    pub fn ranking_rules(&self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<Vec<RankingRule>>> {
-        self.main.get::<_, Str, SerdeBincode<Vec<RankingRule>>>(reader, RANKING_RULES_KEY)
-    }
-
-    pub fn put_ranking_rules(self, writer: &mut heed::RwTxn<MainT>, value: &[RankingRule]) -> ZResult<()> {
-        self.main.put::<_, Str, SerdeBincode<Vec<RankingRule>>>(writer, RANKING_RULES_KEY, &value.to_vec())
-    }
-
-    pub fn delete_ranking_rules(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<bool> {
-        self.main.delete::<_, Str>(writer, RANKING_RULES_KEY)
-    }
-
-    pub fn distinct_attribute(&self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<String>> {
-        if let Some(value) = self.main.get::<_, Str, Str>(reader, DISTINCT_ATTRIBUTE_KEY)? {
-            return Ok(Some(value.to_owned()))
-        }
-        return Ok(None)
-    }
-
-    pub fn put_distinct_attribute(self, writer: &mut heed::RwTxn<MainT>, value: &str) -> ZResult<()> {
-        self.main.put::<_, Str, Str>(writer, DISTINCT_ATTRIBUTE_KEY, value)
-    }
-
-    pub fn delete_distinct_attribute(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<bool> {
-        self.main.delete::<_, Str>(writer, DISTINCT_ATTRIBUTE_KEY)
-    }
-
-    pub fn put_customs(self, writer: &mut heed::RwTxn<MainT>, customs: &[u8]) -> ZResult<()> {
-        self.main
-            .put::<_, Str, ByteSlice>(writer, CUSTOMS_KEY, customs)
-    }
-
-    pub fn customs<'txn>(self, reader: &'txn heed::RoTxn<MainT>) -> ZResult<Option<&'txn [u8]>> {
-        self.main.get::<_, Str, ByteSlice>(reader, CUSTOMS_KEY)
-    }
-}

meilisearch-core/src/store/mod.rs

@@ -1,518 +0,0 @@
-mod docs_words;
-mod prefix_documents_cache;
-mod prefix_postings_lists_cache;
-mod documents_fields;
-mod documents_fields_counts;
-mod main;
-mod postings_lists;
-mod synonyms;
-mod updates;
-mod updates_results;
-
-pub use self::docs_words::DocsWords;
-pub use self::prefix_documents_cache::PrefixDocumentsCache;
-pub use self::prefix_postings_lists_cache::PrefixPostingsListsCache;
-pub use self::documents_fields::{DocumentFieldsIter, DocumentsFields};
-pub use self::documents_fields_counts::{
-    DocumentFieldsCountsIter, DocumentsFieldsCounts, DocumentsIdsIter,
-};
-pub use self::main::Main;
-pub use self::postings_lists::PostingsLists;
-pub use self::synonyms::Synonyms;
-pub use self::updates::Updates;
-pub use self::updates_results::UpdatesResults;
-
-use std::borrow::Cow;
-use std::collections::HashSet;
-use std::convert::TryInto;
-use std::{mem, ptr};
-
-use heed::Result as ZResult;
-use heed::{BytesEncode, BytesDecode};
-use meilisearch_schema::{IndexedPos, FieldId};
-use sdset::{Set, SetBuf};
-use serde::de::{self, Deserialize};
-use zerocopy::{AsBytes, FromBytes};
-
-use crate::criterion::Criteria;
-use crate::database::{MainT, UpdateT};
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::serde::Deserializer;
-use crate::settings::SettingsUpdate;
-use crate::{query_builder::QueryBuilder, update, DocIndex, DocumentId, Error, MResult};
-
-type BEU64 = zerocopy::U64<byteorder::BigEndian>;
-type BEU16 = zerocopy::U16<byteorder::BigEndian>;
-
-#[derive(Debug, Copy, Clone, AsBytes, FromBytes)]
-#[repr(C)]
-pub struct DocumentFieldIndexedKey {
-    docid: BEU64,
-    indexed_pos: BEU16,
-}
-
-impl DocumentFieldIndexedKey {
-    fn new(docid: DocumentId, indexed_pos: IndexedPos) -> DocumentFieldIndexedKey {
-        DocumentFieldIndexedKey {
-            docid: BEU64::new(docid.0),
-            indexed_pos: BEU16::new(indexed_pos.0),
-        }
-    }
-}
-
-#[derive(Debug, Copy, Clone, AsBytes, FromBytes)]
-#[repr(C)]
-pub struct DocumentFieldStoredKey {
-    docid: BEU64,
-    field_id: BEU16,
-}
-
-impl DocumentFieldStoredKey {
-    fn new(docid: DocumentId, field_id: FieldId) -> DocumentFieldStoredKey {
-        DocumentFieldStoredKey {
-            docid: BEU64::new(docid.0),
-            field_id: BEU16::new(field_id.0),
-        }
-    }
-}
-
-#[derive(Default, Debug)]
-pub struct Postings<'a> {
-    pub docids: Cow<'a, Set<DocumentId>>,
-    pub matches: Cow<'a, Set<DocIndex>>,
-}
-
-pub struct PostingsCodec;
-
-impl<'a> BytesEncode<'a> for PostingsCodec {
-    type EItem = Postings<'a>;
-
-    fn bytes_encode(item: &'a Self::EItem) -> Option<Cow<'a, [u8]>> {
-        let u64_size = mem::size_of::<u64>();
-        let docids_size = item.docids.len() * mem::size_of::<DocumentId>();
-        let matches_size = item.matches.len() * mem::size_of::<DocIndex>();
-
-        let mut buffer = Vec::with_capacity(u64_size + docids_size + matches_size);
-
-        let docids_len = item.docids.len();
-        buffer.extend_from_slice(&docids_len.to_be_bytes());
-        buffer.extend_from_slice(item.docids.as_bytes());
-        buffer.extend_from_slice(item.matches.as_bytes());
-
-        Some(Cow::Owned(buffer))
-    }
-}
-
-fn aligned_to(bytes: &[u8], align: usize) -> bool {
-    (bytes as *const _ as *const () as usize) % align == 0
-}
-
-fn from_bytes_to_set<'a, T: 'a>(bytes: &'a [u8]) -> Option<Cow<'a, Set<T>>>
-where T: Clone + FromBytes
-{
-    match zerocopy::LayoutVerified::<_, [T]>::new_slice(bytes) {
-        Some(layout) => Some(Cow::Borrowed(Set::new_unchecked(layout.into_slice()))),
-        None => {
-            let len = bytes.len();
-            let elem_size = mem::size_of::<T>();
-
-            // ensure that it is the alignment that is wrong
-            // and the length is valid
-            if len % elem_size == 0 && !aligned_to(bytes, mem::align_of::<T>()) {
-                let elems = len / elem_size;
-                let mut vec = Vec::<T>::with_capacity(elems);
-
-                unsafe {
-                    let dst = vec.as_mut_ptr() as *mut u8;
-                    ptr::copy_nonoverlapping(bytes.as_ptr(), dst, len);
-                    vec.set_len(elems);
-                }
-
-                return Some(Cow::Owned(SetBuf::new_unchecked(vec)));
-            }
-
-            None
-        }
-    }
-}
-
-impl<'a> BytesDecode<'a> for PostingsCodec {
-    type DItem = Postings<'a>;
-
-    fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
-        let u64_size = mem::size_of::<u64>();
-        let docid_size = mem::size_of::<DocumentId>();
-
-        let (len_bytes, bytes) = bytes.split_at(u64_size);
-        let docids_len = len_bytes.try_into().ok().map(u64::from_be_bytes)? as usize;
-        let docids_size = docids_len * docid_size;
-
-        let docids_bytes = &bytes[..docids_size];
-        let matches_bytes = &bytes[docids_size..];
-
-        let docids = from_bytes_to_set(docids_bytes)?;
-        let matches = from_bytes_to_set(matches_bytes)?;
-
-        Some(Postings { docids, matches })
-    }
-}
-
-fn main_name(name: &str) -> String {
-    format!("store-{}", name)
-}
-
-fn postings_lists_name(name: &str) -> String {
-    format!("store-{}-postings-lists", name)
-}
-
-fn documents_fields_name(name: &str) -> String {
-    format!("store-{}-documents-fields", name)
-}
-
-fn documents_fields_counts_name(name: &str) -> String {
-    format!("store-{}-documents-fields-counts", name)
-}
-
-fn synonyms_name(name: &str) -> String {
-    format!("store-{}-synonyms", name)
-}
-
-fn docs_words_name(name: &str) -> String {
-    format!("store-{}-docs-words", name)
-}
-
-fn prefix_documents_cache_name(name: &str) -> String {
-    format!("store-{}-prefix-documents-cache", name)
-}
-
-fn prefix_postings_lists_cache_name(name: &str) -> String {
-    format!("store-{}-prefix-postings-lists-cache", name)
-}
-
-fn updates_name(name: &str) -> String {
-    format!("store-{}-updates", name)
-}
-
-fn updates_results_name(name: &str) -> String {
-    format!("store-{}-updates-results", name)
-}
-
-#[derive(Clone)]
-pub struct Index {
-    pub main: Main,
-    pub postings_lists: PostingsLists,
-    pub documents_fields: DocumentsFields,
-    pub documents_fields_counts: DocumentsFieldsCounts,
-    pub synonyms: Synonyms,
-    pub docs_words: DocsWords,
-    pub prefix_documents_cache: PrefixDocumentsCache,
-    pub prefix_postings_lists_cache: PrefixPostingsListsCache,
-
-    pub updates: Updates,
-    pub updates_results: UpdatesResults,
-    pub(crate) updates_notifier: UpdateEventsEmitter,
-}
-
-impl Index {
-    pub fn document<T: de::DeserializeOwned>(
-        &self,
-        reader: &heed::RoTxn<MainT>,
-        attributes: Option<&HashSet<&str>>,
-        document_id: DocumentId,
-    ) -> MResult<Option<T>> {
-        let schema = self.main.schema(reader)?;
-        let schema = schema.ok_or(Error::SchemaMissing)?;
-
-        let attributes = match attributes {
-            Some(attributes) => Some(attributes.iter().filter_map(|name| schema.id(*name)).collect()),
-            None => None,
-        };
-
-        let mut deserializer = Deserializer {
-            document_id,
-            reader,
-            documents_fields: self.documents_fields,
-            schema: &schema,
-            fields: attributes.as_ref(),
-        };
-
-        Ok(Option::<T>::deserialize(&mut deserializer)?)
-    }
-
-    pub fn document_attribute<T: de::DeserializeOwned>(
-        &self,
-        reader: &heed::RoTxn<MainT>,
-        document_id: DocumentId,
-        attribute: FieldId,
-    ) -> MResult<Option<T>> {
-        let bytes = self
-            .documents_fields
-            .document_attribute(reader, document_id, attribute)?;
-        match bytes {
-            Some(bytes) => Ok(Some(serde_json::from_slice(bytes)?)),
-            None => Ok(None),
-        }
-    }
-
-    pub fn document_attribute_bytes<'txn>(
-        &self,
-        reader: &'txn heed::RoTxn<MainT>,
-        document_id: DocumentId,
-        attribute: FieldId,
-    ) -> MResult<Option<&'txn [u8]>> {
-        let bytes = self
-            .documents_fields
-            .document_attribute(reader, document_id, attribute)?;
-        match bytes {
-            Some(bytes) => Ok(Some(bytes)),
-            None => Ok(None),
-        }
-    }
-
-    pub fn customs_update(&self, writer: &mut heed::RwTxn<UpdateT>, customs: Vec<u8>) -> ZResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        update::push_customs_update(writer, self.updates, self.updates_results, customs)
-    }
-
-    pub fn settings_update(&self, writer: &mut heed::RwTxn<UpdateT>, update: SettingsUpdate) -> ZResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        update::push_settings_update(writer, self.updates, self.updates_results, update)
-    }
-
-    pub fn documents_addition<D>(&self) -> update::DocumentsAddition<D> {
-        update::DocumentsAddition::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn documents_partial_addition<D>(&self) -> update::DocumentsAddition<D> {
-        update::DocumentsAddition::new_partial(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn documents_deletion(&self) -> update::DocumentsDeletion {
-        update::DocumentsDeletion::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn clear_all(&self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        update::push_clear_all(writer, self.updates, self.updates_results)
-    }
-
-    pub fn current_update_id(&self, reader: &heed::RoTxn<UpdateT>) -> MResult<Option<u64>> {
-        match self.updates.last_update(reader)? {
-            Some((id, _)) => Ok(Some(id)),
-            None => Ok(None),
-        }
-    }
-
-    pub fn update_status(
-        &self,
-        reader: &heed::RoTxn<UpdateT>,
-        update_id: u64,
-    ) -> MResult<Option<update::UpdateStatus>> {
-        update::update_status(reader, self.updates, self.updates_results, update_id)
-    }
-
-    pub fn all_updates_status(&self, reader: &heed::RoTxn<UpdateT>) -> MResult<Vec<update::UpdateStatus>> {
-        let mut updates = Vec::new();
-        let mut last_update_result_id = 0;
-
-        // retrieve all updates results
-        if let Some((last_id, _)) = self.updates_results.last_update(reader)? {
-            updates.reserve(last_id as usize);
-
-            for id in 0..=last_id {
-                if let Some(update) = self.update_status(reader, id)? {
-                    updates.push(update);
-                    last_update_result_id = id;
-                }
-            }
-        }
-
-        // retrieve all enqueued updates
-        if let Some((last_id, _)) = self.updates.last_update(reader)? {
-            for id in last_update_result_id + 1..=last_id {
-                if let Some(update) = self.update_status(reader, id)? {
-                    updates.push(update);
-                }
-            }
-        }
-
-        Ok(updates)
-    }
-
-    pub fn query_builder(&self) -> QueryBuilder {
-        QueryBuilder::new(
-            self.main,
-            self.postings_lists,
-            self.documents_fields_counts,
-            self.synonyms,
-            self.prefix_documents_cache,
-            self.prefix_postings_lists_cache,
-        )
-    }
-
-    pub fn query_builder_with_criteria<'c, 'f, 'd>(
-        &self,
-        criteria: Criteria<'c>,
-    ) -> QueryBuilder<'c, 'f, 'd> {
-        QueryBuilder::with_criteria(
-            self.main,
-            self.postings_lists,
-            self.documents_fields_counts,
-            self.synonyms,
-            self.prefix_documents_cache,
-            self.prefix_postings_lists_cache,
-            criteria,
-        )
-    }
-}
-
-pub fn create(
-    env: &heed::Env,
-    update_env: &heed::Env,
-    name: &str,
-    updates_notifier: UpdateEventsEmitter,
-) -> MResult<Index> {
-    // create all the store names
-    let main_name = main_name(name);
-    let postings_lists_name = postings_lists_name(name);
-    let documents_fields_name = documents_fields_name(name);
-    let documents_fields_counts_name = documents_fields_counts_name(name);
-    let synonyms_name = synonyms_name(name);
-    let docs_words_name = docs_words_name(name);
-    let prefix_documents_cache_name = prefix_documents_cache_name(name);
-    let prefix_postings_lists_cache_name = prefix_postings_lists_cache_name(name);
-    let updates_name = updates_name(name);
-    let updates_results_name = updates_results_name(name);
-
-    // open all the stores
-    let main = env.create_poly_database(Some(&main_name))?;
-    let postings_lists = env.create_database(Some(&postings_lists_name))?;
-    let documents_fields = env.create_database(Some(&documents_fields_name))?;
-    let documents_fields_counts = env.create_database(Some(&documents_fields_counts_name))?;
-    let synonyms = env.create_database(Some(&synonyms_name))?;
-    let docs_words = env.create_database(Some(&docs_words_name))?;
-    let prefix_documents_cache = env.create_database(Some(&prefix_documents_cache_name))?;
-    let prefix_postings_lists_cache = env.create_database(Some(&prefix_postings_lists_cache_name))?;
-    let updates = update_env.create_database(Some(&updates_name))?;
-    let updates_results = update_env.create_database(Some(&updates_results_name))?;
-
-    Ok(Index {
-        main: Main { main },
-        postings_lists: PostingsLists { postings_lists },
-        documents_fields: DocumentsFields { documents_fields },
-        documents_fields_counts: DocumentsFieldsCounts { documents_fields_counts },
-        synonyms: Synonyms { synonyms },
-        docs_words: DocsWords { docs_words },
-        prefix_postings_lists_cache: PrefixPostingsListsCache { prefix_postings_lists_cache },
-        prefix_documents_cache: PrefixDocumentsCache { prefix_documents_cache },
-        updates: Updates { updates },
-        updates_results: UpdatesResults { updates_results },
-        updates_notifier,
-    })
-}
-
-pub fn open(
-    env: &heed::Env,
-    update_env: &heed::Env,
-    name: &str,
-    updates_notifier: UpdateEventsEmitter,
-) -> MResult<Option<Index>> {
-    // create all the store names
-    let main_name = main_name(name);
-    let postings_lists_name = postings_lists_name(name);
-    let documents_fields_name = documents_fields_name(name);
-    let documents_fields_counts_name = documents_fields_counts_name(name);
-    let synonyms_name = synonyms_name(name);
-    let docs_words_name = docs_words_name(name);
-    let prefix_documents_cache_name = prefix_documents_cache_name(name);
-    let prefix_postings_lists_cache_name = prefix_postings_lists_cache_name(name);
-    let updates_name = updates_name(name);
-    let updates_results_name = updates_results_name(name);
-
-    // open all the stores
-    let main = match env.open_poly_database(Some(&main_name))? {
-        Some(main) => main,
-        None => return Ok(None),
-    };
-    let postings_lists = match env.open_database(Some(&postings_lists_name))? {
-        Some(postings_lists) => postings_lists,
-        None => return Ok(None),
-    };
-    let documents_fields = match env.open_database(Some(&documents_fields_name))? {
-        Some(documents_fields) => documents_fields,
-        None => return Ok(None),
-    };
-    let documents_fields_counts = match env.open_database(Some(&documents_fields_counts_name))? {
-        Some(documents_fields_counts) => documents_fields_counts,
-        None => return Ok(None),
-    };
-    let synonyms = match env.open_database(Some(&synonyms_name))? {
-        Some(synonyms) => synonyms,
-        None => return Ok(None),
-    };
-    let docs_words = match env.open_database(Some(&docs_words_name))? {
-        Some(docs_words) => docs_words,
-        None => return Ok(None),
-    };
-    let prefix_documents_cache = match env.open_database(Some(&prefix_documents_cache_name))? {
-        Some(prefix_documents_cache) => prefix_documents_cache,
-        None => return Ok(None),
-    };
-    let prefix_postings_lists_cache = match env.open_database(Some(&prefix_postings_lists_cache_name))? {
-        Some(prefix_postings_lists_cache) => prefix_postings_lists_cache,
-        None => return Ok(None),
-    };
-    let updates = match update_env.open_database(Some(&updates_name))? {
-        Some(updates) => updates,
-        None => return Ok(None),
-    };
-    let updates_results = match update_env.open_database(Some(&updates_results_name))? {
-        Some(updates_results) => updates_results,
-        None => return Ok(None),
-    };
-
-    Ok(Some(Index {
-        main: Main { main },
-        postings_lists: PostingsLists { postings_lists },
-        documents_fields: DocumentsFields { documents_fields },
-        documents_fields_counts: DocumentsFieldsCounts { documents_fields_counts },
-        synonyms: Synonyms { synonyms },
-        docs_words: DocsWords { docs_words },
-        prefix_documents_cache: PrefixDocumentsCache { prefix_documents_cache },
-        prefix_postings_lists_cache: PrefixPostingsListsCache { prefix_postings_lists_cache },
-        updates: Updates { updates },
-        updates_results: UpdatesResults { updates_results },
-        updates_notifier,
-    }))
-}
-
-pub fn clear(
-    writer: &mut heed::RwTxn<MainT>,
-    update_writer: &mut heed::RwTxn<UpdateT>,
-    index: &Index,
-) -> MResult<()> {
-    // clear all the stores
-    index.main.clear(writer)?;
-    index.postings_lists.clear(writer)?;
-    index.documents_fields.clear(writer)?;
-    index.documents_fields_counts.clear(writer)?;
-    index.synonyms.clear(writer)?;
-    index.docs_words.clear(writer)?;
-    index.prefix_documents_cache.clear(writer)?;
-    index.prefix_postings_lists_cache.clear(writer)?;
-    index.updates.clear(update_writer)?;
-    index.updates_results.clear(update_writer)?;
-    Ok(())
-}

meilisearch-core/src/store/postings_lists.rs

@@ -1,47 +0,0 @@
-use std::borrow::Cow;
-
-use heed::Result as ZResult;
-use heed::types::ByteSlice;
-use sdset::{Set, SetBuf};
-use slice_group_by::GroupBy;
-
-use crate::database::MainT;
-use crate::DocIndex;
-use crate::store::{Postings, PostingsCodec};
-
-#[derive(Copy, Clone)]
-pub struct PostingsLists {
-    pub(crate) postings_lists: heed::Database<ByteSlice, PostingsCodec>,
-}
-
-impl PostingsLists {
-    pub fn put_postings_list(
-        self,
-        writer: &mut heed::RwTxn<MainT>,
-        word: &[u8],
-        matches: &Set<DocIndex>,
-    ) -> ZResult<()> {
-        let docids = matches.linear_group_by_key(|m| m.document_id).map(|g| g[0].document_id).collect();
-        let docids = Cow::Owned(SetBuf::new_unchecked(docids));
-        let matches = Cow::Borrowed(matches);
-        let postings = Postings { docids, matches };
-
-        self.postings_lists.put(writer, word, &postings)
-    }
-
-    pub fn del_postings_list(self, writer: &mut heed::RwTxn<MainT>, word: &[u8]) -> ZResult<bool> {
-        self.postings_lists.delete(writer, word)
-    }
-
-    pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
-        self.postings_lists.clear(writer)
-    }
-
-    pub fn postings_list<'txn>(
-        self,
-        reader: &'txn heed::RoTxn<MainT>,
-        word: &[u8],
-    ) -> ZResult<Option<Postings<'txn>>> {
-        self.postings_lists.get(reader, word)
-    }
-}

meilisearch-core/src/store/prefix_documents_cache.rs

@@ -1,80 +0,0 @@
-use std::borrow::Cow;
-
-use heed::types::{OwnedType, CowSlice};
-use heed::Result as ZResult;
-use zerocopy::{AsBytes, FromBytes};
-
-use super::BEU64;
-use crate::{DocumentId, Highlight};
-use crate::database::MainT;
-
-#[derive(Debug, Copy, Clone, AsBytes, FromBytes)]
-#[repr(C)]
-pub struct PrefixKey {
-    prefix: [u8; 4],
-    index: BEU64,
-    docid: BEU64,
-}
-
-impl PrefixKey {
-    pub fn new(prefix: [u8; 4], index: u64, docid: u64) -> PrefixKey {
-        PrefixKey {
-            prefix,
-            index: BEU64::new(index),
-            docid: BEU64::new(docid),
-        }
-    }
-}
-
-#[derive(Copy, Clone)]
-pub struct PrefixDocumentsCache {
-    pub(crate) prefix_documents_cache: heed::Database<OwnedType<PrefixKey>, CowSlice<Highlight>>,
-}
-
-impl PrefixDocumentsCache {
-    pub fn put_prefix_document(
-        self,
-        writer: &mut heed::RwTxn<MainT>,
-        prefix: [u8; 4],
-        index: usize,
-        docid: DocumentId,
-        highlights: &[Highlight],
-    ) -> ZResult<()> {
-        let key = PrefixKey::new(prefix, index as u64, docid.0);
-        self.prefix_documents_cache.put(writer, &key, highlights)
-    }
-
-    pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
-        self.prefix_documents_cache.clear(writer)
-    }
-
-    pub fn prefix_documents<'txn>(
-        self,
-        reader: &'txn heed::RoTxn<MainT>,
-        prefix: [u8; 4],
-    ) -> ZResult<PrefixDocumentsIter<'txn>> {
-        let start = PrefixKey::new(prefix, 0, 0);
-        let end = PrefixKey::new(prefix, u64::max_value(), u64::max_value());
-        let iter = self.prefix_documents_cache.range(reader, &(start..=end))?;
-        Ok(PrefixDocumentsIter { iter })
-    }
-}
-
-pub struct PrefixDocumentsIter<'txn> {
-    iter: heed::RoRange<'txn, OwnedType<PrefixKey>, CowSlice<Highlight>>,
-}
-
-impl<'txn> Iterator for PrefixDocumentsIter<'txn> {
-    type Item = ZResult<(DocumentId, Cow<'txn, [Highlight]>)>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        match self.iter.next() {
-            Some(Ok((key, highlights))) => {
-                let docid = DocumentId(key.docid.get());
-                Some(Ok((docid, highlights)))
-            }
-            Some(Err(e)) => Some(Err(e)),
-            None => None,
-        }
-    }
-}

meilisearch-core/src/store/prefix_postings_lists_cache.rs

@@ -1,45 +0,0 @@
-use std::borrow::Cow;
-
-use heed::Result as ZResult;
-use heed::types::OwnedType;
-use sdset::{Set, SetBuf};
-use slice_group_by::GroupBy;
-
-use crate::database::MainT;
-use crate::DocIndex;
-use crate::store::{PostingsCodec, Postings};
-
-#[derive(Copy, Clone)]
-pub struct PrefixPostingsListsCache {
-    pub(crate) prefix_postings_lists_cache: heed::Database<OwnedType<[u8; 4]>, PostingsCodec>,
-}
-
-impl PrefixPostingsListsCache {
-    pub fn put_prefix_postings_list(
-        self,
-        writer: &mut heed::RwTxn<MainT>,
-        prefix: [u8; 4],
-        matches: &Set<DocIndex>,
-    ) -> ZResult<()>
-    {
-        let docids = matches.linear_group_by_key(|m| m.document_id).map(|g| g[0].document_id).collect();
-        let docids = Cow::Owned(SetBuf::new_unchecked(docids));
-        let matches = Cow::Borrowed(matches);
-        let postings = Postings { docids, matches };
-
-        self.prefix_postings_lists_cache.put(writer, &prefix, &postings)
-    }
-
-    pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
-        self.prefix_postings_lists_cache.clear(writer)
-    }
-
-    pub fn prefix_postings_list<'txn>(
-        self,
-        reader: &'txn heed::RoTxn<MainT>,
-        prefix: [u8; 4],
-    ) -> ZResult<Option<Postings<'txn>>>
-    {
-        self.prefix_postings_lists_cache.get(reader, &prefix)
-    }
-}

meilisearch-core/src/store/synonyms.rs

@@ -1,41 +0,0 @@
-use heed::types::ByteSlice;
-use crate::database::MainT;
-use heed::Result as ZResult;
-use std::sync::Arc;
-
-#[derive(Copy, Clone)]
-pub struct Synonyms {
-    pub(crate) synonyms: heed::Database<ByteSlice, ByteSlice>,
-}
-
-impl Synonyms {
-    pub fn put_synonyms(
-        self,
-        writer: &mut heed::RwTxn<MainT>,
-        word: &[u8],
-        synonyms: &fst::Set,
-    ) -> ZResult<()> {
-        let bytes = synonyms.as_fst().as_bytes();
-        self.synonyms.put(writer, word, bytes)
-    }
-
-    pub fn del_synonyms(self, writer: &mut heed::RwTxn<MainT>, word: &[u8]) -> ZResult<bool> {
-        self.synonyms.delete(writer, word)
-    }
-
-    pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
-        self.synonyms.clear(writer)
-    }
-
-    pub fn synonyms(self, reader: &heed::RoTxn<MainT>, word: &[u8]) -> ZResult<Option<fst::Set>> {
-        match self.synonyms.get(reader, word)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
-        }
-    }
-}

meilisearch-core/src/store/updates.rs

@@ -1,65 +0,0 @@
-use super::BEU64;
-use crate::database::UpdateT;
-use crate::update::Update;
-use heed::types::{OwnedType, SerdeJson};
-use heed::Result as ZResult;
-
-#[derive(Copy, Clone)]
-pub struct Updates {
-    pub(crate) updates: heed::Database<OwnedType<BEU64>, SerdeJson<Update>>,
-}
-
-impl Updates {
-    // TODO do not trigger deserialize if possible
-    pub fn last_update(self, reader: &heed::RoTxn<UpdateT>) -> ZResult<Option<(u64, Update)>> {
-        match self.updates.last(reader)? {
-            Some((key, data)) => Ok(Some((key.get(), data))),
-            None => Ok(None),
-        }
-    }
-
-    // TODO do not trigger deserialize if possible
-    pub fn first_update(self, reader: &heed::RoTxn<UpdateT>) -> ZResult<Option<(u64, Update)>> {
-        match self.updates.first(reader)? {
-            Some((key, data)) => Ok(Some((key.get(), data))),
-            None => Ok(None),
-        }
-    }
-
-    // TODO do not trigger deserialize if possible
-    pub fn get(self, reader: &heed::RoTxn<UpdateT>, update_id: u64) -> ZResult<Option<Update>> {
-        let update_id = BEU64::new(update_id);
-        self.updates.get(reader, &update_id)
-    }
-
-    pub fn put_update(
-        self,
-        writer: &mut heed::RwTxn<UpdateT>,
-        update_id: u64,
-        update: &Update,
-    ) -> ZResult<()> {
-        // TODO prefer using serde_json?
-        let update_id = BEU64::new(update_id);
-        self.updates.put(writer, &update_id, update)
-    }
-
-    pub fn del_update(self, writer: &mut heed::RwTxn<UpdateT>, update_id: u64) -> ZResult<bool> {
-        let update_id = BEU64::new(update_id);
-        self.updates.delete(writer, &update_id)
-    }
-
-    pub fn pop_front(self, writer: &mut heed::RwTxn<UpdateT>) -> ZResult<Option<(u64, Update)>> {
-        match self.first_update(writer)? {
-            Some((update_id, update)) => {
-                let key = BEU64::new(update_id);
-                self.updates.delete(writer, &key)?;
-                Ok(Some((update_id, update)))
-            }
-            None => Ok(None),
-        }
-    }
-
-    pub fn clear(self, writer: &mut heed::RwTxn<UpdateT>) -> ZResult<()> {
-        self.updates.clear(writer)
-    }
-}

meilisearch-core/src/store/updates_results.rs

@@ -1,45 +0,0 @@
-use super::BEU64;
-use crate::database::UpdateT;
-use crate::update::ProcessedUpdateResult;
-use heed::types::{OwnedType, SerdeJson};
-use heed::Result as ZResult;
-
-#[derive(Copy, Clone)]
-pub struct UpdatesResults {
-    pub(crate) updates_results: heed::Database<OwnedType<BEU64>, SerdeJson<ProcessedUpdateResult>>,
-}
-
-impl UpdatesResults {
-    pub fn last_update(
-        self,
-        reader: &heed::RoTxn<UpdateT>,
-    ) -> ZResult<Option<(u64, ProcessedUpdateResult)>> {
-        match self.updates_results.last(reader)? {
-            Some((key, data)) => Ok(Some((key.get(), data))),
-            None => Ok(None),
-        }
-    }
-
-    pub fn put_update_result(
-        self,
-        writer: &mut heed::RwTxn<UpdateT>,
-        update_id: u64,
-        update_result: &ProcessedUpdateResult,
-    ) -> ZResult<()> {
-        let update_id = BEU64::new(update_id);
-        self.updates_results.put(writer, &update_id, update_result)
-    }
-
-    pub fn update_result(
-        self,
-        reader: &heed::RoTxn<UpdateT>,
-        update_id: u64,
-    ) -> ZResult<Option<ProcessedUpdateResult>> {
-        let update_id = BEU64::new(update_id);
-        self.updates_results.get(reader, &update_id)
-    }
-
-    pub fn clear(self, writer: &mut heed::RwTxn<UpdateT>) -> ZResult<()> {
-        self.updates_results.clear(writer)
-    }
-}

meilisearch-core/src/update/clear_all.rs

@@ -1,32 +0,0 @@
-use crate::database::{MainT, UpdateT};
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult, RankedMap};
-
-pub fn apply_clear_all(
-    writer: &mut heed::RwTxn<MainT>,
-    index: &store::Index,
-) -> MResult<()> {
-    index.main.put_words_fst(writer, &fst::Set::default())?;
-    index.main.put_ranked_map(writer, &RankedMap::default())?;
-    index.main.put_number_of_documents(writer, |_| 0)?;
-    index.documents_fields.clear(writer)?;
-    index.documents_fields_counts.clear(writer)?;
-    index.postings_lists.clear(writer)?;
-    index.docs_words.clear(writer)?;
-    index.prefix_documents_cache.clear(writer)?;
-    index.prefix_postings_lists_cache.clear(writer)?;
-
-    Ok(())
-}
-
-pub fn push_clear_all(
-    writer: &mut heed::RwTxn<UpdateT>,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-    let update = Update::clear_all();
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}

meilisearch-core/src/update/customs_update.rs

@@ -1,27 +0,0 @@
-use heed::Result as ZResult;
-
-use crate::database::{MainT, UpdateT};
-use crate::store;
-use crate::update::{next_update_id, Update};
-
-pub fn apply_customs_update(
-    writer: &mut heed::RwTxn<MainT>,
-    main_store: store::Main,
-    customs: &[u8],
-) -> ZResult<()> {
-    main_store.put_customs(writer, customs)
-}
-
-pub fn push_customs_update(
-    writer: &mut heed::RwTxn<UpdateT>,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    customs: Vec<u8>,
-) -> ZResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::customs(customs);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}

meilisearch-core/src/update/documents_addition.rs

@@ -1,382 +0,0 @@
-use std::collections::HashMap;
-
-use fst::{set::OpBuilder, SetBuilder};
-use indexmap::IndexMap;
-use sdset::{duo::Union, SetOperation};
-use serde::{Deserialize, Serialize};
-
-use crate::database::{MainT, UpdateT};
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::raw_indexer::RawIndexer;
-use crate::serde::{extract_document_id, serialize_value_with_id, Deserializer, Serializer};
-use crate::store;
-use crate::update::{apply_documents_deletion, compute_short_prefixes, next_update_id, Update};
-use crate::{Error, MResult, RankedMap};
-
-pub struct DocumentsAddition<D> {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: UpdateEventsEmitter,
-    documents: Vec<D>,
-    is_partial: bool,
-}
-
-impl<D> DocumentsAddition<D> {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: UpdateEventsEmitter,
-    ) -> DocumentsAddition<D> {
-        DocumentsAddition {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            documents: Vec::new(),
-            is_partial: false,
-        }
-    }
-
-    pub fn new_partial(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: UpdateEventsEmitter,
-    ) -> DocumentsAddition<D> {
-        DocumentsAddition {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            documents: Vec::new(),
-            is_partial: true,
-        }
-    }
-
-    pub fn update_document(&mut self, document: D) {
-        self.documents.push(document);
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64>
-    where
-        D: serde::Serialize,
-    {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        let update_id = push_documents_addition(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.documents,
-            self.is_partial,
-        )?;
-        Ok(update_id)
-    }
-}
-
-impl<D> Extend<D> for DocumentsAddition<D> {
-    fn extend<T: IntoIterator<Item = D>>(&mut self, iter: T) {
-        self.documents.extend(iter)
-    }
-}
-
-pub fn push_documents_addition<D: serde::Serialize>(
-    writer: &mut heed::RwTxn<UpdateT>,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    addition: Vec<D>,
-    is_partial: bool,
-) -> MResult<u64> {
-    let mut values = Vec::with_capacity(addition.len());
-    for add in addition {
-        let vec = serde_json::to_vec(&add)?;
-        let add = serde_json::from_slice(&vec)?;
-        values.push(add);
-    }
-
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = if is_partial {
-        Update::documents_partial(values)
-    } else {
-        Update::documents_addition(values)
-    };
-
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_documents_addition<'a, 'b>(
-    writer: &'a mut heed::RwTxn<'b, MainT>,
-    index: &store::Index,
-    addition: Vec<IndexMap<String, serde_json::Value>>,
-) -> MResult<()> {
-    let mut documents_additions = HashMap::new();
-
-    let mut schema = match index.main.schema(writer)? {
-        Some(schema) => schema,
-        None => return Err(Error::SchemaMissing),
-    };
-
-    let primary_key = schema.primary_key().ok_or(Error::MissingPrimaryKey)?;
-
-    // 1. store documents ids for future deletion
-    for document in addition {
-        let document_id = match extract_document_id(&primary_key, &document)? {
-            Some(id) => id,
-            None => return Err(Error::MissingDocumentId),
-        };
-
-        documents_additions.insert(document_id, document);
-    }
-
-    // 2. remove the documents posting lists
-    let number_of_inserted_documents = documents_additions.len();
-    let documents_ids = documents_additions.iter().map(|(id, _)| *id).collect();
-    apply_documents_deletion(writer, index, documents_ids)?;
-
-    let mut ranked_map = match index.main.ranked_map(writer)? {
-        Some(ranked_map) => ranked_map,
-        None => RankedMap::default(),
-    };
-
-    let stop_words = match index.main.stop_words_fst(writer)? {
-        Some(stop_words) => stop_words,
-        None => fst::Set::default(),
-    };
-
-    // 3. index the documents fields in the stores
-    let mut indexer = RawIndexer::new(stop_words);
-
-    for (document_id, document) in documents_additions {
-        let serializer = Serializer {
-            txn: writer,
-            schema: &mut schema,
-            document_store: index.documents_fields,
-            document_fields_counts: index.documents_fields_counts,
-            indexer: &mut indexer,
-            ranked_map: &mut ranked_map,
-            document_id,
-        };
-
-        document.serialize(serializer)?;
-    }
-
-    write_documents_addition_index(
-        writer,
-        index,
-        &ranked_map,
-        number_of_inserted_documents,
-        indexer,
-    )?;
-
-    index.main.put_schema(writer, &schema)?;
-
-    Ok(())
-}
-
-pub fn apply_documents_partial_addition<'a, 'b>(
-    writer: &'a mut heed::RwTxn<'b, MainT>,
-    index: &store::Index,
-    addition: Vec<IndexMap<String, serde_json::Value>>,
-) -> MResult<()> {
-    let mut documents_additions = HashMap::new();
-
-    let mut schema = match index.main.schema(writer)? {
-        Some(schema) => schema,
-        None => return Err(Error::SchemaMissing),
-    };
-
-    let primary_key = schema.primary_key().ok_or(Error::MissingPrimaryKey)?;
-
-    // 1. store documents ids for future deletion
-    for mut document in addition {
-        let document_id = match extract_document_id(&primary_key, &document)? {
-            Some(id) => id,
-            None => return Err(Error::MissingDocumentId),
-        };
-
-        let mut deserializer = Deserializer {
-            document_id,
-            reader: writer,
-            documents_fields: index.documents_fields,
-            schema: &schema,
-            fields: None,
-        };
-
-        // retrieve the old document and
-        // update the new one with missing keys found in the old one
-        let result = Option::<HashMap<String, serde_json::Value>>::deserialize(&mut deserializer)?;
-        if let Some(old_document) = result {
-            for (key, value) in old_document {
-                document.entry(key).or_insert(value);
-            }
-        }
-
-        documents_additions.insert(document_id, document);
-    }
-
-    // 2. remove the documents posting lists
-    let number_of_inserted_documents = documents_additions.len();
-    let documents_ids = documents_additions.iter().map(|(id, _)| *id).collect();
-    apply_documents_deletion(writer, index, documents_ids)?;
-
-    let mut ranked_map = match index.main.ranked_map(writer)? {
-        Some(ranked_map) => ranked_map,
-        None => RankedMap::default(),
-    };
-
-    let stop_words = match index.main.stop_words_fst(writer)? {
-        Some(stop_words) => stop_words,
-        None => fst::Set::default(),
-    };
-
-    // 3. index the documents fields in the stores
-    let mut indexer = RawIndexer::new(stop_words);
-
-    for (document_id, document) in documents_additions {
-        let serializer = Serializer {
-            txn: writer,
-            schema: &mut schema,
-            document_store: index.documents_fields,
-            document_fields_counts: index.documents_fields_counts,
-            indexer: &mut indexer,
-            ranked_map: &mut ranked_map,
-            document_id,
-        };
-
-        document.serialize(serializer)?;
-    }
-
-    write_documents_addition_index(
-        writer,
-        index,
-        &ranked_map,
-        number_of_inserted_documents,
-        indexer,
-    )?;
-
-    index.main.put_schema(writer, &schema)?;
-
-    Ok(())
-}
-
-pub fn reindex_all_documents(writer: &mut heed::RwTxn<MainT>, index: &store::Index) -> MResult<()> {
-    let schema = match index.main.schema(writer)? {
-        Some(schema) => schema,
-        None => return Err(Error::SchemaMissing),
-    };
-
-    let mut ranked_map = RankedMap::default();
-
-    // 1. retrieve all documents ids
-    let mut documents_ids_to_reindex = Vec::new();
-    for result in index.documents_fields_counts.documents_ids(writer)? {
-        let document_id = result?;
-        documents_ids_to_reindex.push(document_id);
-    }
-
-    // 2. remove the documents posting lists
-    index.main.put_words_fst(writer, &fst::Set::default())?;
-    index.main.put_ranked_map(writer, &ranked_map)?;
-    index.main.put_number_of_documents(writer, |_| 0)?;
-    index.postings_lists.clear(writer)?;
-    index.docs_words.clear(writer)?;
-
-    let stop_words = match index.main.stop_words_fst(writer)? {
-        Some(stop_words) => stop_words,
-        None => fst::Set::default(),
-    };
-
-    let number_of_inserted_documents = documents_ids_to_reindex.len();
-    let mut indexer = RawIndexer::new(stop_words);
-    let mut ram_store = HashMap::new();
-
-    for document_id in documents_ids_to_reindex {
-        for result in index.documents_fields.document_fields(writer, document_id)? {
-            let (field_id, bytes) = result?;
-            let value: serde_json::Value = serde_json::from_slice(bytes)?;
-            ram_store.insert((document_id, field_id), value);
-        }
-
-        for ((docid, field_id), value) in ram_store.drain() {
-            serialize_value_with_id(
-                writer,
-                field_id,
-                &schema,
-                docid,
-                index.documents_fields,
-                index.documents_fields_counts,
-                &mut indexer,
-                &mut ranked_map,
-                &value
-            )?;
-        }
-    }
-
-    // 4. write the new index in the main store
-    write_documents_addition_index(
-        writer,
-        index,
-        &ranked_map,
-        number_of_inserted_documents,
-        indexer,
-    )?;
-
-    index.main.put_schema(writer, &schema)?;
-
-    Ok(())
-}
-
-pub fn write_documents_addition_index(
-    writer: &mut heed::RwTxn<MainT>,
-    index: &store::Index,
-    ranked_map: &RankedMap,
-    number_of_inserted_documents: usize,
-    indexer: RawIndexer,
-) -> MResult<()> {
-    let indexed = indexer.build();
-    let mut delta_words_builder = SetBuilder::memory();
-
-    for (word, delta_set) in indexed.words_doc_indexes {
-        delta_words_builder.insert(&word).unwrap();
-
-        let set = match index.postings_lists.postings_list(writer, &word)? {
-            Some(postings) => Union::new(&postings.matches, &delta_set).into_set_buf(),
-            None => delta_set,
-        };
-
-        index.postings_lists.put_postings_list(writer, &word, &set)?;
-    }
-
-    for (id, words) in indexed.docs_words {
-        index.docs_words.put_doc_words(writer, id, &words)?;
-    }
-
-    let delta_words = delta_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    let words = match index.main.words_fst(writer)? {
-        Some(words) => {
-            let op = OpBuilder::new()
-                .add(words.stream())
-                .add(delta_words.stream())
-                .r#union();
-
-            let mut words_builder = SetBuilder::memory();
-            words_builder.extend_stream(op).unwrap();
-            words_builder
-                .into_inner()
-                .and_then(fst::Set::from_bytes)
-                .unwrap()
-        }
-        None => delta_words,
-    };
-
-    index.main.put_words_fst(writer, &words)?;
-    index.main.put_ranked_map(writer, ranked_map)?;
-    index.main.put_number_of_documents(writer, |old| old + number_of_inserted_documents as u64)?;
-
-    compute_short_prefixes(writer, index)?;
-
-    Ok(())
-}

meilisearch-core/src/update/documents_deletion.rs

@@ -1,180 +0,0 @@
-use std::collections::{BTreeSet, HashMap, HashSet};
-
-use fst::{SetBuilder, Streamer};
-use meilisearch_schema::Schema;
-use sdset::{duo::DifferenceByKey, SetBuf, SetOperation};
-
-use crate::database::{MainT, UpdateT};
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::serde::extract_document_id;
-use crate::store;
-use crate::update::{next_update_id, compute_short_prefixes, Update};
-use crate::{DocumentId, Error, MResult, RankedMap};
-
-pub struct DocumentsDeletion {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: UpdateEventsEmitter,
-    documents: Vec<DocumentId>,
-}
-
-impl DocumentsDeletion {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: UpdateEventsEmitter,
-    ) -> DocumentsDeletion {
-        DocumentsDeletion {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            documents: Vec::new(),
-        }
-    }
-
-    pub fn delete_document_by_id(&mut self, document_id: DocumentId) {
-        self.documents.push(document_id);
-    }
-
-    pub fn delete_document<D>(&mut self, schema: &Schema, document: D) -> MResult<()>
-    where
-        D: serde::Serialize,
-    {
-        let primary_key = schema.primary_key().ok_or(Error::MissingPrimaryKey)?;
-        let document_id = match extract_document_id(&primary_key, &document)? {
-            Some(id) => id,
-            None => return Err(Error::MissingDocumentId),
-        };
-
-        self.delete_document_by_id(document_id);
-
-        Ok(())
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        let update_id = push_documents_deletion(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.documents,
-        )?;
-        Ok(update_id)
-    }
-}
-
-impl Extend<DocumentId> for DocumentsDeletion {
-    fn extend<T: IntoIterator<Item = DocumentId>>(&mut self, iter: T) {
-        self.documents.extend(iter)
-    }
-}
-
-pub fn push_documents_deletion(
-    writer: &mut heed::RwTxn<UpdateT>,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    deletion: Vec<DocumentId>,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::documents_deletion(deletion);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_documents_deletion(
-    writer: &mut heed::RwTxn<MainT>,
-    index: &store::Index,
-    deletion: Vec<DocumentId>,
-) -> MResult<()> {
-    let idset = SetBuf::from_dirty(deletion);
-
-    let schema = match index.main.schema(writer)? {
-        Some(schema) => schema,
-        None => return Err(Error::SchemaMissing),
-    };
-
-    let mut ranked_map = match index.main.ranked_map(writer)? {
-        Some(ranked_map) => ranked_map,
-        None => RankedMap::default(),
-    };
-
-    // collect the ranked attributes according to the schema
-    let ranked_fields = schema.ranked();
-
-    let mut words_document_ids = HashMap::new();
-    for id in idset {
-        // remove all the ranked attributes from the ranked_map
-        for ranked_attr in ranked_fields {
-            ranked_map.remove(id, *ranked_attr);
-        }
-
-        if let Some(words) = index.docs_words.doc_words(writer, id)? {
-            let mut stream = words.stream();
-            while let Some(word) = stream.next() {
-                let word = word.to_vec();
-                words_document_ids
-                    .entry(word)
-                    .or_insert_with(Vec::new)
-                    .push(id);
-            }
-        }
-    }
-
-    let mut deleted_documents = HashSet::new();
-    let mut removed_words = BTreeSet::new();
-    for (word, document_ids) in words_document_ids {
-        let document_ids = SetBuf::from_dirty(document_ids);
-
-        if let Some(postings) = index.postings_lists.postings_list(writer, &word)? {
-            let op = DifferenceByKey::new(&postings.matches, &document_ids, |d| d.document_id, |id| *id);
-            let doc_indexes = op.into_set_buf();
-
-            if !doc_indexes.is_empty() {
-                index.postings_lists.put_postings_list(writer, &word, &doc_indexes)?;
-            } else {
-                index.postings_lists.del_postings_list(writer, &word)?;
-                removed_words.insert(word);
-            }
-        }
-
-        for id in document_ids {
-            index.documents_fields_counts.del_all_document_fields_counts(writer, id)?;
-            if index.documents_fields.del_all_document_fields(writer, id)? != 0 {
-                deleted_documents.insert(id);
-            }
-        }
-    }
-
-    let deleted_documents_len = deleted_documents.len() as u64;
-    for id in deleted_documents {
-        index.docs_words.del_doc_words(writer, id)?;
-    }
-
-    let removed_words = fst::Set::from_iter(removed_words).unwrap();
-    let words = match index.main.words_fst(writer)? {
-        Some(words_set) => {
-            let op = fst::set::OpBuilder::new()
-                .add(words_set.stream())
-                .add(removed_words.stream())
-                .difference();
-
-            let mut words_builder = SetBuilder::memory();
-            words_builder.extend_stream(op).unwrap();
-            words_builder
-                .into_inner()
-                .and_then(fst::Set::from_bytes)
-                .unwrap()
-        }
-        None => fst::Set::default(),
-    };
-
-    index.main.put_words_fst(writer, &words)?;
-    index.main.put_ranked_map(writer, &ranked_map)?;
-    index.main.put_number_of_documents(writer, |old| old - deleted_documents_len)?;
-
-    compute_short_prefixes(writer, index)?;
-
-    Ok(())
-}

meilisearch-core/src/update/mod.rs

@@ -1,361 +0,0 @@
-mod clear_all;
-mod customs_update;
-mod documents_addition;
-mod documents_deletion;
-mod settings_update;
-
-pub use self::clear_all::{apply_clear_all, push_clear_all};
-pub use self::customs_update::{apply_customs_update, push_customs_update};
-pub use self::documents_addition::{
-    apply_documents_addition, apply_documents_partial_addition, DocumentsAddition,
-};
-pub use self::documents_deletion::{apply_documents_deletion, DocumentsDeletion};
-pub use self::settings_update::{apply_settings_update, push_settings_update};
-
-use std::cmp;
-use std::time::Instant;
-
-use chrono::{DateTime, Utc};
-use fst::{IntoStreamer, Streamer};
-use heed::Result as ZResult;
-use indexmap::IndexMap;
-use log::debug;
-use sdset::Set;
-use serde::{Deserialize, Serialize};
-
-use crate::{store, DocumentId, MResult};
-use crate::database::{MainT, UpdateT};
-use crate::settings::SettingsUpdate;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct Update {
-    data: UpdateData,
-    enqueued_at: DateTime<Utc>,
-}
-
-impl Update {
-    fn clear_all() -> Update {
-        Update {
-            data: UpdateData::ClearAll,
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn customs(data: Vec<u8>) -> Update {
-        Update {
-            data: UpdateData::Customs(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn documents_addition(data: Vec<IndexMap<String, serde_json::Value>>) -> Update {
-        Update {
-            data: UpdateData::DocumentsAddition(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn documents_partial(data: Vec<IndexMap<String, serde_json::Value>>) -> Update {
-        Update {
-            data: UpdateData::DocumentsPartial(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn documents_deletion(data: Vec<DocumentId>) -> Update {
-        Update {
-            data: UpdateData::DocumentsDeletion(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn settings(data: SettingsUpdate) -> Update {
-        Update {
-            data: UpdateData::Settings(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum UpdateData {
-    ClearAll,
-    Customs(Vec<u8>),
-    DocumentsAddition(Vec<IndexMap<String, serde_json::Value>>),
-    DocumentsPartial(Vec<IndexMap<String, serde_json::Value>>),
-    DocumentsDeletion(Vec<DocumentId>),
-    Settings(SettingsUpdate)
-}
-
-impl UpdateData {
-    pub fn update_type(&self) -> UpdateType {
-        match self {
-            UpdateData::ClearAll => UpdateType::ClearAll,
-            UpdateData::Customs(_) => UpdateType::Customs,
-            UpdateData::DocumentsAddition(addition) => UpdateType::DocumentsAddition {
-                number: addition.len(),
-            },
-            UpdateData::DocumentsPartial(addition) => UpdateType::DocumentsPartial {
-                number: addition.len(),
-            },
-            UpdateData::DocumentsDeletion(deletion) => UpdateType::DocumentsDeletion {
-                number: deletion.len(),
-            },
-            UpdateData::Settings(update) => UpdateType::Settings {
-                settings: update.clone(),
-            },
-        }
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(tag = "name")]
-pub enum UpdateType {
-    ClearAll,
-    Customs,
-    DocumentsAddition { number: usize },
-    DocumentsPartial { number: usize },
-    DocumentsDeletion { number: usize },
-    Settings { settings: SettingsUpdate },
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct ProcessedUpdateResult {
-    pub update_id: u64,
-    #[serde(rename = "type")]
-    pub update_type: UpdateType,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub error: Option<String>,
-    pub duration: f64, // in seconds
-    pub enqueued_at: DateTime<Utc>,
-    pub processed_at: DateTime<Utc>,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct EnqueuedUpdateResult {
-    pub update_id: u64,
-    #[serde(rename = "type")]
-    pub update_type: UpdateType,
-    pub enqueued_at: DateTime<Utc>,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase", tag = "status")]
-pub enum UpdateStatus {
-    Enqueued {
-        #[serde(flatten)]
-        content: EnqueuedUpdateResult,
-    },
-    Failed {
-        #[serde(flatten)]
-        content: ProcessedUpdateResult,
-    },
-    Processed {
-        #[serde(flatten)]
-        content: ProcessedUpdateResult,
-    },
-}
-
-pub fn update_status(
-    update_reader: &heed::RoTxn<UpdateT>,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    update_id: u64,
-) -> MResult<Option<UpdateStatus>> {
-    match updates_results_store.update_result(update_reader, update_id)? {
-        Some(result) => {
-            if result.error.is_some() {
-                Ok(Some(UpdateStatus::Failed { content: result }))
-            } else {
-                Ok(Some(UpdateStatus::Processed { content: result }))
-            }
-        },
-        None => match updates_store.get(update_reader, update_id)? {
-            Some(update) => Ok(Some(UpdateStatus::Enqueued {
-                content: EnqueuedUpdateResult {
-                    update_id,
-                    update_type: update.data.update_type(),
-                    enqueued_at: update.enqueued_at,
-                },
-            })),
-            None => Ok(None),
-        },
-    }
-}
-
-pub fn next_update_id(
-    update_writer: &mut heed::RwTxn<UpdateT>,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-) -> ZResult<u64> {
-    let last_update = updates_store.last_update(update_writer)?;
-    let last_update = last_update.map(|(n, _)| n);
-
-    let last_update_results_id = updates_results_store.last_update(update_writer)?;
-    let last_update_results_id = last_update_results_id.map(|(n, _)| n);
-
-    let max_update_id = cmp::max(last_update, last_update_results_id);
-    let new_update_id = max_update_id.map_or(0, |n| n + 1);
-
-    Ok(new_update_id)
-}
-
-pub fn update_task<'a, 'b>(
-    writer: &'a mut heed::RwTxn<'b, MainT>,
-    index: &store::Index,
-    update_id: u64,
-    update: Update,
-) -> MResult<ProcessedUpdateResult> {
-    debug!("Processing update number {}", update_id);
-
-    let Update { enqueued_at, data } = update;
-
-    let (update_type, result, duration) = match data {
-        UpdateData::ClearAll => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::ClearAll;
-            let result = apply_clear_all(writer, index);
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::Customs(customs) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::Customs;
-            let result = apply_customs_update(writer, index.main, &customs).map_err(Into::into);
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::DocumentsAddition(documents) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::DocumentsAddition {
-                number: documents.len(),
-            };
-
-            let result = apply_documents_addition(writer, index, documents);
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::DocumentsPartial(documents) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::DocumentsPartial {
-                number: documents.len(),
-            };
-
-            let result = apply_documents_partial_addition(writer, index, documents);
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::DocumentsDeletion(documents) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::DocumentsDeletion {
-                number: documents.len(),
-            };
-
-            let result = apply_documents_deletion(writer, index, documents);
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::Settings(settings) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::Settings {
-                settings: settings.clone(),
-            };
-
-            let result = apply_settings_update(
-                writer,
-                index,
-                settings,
-            );
-
-            (update_type, result, start.elapsed())
-        }
-    };
-
-    debug!(
-        "Processed update number {} {:?} {:?}",
-        update_id, update_type, result
-    );
-
-    let status = ProcessedUpdateResult {
-        update_id,
-        update_type,
-        error: result.map_err(|e| e.to_string()).err(),
-        duration: duration.as_secs_f64(),
-        enqueued_at,
-        processed_at: Utc::now(),
-    };
-
-    Ok(status)
-}
-
-fn compute_short_prefixes(writer: &mut heed::RwTxn<MainT>, index: &store::Index) -> MResult<()> {
-    // retrieve the words fst to compute all those prefixes
-    let words_fst = match index.main.words_fst(writer)? {
-        Some(fst) => fst,
-        None => return Ok(()),
-    };
-
-    // clear the prefixes
-    let pplc_store = index.prefix_postings_lists_cache;
-    pplc_store.clear(writer)?;
-
-    for prefix_len in 1..=2 {
-        // compute prefixes and store those in the PrefixPostingsListsCache store.
-        let mut previous_prefix: Option<([u8; 4], Vec<_>)> = None;
-        let mut stream = words_fst.into_stream();
-        while let Some(input) = stream.next() {
-
-            // We skip the prefixes that are shorter than the current length
-            // we want to cache (<). We must ignore the input when it is exactly the
-            // same word as the prefix because if we match exactly on it we need
-            // to consider it as an exact match and not as a prefix (=).
-            if input.len() <= prefix_len { continue }
-
-            if let Some(postings_list) = index.postings_lists.postings_list(writer, input)?.map(|p| p.matches.into_owned()) {
-                let prefix = &input[..prefix_len];
-
-                let mut arr_prefix = [0; 4];
-                arr_prefix[..prefix_len].copy_from_slice(prefix);
-
-                match previous_prefix {
-                    Some((ref mut prev_prefix, ref mut prev_pl)) if *prev_prefix != arr_prefix => {
-                        prev_pl.sort_unstable();
-                        prev_pl.dedup();
-
-                        if let Ok(prefix) = std::str::from_utf8(&prev_prefix[..prefix_len]) {
-                            debug!("writing the prefix of {:?} of length {}", prefix, prev_pl.len());
-                        }
-
-                        let pls = Set::new_unchecked(&prev_pl);
-                        pplc_store.put_prefix_postings_list(writer, *prev_prefix, &pls)?;
-
-                        *prev_prefix = arr_prefix;
-                        prev_pl.clear();
-                        prev_pl.extend_from_slice(&postings_list);
-                    },
-                    Some((_, ref mut prev_pl)) => prev_pl.extend_from_slice(&postings_list),
-                    None => previous_prefix = Some((arr_prefix, postings_list.to_vec())),
-                }
-            }
-        }
-
-        // write the last prefix postings lists
-        if let Some((prev_prefix, mut prev_pl)) = previous_prefix.take() {
-            prev_pl.sort_unstable();
-            prev_pl.dedup();
-
-            let pls = Set::new_unchecked(&prev_pl);
-            pplc_store.put_prefix_postings_list(writer, prev_prefix, &pls)?;
-        }
-    }
-
-    Ok(())
-}

meilisearch-core/src/update/settings_update.rs

@@ -1,300 +0,0 @@
-use std::collections::{BTreeMap, BTreeSet};
-
-use heed::Result as ZResult;
-use fst::{set::OpBuilder, SetBuilder};
-use sdset::SetBuf;
-use meilisearch_schema::Schema;
-
-use crate::database::{MainT, UpdateT};
-use crate::settings::{UpdateState, SettingsUpdate, RankingRule};
-use crate::update::documents_addition::reindex_all_documents;
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult, Error};
-
-pub fn push_settings_update(
-    writer: &mut heed::RwTxn<UpdateT>,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    settings: SettingsUpdate,
-) -> ZResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::settings(settings);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_settings_update(
-    writer: &mut heed::RwTxn<MainT>,
-    index: &store::Index,
-    settings: SettingsUpdate,
-) -> MResult<()> {
-    let mut must_reindex = false;
-
-    let mut schema = match index.main.schema(writer)? {
-        Some(schema) => schema,
-        None => {
-            match settings.primary_key.clone() {
-                UpdateState::Update(id) => Schema::with_primary_key(&id),
-                _ => return Err(Error::MissingPrimaryKey)
-            }
-        }
-    };
-
-    match settings.ranking_rules {
-        UpdateState::Update(v) => {
-            let ranked_field: Vec<&str> = v.iter().filter_map(RankingRule::field).collect();
-            schema.update_ranked(&ranked_field)?;
-            for name in ranked_field {
-                if schema.accept_new_fields() {
-                    schema.set_indexed(name.as_ref())?;
-                    schema.set_displayed(name.as_ref())?;
-                }
-            }
-            index.main.put_ranking_rules(writer, &v)?;
-            must_reindex = true;
-        },
-        UpdateState::Clear => {
-            index.main.delete_ranking_rules(writer)?;
-            schema.clear_ranked();
-            must_reindex = true;
-        },
-        UpdateState::Nothing => (),
-    }
-
-    match settings.distinct_attribute {
-        UpdateState::Update(v) => {
-            index.main.put_distinct_attribute(writer, &v)?;
-        },
-        UpdateState::Clear => {
-            index.main.delete_distinct_attribute(writer)?;
-        },
-        UpdateState::Nothing => (),
-    }
-
-    match settings.accept_new_fields {
-        UpdateState::Update(v) => {
-            schema.set_accept_new_fields(v);
-        },
-        UpdateState::Clear => {
-            schema.set_accept_new_fields(true);
-        },
-        UpdateState::Nothing => (),
-    }
-
-    match settings.searchable_attributes.clone() {
-        UpdateState::Update(v) => {
-            schema.update_indexed(v)?;
-            must_reindex = true;
-        },
-        UpdateState::Clear => {
-            schema.set_all_fields_as_indexed();
-            must_reindex = true;
-        },
-        UpdateState::Nothing => (),
-    }
-    match settings.displayed_attributes.clone() {
-        UpdateState::Update(v) => schema.update_displayed(v)?,
-        UpdateState::Clear => {
-            schema.set_all_fields_as_displayed();
-        },
-        UpdateState::Nothing => (),
-    }
-
-    index.main.put_schema(writer, &schema)?;
-
-    match settings.stop_words {
-        UpdateState::Update(stop_words) => {
-            if apply_stop_words_update(writer, index, stop_words)? {
-                must_reindex = true;
-            }
-        },
-        UpdateState::Clear => {
-            if apply_stop_words_update(writer, index, BTreeSet::new())? {
-                must_reindex = true;
-            }
-        },
-        UpdateState::Nothing => (),
-    }
-
-    match settings.synonyms {
-        UpdateState::Update(synonyms) => apply_synonyms_update(writer, index, synonyms)?,
-        UpdateState::Clear => apply_synonyms_update(writer, index, BTreeMap::new())?,
-        UpdateState::Nothing => (),
-    }
-
-    if must_reindex {
-        reindex_all_documents(writer, index)?;
-    }
-
-    Ok(())
-}
-
-pub fn apply_stop_words_update(
-    writer: &mut heed::RwTxn<MainT>,
-    index: &store::Index,
-    stop_words: BTreeSet<String>,
-) -> MResult<bool> {
-
-    let old_stop_words: BTreeSet<String> = index.main
-        .stop_words_fst(writer)?
-        .unwrap_or_default()
-        .stream()
-        .into_strs().unwrap().into_iter().collect();
-
-    let deletion: BTreeSet<String> = old_stop_words.difference(&stop_words).cloned().collect();
-    let addition: BTreeSet<String> = stop_words.difference(&old_stop_words).cloned().collect();
-
-    if !addition.is_empty() {
-        apply_stop_words_addition(
-            writer,
-            index,
-            addition
-        )?;
-    }
-
-    if !deletion.is_empty() {
-        apply_stop_words_deletion(
-            writer,
-            index,
-            deletion
-        )?;
-        return Ok(true)
-    }
-
-    let stop_words_fst = fst::Set::from_iter(stop_words)?;
-    index.main.put_words_fst(writer, &stop_words_fst)?;
-    Ok(false)
-}
-
-fn apply_stop_words_addition(
-    writer: &mut heed::RwTxn<MainT>,
-    index: &store::Index,
-    addition: BTreeSet<String>,
-) -> MResult<()> {
-
-    let main_store = index.main;
-    let postings_lists_store = index.postings_lists;
-
-    let mut stop_words_builder = SetBuilder::memory();
-
-    for word in addition {
-        stop_words_builder.insert(&word).unwrap();
-        // we remove every posting list associated to a new stop word
-        postings_lists_store.del_postings_list(writer, word.as_bytes())?;
-    }
-
-    // create the new delta stop words fst
-    let delta_stop_words = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    // we also need to remove all the stop words from the main fst
-    if let Some(word_fst) = main_store.words_fst(writer)? {
-        let op = OpBuilder::new()
-            .add(&word_fst)
-            .add(&delta_stop_words)
-            .difference();
-
-        let mut word_fst_builder = SetBuilder::memory();
-        word_fst_builder.extend_stream(op).unwrap();
-        let word_fst = word_fst_builder
-            .into_inner()
-            .and_then(fst::Set::from_bytes)
-            .unwrap();
-
-        main_store.put_words_fst(writer, &word_fst)?;
-    }
-
-    // now we add all of these stop words from the main store
-    let stop_words_fst = main_store.stop_words_fst(writer)?.unwrap_or_default();
-
-    let op = OpBuilder::new()
-        .add(&stop_words_fst)
-        .add(&delta_stop_words)
-        .r#union();
-
-    let mut stop_words_builder = SetBuilder::memory();
-    stop_words_builder.extend_stream(op).unwrap();
-    let stop_words_fst = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    main_store.put_stop_words_fst(writer, &stop_words_fst)?;
-
-    Ok(())
-}
-
-fn apply_stop_words_deletion(
-    writer: &mut heed::RwTxn<MainT>,
-    index: &store::Index,
-    deletion: BTreeSet<String>,
-) -> MResult<()> {
-
-    let mut stop_words_builder = SetBuilder::memory();
-
-    for word in deletion {
-        stop_words_builder.insert(&word).unwrap();
-    }
-
-    // create the new delta stop words fst
-    let delta_stop_words = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    // now we delete all of these stop words from the main store
-    let stop_words_fst = index.main.stop_words_fst(writer)?.unwrap_or_default();
-
-    let op = OpBuilder::new()
-        .add(&stop_words_fst)
-        .add(&delta_stop_words)
-        .difference();
-
-    let mut stop_words_builder = SetBuilder::memory();
-    stop_words_builder.extend_stream(op).unwrap();
-    let stop_words_fst = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    Ok(index.main.put_stop_words_fst(writer, &stop_words_fst)?)
-}
-
-pub fn apply_synonyms_update(
-    writer: &mut heed::RwTxn<MainT>,
-    index: &store::Index,
-    synonyms: BTreeMap<String, Vec<String>>,
-) -> MResult<()> {
-
-    let main_store = index.main;
-    let synonyms_store = index.synonyms;
-
-    let mut synonyms_builder = SetBuilder::memory();
-    synonyms_store.clear(writer)?;
-    for (word, alternatives) in synonyms.clone() {
-        synonyms_builder.insert(&word).unwrap();
-
-        let alternatives = {
-            let alternatives = SetBuf::from_dirty(alternatives);
-            let mut alternatives_builder = SetBuilder::memory();
-            alternatives_builder.extend_iter(alternatives).unwrap();
-            let bytes = alternatives_builder.into_inner().unwrap();
-            fst::Set::from_bytes(bytes).unwrap()
-        };
-
-        synonyms_store.put_synonyms(writer, word.as_bytes(), &alternatives)?;
-    }
-
-    let synonyms_set = synonyms_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    main_store.put_synonyms_fst(writer, &synonyms_set)?;
-
-    Ok(())
-}

meilisearch-error/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "meilisearch-error"
+version = "0.24.0"
+authors = ["marin <postma.marin@protonmail.com>"]
+edition = "2018"
+
+[dependencies]
+actix-http = "=3.0.0-beta.10"
+serde = { version = "1.0.130", features = ["derive"] }

meilisearch-error/src/lib.rs

@@ -0,0 +1,239 @@
+use std::fmt;
+
+use actix_http::http::StatusCode;
+use serde::{Deserialize, Serialize};
+
+pub trait ErrorCode: std::error::Error {
+    fn error_code(&self) -> Code;
+
+    /// returns the HTTP status code ascociated with the error
+    fn http_status(&self) -> StatusCode {
+        self.error_code().http()
+    }
+
+    /// returns the doc url ascociated with the error
+    fn error_url(&self) -> String {
+        self.error_code().url()
+    }
+
+    /// returns error name, used as error code
+    fn error_name(&self) -> String {
+        self.error_code().name()
+    }
+
+    /// return the error type
+    fn error_type(&self) -> String {
+        self.error_code().type_()
+    }
+}
+
+#[allow(clippy::enum_variant_names)]
+enum ErrorType {
+    InternalError,
+    InvalidRequestError,
+    AuthenticationError,
+}
+
+impl fmt::Display for ErrorType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use ErrorType::*;
+
+        match self {
+            InternalError => write!(f, "internal"),
+            InvalidRequestError => write!(f, "invalid_request"),
+            AuthenticationError => write!(f, "auth"),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, Copy)]
+pub enum Code {
+    // index related error
+    CreateIndex,
+    IndexAlreadyExists,
+    IndexNotFound,
+    InvalidIndexUid,
+
+    // invalid state error
+    InvalidState,
+    MissingPrimaryKey,
+    PrimaryKeyAlreadyPresent,
+
+    MaxFieldsLimitExceeded,
+    MissingDocumentId,
+    InvalidDocumentId,
+
+    Filter,
+    Sort,
+
+    BadParameter,
+    BadRequest,
+    DatabaseSizeLimitReached,
+    DocumentNotFound,
+    Internal,
+    InvalidGeoField,
+    InvalidRankingRule,
+    InvalidStore,
+    InvalidToken,
+    MissingAuthorizationHeader,
+    NoSpaceLeftOnDevice,
+    DumpNotFound,
+    TaskNotFound,
+    PayloadTooLarge,
+    RetrieveDocument,
+    SearchDocuments,
+    UnsupportedMediaType,
+
+    DumpAlreadyInProgress,
+    DumpProcessFailed,
+
+    InvalidContentType,
+    MissingContentType,
+    MalformedPayload,
+    MissingPayload,
+}
+
+impl Code {
+    /// ascociate a `Code` variant to the actual ErrCode
+    fn err_code(&self) -> ErrCode {
+        use Code::*;
+
+        match self {
+            // index related errors
+            // create index is thrown on internal error while creating an index.
+            CreateIndex => {
+                ErrCode::internal("index_creation_failed", StatusCode::INTERNAL_SERVER_ERROR)
+            }
+            IndexAlreadyExists => ErrCode::invalid("index_already_exists", StatusCode::CONFLICT),
+            // thrown when requesting an unexisting index
+            IndexNotFound => ErrCode::invalid("index_not_found", StatusCode::NOT_FOUND),
+            InvalidIndexUid => ErrCode::invalid("invalid_index_uid", StatusCode::BAD_REQUEST),
+
+            // invalid state error
+            InvalidState => ErrCode::internal("invalid_state", StatusCode::INTERNAL_SERVER_ERROR),
+            // thrown when no primary key has been set
+            MissingPrimaryKey => {
+                ErrCode::invalid("primary_key_inference_failed", StatusCode::BAD_REQUEST)
+            }
+            // error thrown when trying to set an already existing primary key
+            PrimaryKeyAlreadyPresent => {
+                ErrCode::invalid("index_primary_key_already_exists", StatusCode::BAD_REQUEST)
+            }
+            // invalid ranking rule
+            InvalidRankingRule => ErrCode::invalid("invalid_ranking_rule", StatusCode::BAD_REQUEST),
+
+            // invalid database
+            InvalidStore => {
+                ErrCode::internal("invalid_store_file", StatusCode::INTERNAL_SERVER_ERROR)
+            }
+
+            // invalid document
+            MaxFieldsLimitExceeded => {
+                ErrCode::invalid("max_fields_limit_exceeded", StatusCode::BAD_REQUEST)
+            }
+            MissingDocumentId => ErrCode::invalid("missing_document_id", StatusCode::BAD_REQUEST),
+            InvalidDocumentId => ErrCode::invalid("invalid_document_id", StatusCode::BAD_REQUEST),
+
+            // error related to filters
+            Filter => ErrCode::invalid("invalid_filter", StatusCode::BAD_REQUEST),
+            // error related to sorts
+            Sort => ErrCode::invalid("invalid_sort", StatusCode::BAD_REQUEST),
+
+            BadParameter => ErrCode::invalid("bad_parameter", StatusCode::BAD_REQUEST),
+            BadRequest => ErrCode::invalid("bad_request", StatusCode::BAD_REQUEST),
+            DatabaseSizeLimitReached => ErrCode::internal(
+                "database_size_limit_reached",
+                StatusCode::INTERNAL_SERVER_ERROR,
+            ),
+            DocumentNotFound => ErrCode::invalid("document_not_found", StatusCode::NOT_FOUND),
+            Internal => ErrCode::internal("internal", StatusCode::INTERNAL_SERVER_ERROR),
+            InvalidGeoField => ErrCode::invalid("invalid_geo_field", StatusCode::BAD_REQUEST),
+            InvalidToken => ErrCode::authentication("invalid_api_key", StatusCode::FORBIDDEN),
+            MissingAuthorizationHeader => {
+                ErrCode::authentication("missing_authorization_header", StatusCode::UNAUTHORIZED)
+            }
+            TaskNotFound => ErrCode::invalid("task_not_found", StatusCode::NOT_FOUND),
+            DumpNotFound => ErrCode::invalid("dump_not_found", StatusCode::NOT_FOUND),
+            NoSpaceLeftOnDevice => {
+                ErrCode::internal("no_space_left_on_device", StatusCode::INTERNAL_SERVER_ERROR)
+            }
+            PayloadTooLarge => ErrCode::invalid("payload_too_large", StatusCode::PAYLOAD_TOO_LARGE),
+            RetrieveDocument => {
+                ErrCode::internal("unretrievable_document", StatusCode::BAD_REQUEST)
+            }
+            SearchDocuments => ErrCode::internal("search_error", StatusCode::BAD_REQUEST),
+            UnsupportedMediaType => {
+                ErrCode::invalid("unsupported_media_type", StatusCode::UNSUPPORTED_MEDIA_TYPE)
+            }
+
+            // error related to dump
+            DumpAlreadyInProgress => {
+                ErrCode::invalid("dump_already_processing", StatusCode::CONFLICT)
+            }
+            DumpProcessFailed => {
+                ErrCode::internal("dump_process_failed", StatusCode::INTERNAL_SERVER_ERROR)
+            }
+            MissingContentType => {
+                ErrCode::invalid("missing_content_type", StatusCode::UNSUPPORTED_MEDIA_TYPE)
+            }
+            MalformedPayload => ErrCode::invalid("malformed_payload", StatusCode::BAD_REQUEST),
+            InvalidContentType => {
+                ErrCode::invalid("invalid_content_type", StatusCode::UNSUPPORTED_MEDIA_TYPE)
+            }
+            MissingPayload => ErrCode::invalid("missing_payload", StatusCode::BAD_REQUEST),
+        }
+    }
+
+    /// return the HTTP status code ascociated with the `Code`
+    fn http(&self) -> StatusCode {
+        self.err_code().status_code
+    }
+
+    /// return error name, used as error code
+    fn name(&self) -> String {
+        self.err_code().error_name.to_string()
+    }
+
+    /// return the error type
+    fn type_(&self) -> String {
+        self.err_code().error_type.to_string()
+    }
+
+    /// return the doc url ascociated with the error
+    fn url(&self) -> String {
+        format!("https://docs.meilisearch.com/errors#{}", self.name())
+    }
+}
+
+/// Internal structure providing a convenient way to create error codes
+struct ErrCode {
+    status_code: StatusCode,
+    error_type: ErrorType,
+    error_name: &'static str,
+}
+
+impl ErrCode {
+    fn authentication(error_name: &'static str, status_code: StatusCode) -> ErrCode {
+        ErrCode {
+            status_code,
+            error_name,
+            error_type: ErrorType::AuthenticationError,
+        }
+    }
+
+    fn internal(error_name: &'static str, status_code: StatusCode) -> ErrCode {
+        ErrCode {
+            status_code,
+            error_name,
+            error_type: ErrorType::InternalError,
+        }
+    }
+
+    fn invalid(error_name: &'static str, status_code: StatusCode) -> ErrCode {
+        ErrCode {
+            status_code,
+            error_name,
+            error_type: ErrorType::InvalidRequestError,
+        }
+    }
+}

meilisearch-http/Cargo.toml

@@ -1,58 +1,102 @@
 [package]
-name = "meilisearch-http"
+authors = ["Quentin de Quelen <quentin@dequelen.me>", "Clément Renault <clement@meilisearch.com>"]
 description = "MeiliSearch HTTP server"
-version = "0.9.0"
-license = "MIT"
-authors = [
-    "Quentin de Quelen <quentin@dequelen.me>",
-    "Clément Renault <clement@meilisearch.com>",
-]
 edition = "2018"
+license = "MIT"
+name = "meilisearch-http"
+version = "0.24.0"
 
 [[bin]]
 name = "meilisearch"
 path = "src/main.rs"
 
+[build-dependencies]
+actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "39d8006", optional = true }
+anyhow = { version = "1.0.43", optional = true }
+cargo_toml = { version = "0.9", optional = true }
+hex = { version = "0.4.3", optional = true }
+reqwest = { version = "0.11.4", features = ["blocking", "rustls-tls"], default-features = false, optional = true }
+sha-1 = { version = "0.9.8", optional = true }
+tempfile = { version = "3.2.0", optional = true }
+vergen = { version = "5.1.15", default-features = false, features = ["git"] }
+zip = { version = "0.5.13", optional = true }
+
 [dependencies]
-async-std = { version = "1.0.1", features = ["attributes"] }
-chrono = { version = "0.4.9", features = ["serde"] }
-crossbeam-channel = "0.4.0"
-env_logger = "0.7.1"
-futures = "0.3.1"
-heed = "0.6.1"
-http = "0.1.19"
-http-service = "0.4.0"
-indexmap = { version = "1.3.0", features = ["serde-1"] }
-log = "0.4.8"
-main_error = "0.1.0"
-meilisearch-core = { path = "../meilisearch-core", version = "0.9.0" }
-meilisearch-schema = { path = "../meilisearch-schema", version = "0.9.0" }
+actix-cors = { git = "https://github.com/MarinPostma/actix-extras.git", rev = "963ac94d" }
+actix-web = { version = "4.0.0-beta.9", features = ["rustls"] }
+actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "39d8006", optional = true }
+anyhow = { version = "1.0.43", features = ["backtrace"] }
+async-stream = "0.3.2"
+async-trait = "0.1.51"
+arc-swap = "1.3.2"
+byte-unit = { version = "4.0.12", default-features = false, features = ["std"] }
+bytes = "1.1.0"
+chrono = { version = "0.4.19", features = ["serde"] }
+crossbeam-channel = "0.5.1"
+either = "1.6.1"
+env_logger = "0.9.0"
+flate2 = "1.0.21"
+fst = "0.4.7"
+futures = "0.3.17"
+futures-util = "0.3.17"
+heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" }
+http = "0.2.4"
+indexmap = { version = "1.7.0", features = ["serde-1"] }
+itertools = "0.10.1"
+log = "0.4.14"
+meilisearch-lib = { path = "../meilisearch-lib" }
+meilisearch-error = { path = "../meilisearch-error" }
+meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.5" }
 mime = "0.3.16"
-pretty-bytes = "0.2.2"
-rand = "0.7.2"
-rayon = "1.2.0"
-serde = { version = "1.0.101", features = ["derive"] }
-serde_json = { version = "1.0.41", features = ["preserve_order"] }
-serde_qs = "0.5.1"
-siphasher = "0.3.1"
-structopt = "0.3.3"
-sysinfo = "0.9.5"
-tide = "0.6.0"
-ureq = { version = "0.11.2", features = ["tls"], default-features = false }
-walkdir = "2.2.9"
-whoami = "0.6"
-sha2 = "0.8.1"
+num_cpus = "1.13.0"
+once_cell = "1.8.0"
+parking_lot = "0.11.2"
+platform-dirs = "0.3.0"
+rand = "0.8.4"
+rayon = "1.5.1"
+regex = "1.5.4"
+rustls = "0.19.1"
+segment = { version = "0.1.2", optional = true }
+serde = { version = "1.0.130", features = ["derive"] }
+serde_json = { version = "1.0.67", features = ["preserve_order"] }
+sha2 = "0.9.6"
+siphasher = "0.3.7"
+slice-group-by = "0.2.6"
+structopt = "0.3.23"
+tar = "0.4.37"
+tempfile = "3.2.0"
+thiserror = "1.0.28"
+tokio = { version = "1.11.0", features = ["full"] }
+uuid = { version = "0.8.2", features = ["serde"] }
+walkdir = "2.3.2"
+obkv = "0.2.0"
+pin-project = "1.0.8"
+sysinfo = "0.20.2"
+tokio-stream = "0.1.7"
 
 [dev-dependencies]
-http-service-mock = "0.4.0"
-tempdir = "0.3.7"
+actix-rt = "2.2.0"
+paste = "1.0.5"
+serde_url_params = "0.2.1"
+urlencoding = "2.1.0"
 
-[dev-dependencies.assert-json-diff]
-git = "https://github.com/qdequele/assert-json-diff"
-branch = "master"
+[features]
+mini-dashboard = [
+    "actix-web-static-files",
+    "anyhow",
+    "cargo_toml",
+    "hex",
+    "reqwest",
+    "sha-1",
+    "tempfile",
+    "zip",
+]
+analytics = ["segment"]
+default = ["analytics", "mini-dashboard"]
 
-[build-dependencies]
-vergen = "3.0.4"
+[target.'cfg(target_os = "linux")'.dependencies]
+tikv-jemallocator = "0.4.1"
 
-[target.'cfg(unix)'.dependencies]
-jemallocator = "0.3.2"
+[package.metadata.mini-dashboard]
+assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.5/build.zip"
+sha1 = "1d955ea91b7691bd6fc207cb39866b82210783f0"

meilisearch-http/build.rs

@@ -1,10 +1,86 @@
-use vergen::{generate_cargo_keys, ConstantsFlags};
+use vergen::{vergen, Config};
 
 fn main() {
-    // Setup the flags, toggling off the 'SEMVER_FROM_CARGO_PKG' flag
-    let mut flags = ConstantsFlags::all();
-    flags.toggle(ConstantsFlags::SEMVER_FROM_CARGO_PKG);
+    if let Err(e) = vergen(Config::default()) {
+        println!("cargo:warning=vergen: {}", e);
+    }
 
-    // Generate the 'cargo:' key output
-    generate_cargo_keys(ConstantsFlags::all()).expect("Unable to generate the cargo keys!");
+    #[cfg(feature = "mini-dashboard")]
+    mini_dashboard::setup_mini_dashboard().expect("Could not load the mini-dashboard assets");
+}
+
+#[cfg(feature = "mini-dashboard")]
+mod mini_dashboard {
+    use std::env;
+    use std::fs::{create_dir_all, File, OpenOptions};
+    use std::io::{Cursor, Read, Write};
+    use std::path::PathBuf;
+
+    use actix_web_static_files::resource_dir;
+    use anyhow::Context;
+    use cargo_toml::Manifest;
+    use reqwest::blocking::get;
+    use sha1::{Digest, Sha1};
+
+    pub fn setup_mini_dashboard() -> anyhow::Result<()> {
+        let cargo_manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
+        let cargo_toml = cargo_manifest_dir.join("Cargo.toml");
+        let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
+
+        let sha1_path = out_dir.join(".mini-dashboard.sha1");
+        let dashboard_dir = out_dir.join("mini-dashboard");
+
+        let manifest = Manifest::from_path(cargo_toml).unwrap();
+
+        let meta = &manifest
+            .package
+            .as_ref()
+            .context("package not specified in Cargo.toml")?
+            .metadata
+            .as_ref()
+            .context("no metadata specified in Cargo.toml")?["mini-dashboard"];
+
+        // Check if there already is a dashboard built, and if it is up to date.
+        if sha1_path.exists() && dashboard_dir.exists() {
+            let mut sha1_file = File::open(&sha1_path)?;
+            let mut sha1 = String::new();
+            sha1_file.read_to_string(&mut sha1)?;
+            if sha1 == meta["sha1"].as_str().unwrap() {
+                // Nothing to do.
+                return Ok(());
+            }
+        }
+
+        let url = meta["assets-url"].as_str().unwrap();
+
+        let dashboard_assets_bytes = get(url)?.bytes()?;
+
+        let mut hasher = Sha1::new();
+        hasher.update(&dashboard_assets_bytes);
+        let sha1 = hex::encode(hasher.finalize());
+
+        assert_eq!(
+            meta["sha1"].as_str().unwrap(),
+            sha1,
+            "Downloaded mini-dashboard shasum differs from the one specified in the Cargo.toml"
+        );
+
+        create_dir_all(&dashboard_dir)?;
+        let cursor = Cursor::new(&dashboard_assets_bytes);
+        let mut zip = zip::read::ZipArchive::new(cursor)?;
+        zip.extract(&dashboard_dir)?;
+        resource_dir(&dashboard_dir).build()?;
+
+        // Write the sha1 for the dashboard back to file.
+        let mut file = OpenOptions::new()
+            .write(true)
+            .create(true)
+            .truncate(true)
+            .open(sha1_path)?;
+
+        file.write_all(sha1.as_bytes())?;
+        file.flush()?;
+
+        Ok(())
+    }
 }

meilisearch-http/public/interface.html

@@ -1,254 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <link rel="stylesheet" href="/bulma.min.css">
-    <title>MeiliSearch</title>
-    <style>
-      em {
-        color: hsl(204, 86%, 25%);
-        font-style: inherit;
-        background-color: hsl(204, 86%, 88%);
-      }
-
-      #results {
-        max-width: 900px;
-        margin: 20px auto 0 auto;
-        padding: 0;
-      }
-
-      .notification {
-        display: flex;
-        justify-content: center;
-      }
-
-      .level-left {
-        margin-right: 50px;
-      }
-
-      .document {
-        padding: 20px 20px;
-        background-color: #f5f5f5;
-        border-radius: 4px;
-        margin-bottom: 20px;
-        display: flex;
-      }
-
-      .document ol {
-        flex: 0 0 75%;
-        max-width: 75%;
-        padding: 0;
-        margin: 0;
-      }
-
-      .document .image {
-        max-width: 25%;
-        flex: 0 0 25%;
-        padding-left: 30px;
-        box-sizing: border-box;
-      }
-
-      .document .image img {
-        width: 100%;
-      }
-
-      .field {
-        list-style-type: none;
-        display: flex;
-        flex-wrap: wrap;
-      }
-
-      .field:not(:last-child) {
-        margin-bottom: 7px;
-      }
-
-      .attribute {
-        flex: 0 0 25%;
-        max-width: 25%;
-        text-align: right;
-        padding-right: 10px;
-        box-sizing: border-box;
-        text-transform: uppercase;
-        color: rgba(0,0,0,.7);
-      }
-
-      .content {
-        max-width: 75%;
-        flex: 0 0 75%;
-        box-sizing: border-box;
-        padding-left: 10px;
-        color: rgba(0,0,0,.9);
-      }
-    </style>
-  </head>
-  <body>
-
-    <section class="hero is-light">
-      <div class="hero-body">
-        <div class="container">
-          <h1 class="title">
-            Welcome to MeiliSearch
-          </h1>
-          <h2 class="subtitle">
-            This dashboard will help you check the search results with ease.
-          </h2>
-        </div>
-      </div>
-    </section>
-
-    <section class="hero container">
-        <div class="notification" style="border-radius: 0 0 4px 4px;">
-
-          <nav class="level">
-            <!-- Left side -->
-            <div class="level-left">
-              <div class="level-item">
-                <div class="field has-addons has-addons-right">
-                  <p class="control">
-                    <span class="select">
-                      <select id="index">
-                        <!-- indexes names -->
-                      </select>
-                    </span>
-                  </p>
-                  <p class="control">
-                    <input id="search" class="input" type="text" autofocus placeholder="e.g. George Clooney">
-                  </p>
-                </div>
-              </div>
-            </div>
-
-            <!-- Right side -->
-            <nav class="level-right">
-              <div class="level-item has-text-centered">
-                <div>
-                  <p class="heading">Documents</p>
-                  <p id="count" class="title">25</p>
-                </div>
-              </div>
-              <div class="level-item has-text-centered">
-                <div>
-                  <p class="heading">Time Spent</p>
-                  <p id="time" class="title">4ms</p>
-                </div>
-              </div>
-            </nav>
-          </nav>
-
-        </div>
-    </section>
-
-    <section>
-      <ol id="results" class="content">
-        <!-- documents matching resquests -->
-      </ol>
-    </section>
-  </body>
-
-  <script>
-    function httpGet(theUrl) {
-        var xmlHttp = new XMLHttpRequest();
-        xmlHttp.open("GET", theUrl, false); // false for synchronous request
-        xmlHttp.send(null);
-        return xmlHttp.responseText;
-    }
-
-    let lastRequest = undefined;
-
-    function triggerSearch() {
-        var e = document.getElementById("index");
-        if (e.selectedIndex == -1) { return }
-        var index = e.options[e.selectedIndex].value;
-
-        let theUrl = `${baseUrl}/indexes/${index}/search?q=${search.value}&attributesToHighlight=*`;
-
-        if (lastRequest) { lastRequest.abort() }
-        lastRequest = new XMLHttpRequest();
-
-        lastRequest.open("GET", theUrl, true);
-        lastRequest.onload = function (e) {
-            if (lastRequest.readyState === 4 && lastRequest.status === 200) {
-                let httpResults = JSON.parse(lastRequest.responseText);
-                results.innerHTML = '';
-
-                let processingTimeMs = httpResults.processingTimeMs;
-                let numberOfDocuments = httpResults.hits.length;
-                time.innerHTML = `${processingTimeMs}ms`;
-                count.innerHTML = `${numberOfDocuments}`;
-
-                for (result of httpResults.hits) {
-                    const element = {...result, ...result._formatted };
-                    delete element._formatted;
-
-                    const elem = document.createElement('li');
-                    elem.classList.add("document");
-
-                    const ol = document.createElement('ol');
-                    let image = undefined;
-
-                    for (const prop in element) {
-                        // Check if property is an image url link.
-                        if (typeof result[prop] === 'string') {
-                            if (image == undefined && result[prop].match(/^(https|http):\/\/.*(jpe?g|png|gif)(\?.*)?$/g)) {
-                                image = result[prop];
-                            }
-                        }
-
-                        const field = document.createElement('li');
-                        field.classList.add("field");
-
-                        const attribute = document.createElement('div');
-                        attribute.classList.add("attribute");
-                        attribute.innerHTML = prop;
-
-                        const content = document.createElement('div');
-                        content.classList.add("content");
-                        content.innerHTML = element[prop];
-
-                        field.appendChild(attribute);
-                        field.appendChild(content);
-
-                        ol.appendChild(field);
-                    }
-
-                    elem.appendChild(ol);
-
-                    if (image != undefined) {
-                        const div = document.createElement('div');
-                        div.classList.add("image");
-
-                        const img = document.createElement('img');
-                        img.src = image;
-
-                        div.appendChild(img);
-                        elem.appendChild(div);
-                    }
-
-                    results.appendChild(elem)
-                }
-            } else {
-                console.error(lastRequest.statusText);
-            }
-        };
-        lastRequest.send(null);
-    }
-
-    let baseUrl = window.location.origin;
-    // TODO we must not block here
-    let result = JSON.parse(httpGet(`${baseUrl}/indexes`));
-
-    let select = document.getElementById("index");
-    for (index of result) {
-        const option = document.createElement('option');
-        option.value = index.uid;
-        option.innerHTML = index.name;
-        select.appendChild(option);
-    }
-
-    search.oninput = triggerSearch;
-    select.onchange = triggerSearch;
-
-    triggerSearch();
-  </script>
-</html>

meilisearch-http/src/analytics.rs

@@ -1,69 +0,0 @@
-use std::hash::{Hash, Hasher};
-use std::thread;
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
-
-use log::error;
-use serde::Serialize;
-use serde_qs as qs;
-use siphasher::sip::SipHasher;
-
-const AMPLITUDE_API_KEY: &str = "f7fba398780e06d8fe6666a9be7e3d47";
-
-#[derive(Debug, Serialize)]
-struct Event<'a> {
-    user_id: &'a str,
-    event_type: &'a str,
-    device_id: &'a str,
-    time: u64,
-}
-
-#[derive(Debug, Serialize)]
-struct AmplitudeRequest<'a> {
-    api_key: &'a str,
-    event: &'a str,
-}
-
-pub fn analytics_sender() {
-    let username = whoami::username();
-    let hostname = whoami::hostname();
-    let platform = whoami::platform();
-
-    let uid = username + &hostname + &platform.to_string();
-
-    let mut hasher = SipHasher::new();
-    uid.hash(&mut hasher);
-    let hash = hasher.finish();
-
-    let uid = format!("{:X}", hash);
-    let platform = platform.to_string();
-
-    loop {
-        let n = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
-        let user_id = &uid;
-        let device_id = &platform;
-        let time = n.as_secs();
-        let event_type = "runtime_tick";
-
-        let event = Event {
-            user_id,
-            event_type,
-            device_id,
-            time,
-        };
-        let event = serde_json::to_string(&event).unwrap();
-
-        let request = AmplitudeRequest {
-            api_key: AMPLITUDE_API_KEY,
-            event: &event,
-        };
-
-        let body = qs::to_string(&request).unwrap();
-        let response = ureq::post("https://api.amplitude.com/httpapi").send_string(&body);
-        if !response.ok() {
-            let body = response.into_string().unwrap();
-            error!("Unsuccessful call to Amplitude: {}", body);
-        }
-
-        thread::sleep(Duration::from_secs(86_400)) // one day
-    }
-}

meilisearch-http/src/analytics/mock_analytics.rs

@@ -0,0 +1,51 @@
+use std::{any::Any, sync::Arc};
+
+use actix_web::HttpRequest;
+use serde_json::Value;
+
+use crate::{routes::indexes::documents::UpdateDocumentsQuery, Opt};
+
+use super::{find_user_id, Analytics};
+
+pub struct MockAnalytics;
+
+#[derive(Default)]
+pub struct SearchAggregator {}
+
+#[allow(dead_code)]
+impl SearchAggregator {
+    pub fn from_query(_: &dyn Any, _: &dyn Any) -> Self {
+        Self::default()
+    }
+
+    pub fn succeed(&mut self, _: &dyn Any) {}
+}
+
+impl MockAnalytics {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new(opt: &Opt) -> (Arc<dyn Analytics>, String) {
+        let user = find_user_id(&opt.db_path).unwrap_or_default();
+        (Arc::new(Self), user)
+    }
+}
+
+impl Analytics for MockAnalytics {
+    // These methods are noop and should be optimized out
+    fn publish(&self, _event_name: String, _send: Value, _request: Option<&HttpRequest>) {}
+    fn get_search(&self, _aggregate: super::SearchAggregator) {}
+    fn post_search(&self, _aggregate: super::SearchAggregator) {}
+    fn add_documents(
+        &self,
+        _documents_query: &UpdateDocumentsQuery,
+        _index_creation: bool,
+        _request: &HttpRequest,
+    ) {
+    }
+    fn update_documents(
+        &self,
+        _documents_query: &UpdateDocumentsQuery,
+        _index_creation: bool,
+        _request: &HttpRequest,
+    ) {
+    }
+}