Merge #3053

3053: Upgrade alpine 3.14 to 3.16 r=Kerollmops a=curquiza

Otherwise CI is failing https://github.com/meilisearch/meilisearch/actions/runs/3470576605/jobs/5799173168

Co-authored-by: curquiza <clementine@meilisearch.com>

.dockerignore

@@ -0,0 +1,4 @@
+target
+Dockerfile
+.dockerignore
+.gitignore

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,30 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Meilisearch version:** [e.g. v0.20.0]
+
+**Additional context**
+Additional information that may be relevant to the issue.
+[e.g. architecture, device, OS, browser]

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,13 @@
+contact_links:
+  - name: Language support request & feedback
+    url: https://github.com/meilisearch/product/discussions/categories/feedback-feature-proposal?discussions_q=label%3Aproduct%3Acore%3Atokenizer+category%3A%22Feedback+%26+Feature+Proposal%22
+    about:  The requests and feedback regarding Language support are not managed in this repository. Please upvote the related discussion in our dedicated product repository or open a new one if it doesn't exist.
+  - name: Feature request & feedback
+    url: https://github.com/meilisearch/product/discussions/categories/feedback-feature-proposal
+    about: The feature requests and feedback regarding the already existing features are not managed in this repository. Please open a discussion in our dedicated product repository
+  - name: Documentation issue
+    url: https://github.com/meilisearch/documentation/issues/new
+    about: For documentation issues, open an issue or a PR in the documentation repository
+  - name: Support questions & other
+    url: https://github.com/meilisearch/meilisearch/discussions/new
+    about: For any other question, open a discussion in this repository

.github/dependabot.yml

@@ -0,0 +1,13 @@
+# Set update schedule for GitHub Actions only
+
+version: 2
+updates:
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    labels:
+      - 'skip changelog'
+      - 'dependencies'
+    rebase-strategy: disabled

.github/scripts/check-release.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# check_tag $current_tag $file_tag $file_name
+function check_tag {
+  if [[ "$1" != "$2" ]]; then
+      echo "Error: the current tag does not match the version in $3: found $2 - expected $1"
+      ret=1
+  fi
+}
+
+ret=0
+current_tag=${GITHUB_REF#'refs/tags/v'}
+
+toml_files='*/Cargo.toml'
+for toml_file in $toml_files;
+do
+    file_tag="$(grep '^version = ' $toml_file | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')"
+    check_tag $current_tag $file_tag $toml_file
+done
+
+lock_file='Cargo.lock'
+lock_tag=$(grep -A 1 'name = "meilisearch-auth"' $lock_file | grep version | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')
+check_tag $current_tag $lock_tag $lock_file
+
+if [[ "$ret" -eq 0 ]] ; then
+  echo 'OK'
+fi
+exit $ret

.github/scripts/is-latest-release.sh

@@ -0,0 +1,132 @@
+#!/bin/sh
+
+# Was used in our CIs to publish the latest docker image. Not used anymore, will be used again when v1 and v2 will be out and we will want to maintain multiple stable versions.
+# Returns "true" or "false" (as a string) to be used in the `if` in GHA
+
+# Checks if the current tag should be the latest (in terms of semver and not of release date).
+# Ex: previous tag -> v2.1.1
+#     new tag -> v1.20.3
+#     The new tag (v1.20.3) should NOT be the latest
+#     So it returns "false", the `latest` tag should not be updated for the release v1.20.3 and still need to correspond to v2.1.1
+
+# GLOBAL
+GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number]
+
+# FUNCTIONS
+
+# semverParseInto and semverLT from https://github.com/cloudflare/semver_bash/blob/master/semver.sh
+
+# usage: semverParseInto version major minor patch special
+# version: the string version
+# major, minor, patch, special: will be assigned by the function
+semverParseInto() {
+    local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)'
+    #MAJOR
+    eval $2=`echo $1 | sed -e "s#$RE#\1#"`
+    #MINOR
+    eval $3=`echo $1 | sed -e "s#$RE#\2#"`
+    #MINOR
+    eval $4=`echo $1 | sed -e "s#$RE#\3#"`
+    #SPECIAL
+    eval $5=`echo $1 | sed -e "s#$RE#\4#"`
+}
+
+# usage: semverLT version1 version2
+semverLT() {
+    local MAJOR_A=0
+    local MINOR_A=0
+    local PATCH_A=0
+    local SPECIAL_A=0
+
+    local MAJOR_B=0
+    local MINOR_B=0
+    local PATCH_B=0
+    local SPECIAL_B=0
+
+    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
+    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
+
+    if [ $MAJOR_A -lt $MAJOR_B ]; then
+        return 0
+    fi
+    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -lt $MINOR_B ]; then
+        return 0
+    fi
+    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then
+        return 0
+    fi
+    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+        return 1
+    fi
+    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  != "_" ] ; then
+        return 1
+    fi
+    if [ "_$SPECIAL_A"  != "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+        return 0
+    fi
+    if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then
+        return 0
+    fi
+
+    return 1
+}
+
+# Returns the tag of the latest stable release (in terms of semver and not of release date)
+get_latest() {
+    temp_file='temp_file' # temp_file needed because the grep would start before the download is over
+    curl -s 'https://api.github.com/repos/meilisearch/meilisearch/releases' > "$temp_file"
+    releases=$(cat "$temp_file" | \
+        grep -E "tag_name|draft|prerelease" \
+        | tr -d ',"' | cut -d ':' -f2 | tr -d ' ')
+        # Returns a list of [tag_name draft_boolean prerelease_boolean ...]
+        # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false...
+
+    i=0
+    latest=""
+    current_tag=""
+    for release_info in $releases; do
+        if [ $i -eq 0 ]; then # Cheking tag_name
+            if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release
+                current_tag=$release_info
+            else
+                current_tag=""
+            fi
+            i=1
+        elif [ $i -eq 1 ]; then # Checking draft boolean
+            if [ "$release_info" = "true" ]; then
+                current_tag=""
+            fi
+            i=2
+        elif [ $i -eq 2 ]; then # Checking prerelease boolean
+            if [ "$release_info" = "true" ]; then
+                current_tag=""
+            fi
+            i=0
+            if [ "$current_tag" != "" ]; then # If the current_tag is valid
+                if [ "$latest" = "" ]; then # If there is no latest yet
+                    latest="$current_tag"
+                else
+                    semverLT $current_tag $latest # Comparing latest and the current tag
+                    if [ $? -eq 1 ]; then
+                        latest="$current_tag"
+                    fi
+                fi
+            fi
+        fi
+    done
+
+    rm -f "$temp_file"
+    echo $latest
+}
+
+# MAIN
+current_tag="$(echo $GITHUB_REF | tr -d 'refs/tags/')"
+latest="$(get_latest)"
+
+if [ "$current_tag" != "$latest" ]; then
+    # The current release tag is not the latest
+    echo "false"
+else
+    # The current release tag is the latest
+    echo "true"
+fi

.github/workflows/coverage.yml

@@ -0,0 +1,33 @@
+---
+on:
+  workflow_dispatch:
+
+name: Execute code coverage
+
+jobs:
+  nightly-coverage:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: clean
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --all-features --no-fail-fast
+        env:
+          CARGO_INCREMENTAL: "0"
+          RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=unwind -Zpanic_abort_tests"
+      - uses: actions-rs/grcov@v0.1
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ${{ steps.coverage.outputs.report }}
+          yml: ./codecov.yml
+          fail_ci_if_error: true

.github/workflows/create-issue-dependencies.yml

@@ -0,0 +1,23 @@
+name: Create issue to upgrade dependencies
+on:
+  schedule:
+    - cron: '0 0 1 */3 *'
+  workflow_dispatch:
+  
+jobs:
+  create-issue:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Create an issue
+      uses: actions-ecosystem/action-create-issue@v1
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        title: Upgrade dependencies
+        body: |
+          We need to update the dependencies of the Meilisearch repository, and, if possible, the dependencies of all the core-team repositories that Meilisearch depends on (milli, charabia, heed...).
+
+          ⚠️ This issue should only be done at the beginning of the sprint!     
+        labels: |
+          dependencies
+          maintenance

.github/workflows/flaky.yml

@@ -0,0 +1,15 @@
+name: Look for flaky tests
+on:
+  schedule:
+    - cron: "0 12 * * FRI" # every friday at 12:00PM
+
+jobs:
+  flaky:
+    runs-on: ubuntu-18.04
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install cargo-flaky
+      run: cargo install cargo-flaky
+    - name: Run cargo flaky 100 times
+      run: cargo flaky -i 100 --release

.github/workflows/publish-binaries.yml

@@ -0,0 +1,131 @@
+on:
+  release:
+    types: [published]
+
+name: Publish binaries to release
+
+jobs:
+  check-version:
+    name: Check the version validity
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      # Check if the tag has the v<nmumber>.<number>.<number> format.
+      # If yes, it means we are publishing an official release.
+      # If no, we are releasing a RC, so no need to check the version.
+      - name: Check tag format
+        if: github.event_name != 'schedule'
+        id: check-tag-format
+        run: |
+          escaped_tag=$(printf "%q" ${{ github.ref_name }})
+
+          if [[ $escaped_tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo ::set-output name=stable::true
+          else
+            echo ::set-output name=stable::false
+          fi
+      - name: Check release validity
+        if: steps.check-tag-format.outputs.stable == 'true'
+        run: bash .github/scripts/check-release.sh
+
+  publish:
+    name: Publish binary for ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    needs: check-version
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, macos-latest, windows-latest]
+        include:
+          - os: ubuntu-18.04
+            artifact_name: meilisearch
+            asset_name: meilisearch-linux-amd64
+          - os: macos-latest
+            artifact_name: meilisearch
+            asset_name: meilisearch-macos-amd64
+          - os: windows-latest
+            artifact_name: meilisearch.exe
+            asset_name: meilisearch-windows-amd64.exe
+
+    steps:
+    - uses: hecrj/setup-rust-action@master
+      with:
+        rust-version: stable
+    - uses: actions/checkout@v3
+    - name: Build
+      run: cargo build --release --locked
+    - name: Upload binaries to release
+      uses: svenstaro/upload-release-action@v1-release
+      with:
+        repo_token: ${{ secrets.PUBLISH_TOKEN }}
+        file: target/release/${{ matrix.artifact_name }}
+        asset_name: ${{ matrix.asset_name }}
+        tag: ${{ github.ref }}
+
+  publish-aarch64:
+    name: Publish binary for aarch64
+    runs-on: ${{ matrix.os }}
+    needs: check-version
+    continue-on-error: false
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - build: aarch64
+            os: ubuntu-18.04
+            target: aarch64-unknown-linux-gnu
+            linker: gcc-aarch64-linux-gnu
+            use-cross: true
+            asset_name: meilisearch-linux-aarch64
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Installing Rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+          target: ${{ matrix.target }}
+          override: true
+
+      - name: APT update
+        run: |
+          sudo apt update
+
+      - name: Install target specific tools
+        if: matrix.use-cross
+        run: |
+          sudo apt-get install -y ${{ matrix.linker }}
+
+      - name: Configure target aarch64 GNU
+        if: matrix.target == 'aarch64-unknown-linux-gnu'
+        ## Environment variable is not passed using env:
+        ## LD gold won't work with MUSL
+        # env:
+        #   JEMALLOC_SYS_WITH_LG_PAGE: 16
+        #   RUSTFLAGS: '-Clink-arg=-fuse-ld=gold'
+        run: |
+          echo '[target.aarch64-unknown-linux-gnu]' >> ~/.cargo/config
+          echo 'linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config
+          echo 'JEMALLOC_SYS_WITH_LG_PAGE=16' >> $GITHUB_ENV
+          echo RUSTFLAGS="-Clink-arg=-fuse-ld=gold" >> $GITHUB_ENV
+
+      - name: Cargo build
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          use-cross: ${{ matrix.use-cross }}
+          args: --release --target ${{ matrix.target }}
+
+      - name: List target output files
+        run: ls -lR ./target
+
+      - name: Upload the binary to release
+        uses: svenstaro/upload-release-action@v1-release
+        with:
+          repo_token: ${{ secrets.PUBLISH_TOKEN }}
+          file: target/${{ matrix.target }}/release/meilisearch
+          asset_name: ${{ matrix.asset_name }}
+          tag: ${{ github.ref }}

.github/workflows/publish-deb-brew-pkg.yml

@@ -0,0 +1,49 @@
+name: Publish deb pkg to GitHub release & APT repository & Homebrew
+
+on:
+  release:
+    types: [released]
+
+jobs:
+  check-version:
+    name: Check the version validity
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Check release validity
+        run: bash .github/scripts/check-release.sh
+
+  debian:
+    name: Publish debian packagge
+    runs-on: ubuntu-18.04
+    needs: check-version
+    steps:
+    - uses: hecrj/setup-rust-action@master
+      with:
+        rust-version: stable
+    - name: Install cargo-deb
+      run: cargo install cargo-deb
+    - uses: actions/checkout@v3
+    - name: Build deb package
+      run: cargo deb -p meilisearch-http -o target/debian/meilisearch.deb
+    - name: Upload debian pkg to release
+      uses: svenstaro/upload-release-action@v1-release
+      with:
+        repo_token: ${{ secrets.GITHUB_TOKEN }}
+        file: target/debian/meilisearch.deb
+        asset_name: meilisearch.deb
+        tag: ${{ github.ref }}
+    - name: Upload debian pkg to apt repository
+      run: curl -F package=@target/debian/meilisearch.deb https://${{ secrets.GEMFURY_PUSH_TOKEN }}@push.fury.io/meilisearch/
+
+  homebrew:
+    name: Bump Homebrew formula
+    runs-on: ubuntu-18.04
+    needs: check-version
+    steps:
+      - name: Create PR to Homebrew
+        uses: mislav/bump-homebrew-formula-action@v1
+        with:
+          formula-name: meilisearch
+        env:
+          COMMITTER_TOKEN: ${{ secrets.HOMEBREW_COMMITTER_TOKEN }}

.github/workflows/publish-docker-images.yml

@@ -0,0 +1,71 @@
+---
+on:
+  schedule:
+    - cron: '0 4 * * *' # Every day at 4:00am
+  push:
+    tags:
+      - '*'
+
+name: Publish tagged images to Docker Hub
+
+jobs:
+  docker:
+    runs-on: docker
+    steps:
+      - uses: actions/checkout@v2
+
+      # Check if the tag has the v<nmumber>.<number>.<number> format. If yes, it means we are publishing an official release.
+      # In this situation, we need to set `output.stable` to create/update the following tags (additionally to the `vX.Y.Z` Docker tag):
+      # - a `vX.Y` (without patch version) Docker tag
+      # - a `latest` Docker tag
+      - name: Check tag format
+        if: github.event_name != 'schedule'
+        id: check-tag-format
+        run: |
+          escaped_tag=$(printf "%q" ${{ github.ref_name }})
+
+          if [[ $escaped_tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo ::set-output name=stable::true
+          else
+            echo ::set-output name=stable::false
+          fi
+
+      # Check only the validity of the tag for official releases (not for pre-releases or other tags)
+      - name: Check release validity
+        if: github.event_name != 'schedule' && steps.check-tag-format.outputs.stable == 'true'
+        run: bash .github/scripts/check-release.sh
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Docker Hub
+        if: github.event_name != 'schedule'
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: getmeili/meilisearch
+          # The lastest and `vX.Y` tags are only pushed for the official Meilisearch releases
+          # See https://github.com/docker/metadata-action#latest-tag
+          flavor: latest=false
+          tags: |
+            type=ref,event=tag
+            type=semver,pattern=v{{major}}.{{minor}},enable=${{ steps.check-tag-format.outputs.stable == 'true' }}
+            type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable == 'true' }}
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v3
+        with:
+          # We do not push tags for the cron jobs, this is only for test purposes
+          push: ${{ github.event_name != 'schedule' }}
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.meta.outputs.tags }}

.github/workflows/rust.yml

@@ -0,0 +1,92 @@
+name: Rust
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    # trying and staging branches are for Bors config
+    branches:
+      - trying
+      - staging
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+  RUSTFLAGS: "-D warnings"
+
+jobs:
+  tests:
+    name: Tests on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, macos-latest, windows-latest]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Cache dependencies
+      uses: Swatinem/rust-cache@v2.0.0
+    - name: Run cargo check without any default features
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: --locked --release --no-default-features
+    - name: Run cargo test
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --locked --release
+
+  # We run tests in debug also, to make sure that the debug_assertions are hit
+  test-debug:
+    name: Run tests in debug
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v2.0.0
+      - name: Run tests in debug
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --locked
+
+  clippy:
+    name: Run Clippy
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          components: clippy
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v2.0.0
+      - name: Run cargo clippy
+        uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: --all-targets -- --deny warnings
+
+  fmt:
+    name: Run Rustfmt
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          components: rustfmt
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v2.0.0
+      - name: Run cargo fmt
+        run: cargo fmt --all -- --check

.gitignore

@@ -4,3 +4,6 @@
 **/*.rs.bk
 /*.mdb
 /query-history.txt
+/data.ms
+/snapshots
+/dumps

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at bonjour@meilisearch.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

CONTRIBUTING.md

@@ -0,0 +1,114 @@
+# Contributing
+
+First, thank you for contributing to Meilisearch! The goal of this document is to provide everything you need to start contributing to Meilisearch.
+
+Remember that there are many ways to contribute other than writing code: writing [tutorials or blog posts](https://github.com/meilisearch/awesome-meilisearch), improving [the documentation](https://github.com/meilisearch/documentation), submitting [bug reports](https://github.com/meilisearch/meilisearch/issues/new?assignees=&labels=&template=bug_report.md&title=) and [feature requests](https://github.com/meilisearch/product/discussions/categories/feedback-feature-proposal)...
+
+The code in this repository is only concerned with managing multiple indexes, handling the update store, and exposing an HTTP API. Search and indexation are the domain of our core engine, [`milli`](https://github.com/meilisearch/milli), while tokenization is handled by [our `charabia` library](https://github.com/meilisearch/charabia/).
+
+If Meilisearch does not offer optimized support for your language, please consider contributing to `charabia` by following the [CONTRIBUTING.md file](https://github.com/meilisearch/charabia/blob/main/CONTRIBUTING.md) and integrating your intended normalizer/segmenter.
+
+## Table of Contents
+
+- [Assumptions](#assumptions)
+- [How to Contribute](#how-to-contribute)
+- [Development Workflow](#development-workflow)
+- [Git Guidelines](#git-guidelines)
+- [Release Process (for internal team only)](#release-process-for-internal-team-only)
+
+## Assumptions
+
+1. **You're familiar with [GitHub](https://github.com) and the [Pull Requests](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests)(PR) workflow.**
+2. **You've read the Meilisearch [documentation](https://docs.meilisearch.com).**
+3. **You know about the [Meilisearch community](https://docs.meilisearch.com/learn/what_is_meilisearch/contact.html).
+   Please use this for help.**
+
+## How to Contribute
+
+1. Ensure your change has an issue! Find an
+   [existing issue](https://github.com/meilisearch/meilisearch/issues/) or [open a new issue](https://github.com/meilisearch/meilisearch/issues/new).
+   * This is where you can get a feel if the change will be accepted or not.
+2. Once approved, [fork the Meilisearch repository](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) in your own GitHub account.
+3. [Create a new Git branch](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-and-deleting-branches-within-your-repository)
+4. Review the [Development Workflow](#development-workflow) section that describes the steps to maintain the repository.
+5. Make your changes on your branch.
+6. [Submit the branch as a Pull Request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request-from-a-fork) pointing to the `main` branch of the Meilisearch repository. A maintainer should comment and/or review your Pull Request within a few days. Although depending on the circumstances, it may take longer.
+
+## Development Workflow
+
+### Setup and run Meilisearch
+
+```bash
+cargo run --release
+```
+
+We recommend using the `--release` flag to test the full performance of Meilisearch.
+
+### Test
+
+```bash
+cargo test
+```
+
+This command will be triggered to each PR as a requirement for merging it.
+
+If you get a "Too many open files" error you might want to increase the open file limit using this command:
+
+```bash
+ulimit -Sn 3000
+```
+
+## Git Guidelines
+
+### Git Branches
+
+All changes must be made in a branch and submitted as PR.
+
+We do not enforce any branch naming style, but please use something descriptive of your changes.
+
+### Git Commits
+
+As minimal requirements, your commit message should:
+- be capitalized
+- not finish by a dot or any other punctuation character (!,?)
+- start with a verb so that we can read your commit message this way: "This commit will ...", where "..." is the commit message.
+  e.g.: "Fix the home page button" or "Add more tests for create_index method"
+
+We don't follow any other convention, but if you want to use one, we recommend [the Chris Beams one](https://chris.beams.io/posts/git-commit/).
+
+### GitHub Pull Requests
+
+Some notes on GitHub PRs:
+
+- All PRs must be reviewed and approved by at least one maintainer.
+- The PR title should be accurate and descriptive of the changes.
+- [Convert your PR as a draft](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/changing-the-stage-of-a-pull-request) if your changes are a work in progress: no one will review it until you pass your PR as ready for review.<br>
+  The draft PRs are recommended when you want to show that you are working on something and make your work visible.
+- The branch related to the PR must be **up-to-date with `main`** before merging. Fortunately, this project uses [Bors](https://github.com/bors-ng/bors-ng) to automatically enforce this requirement without the PR author having to rebase manually.
+
+## Release Process (for internal team only)
+
+Meilisearch tools follow the [Semantic Versioning Convention](https://semver.org/).
+
+### Automation to rebase and Merge the PRs
+
+This project integrates a bot that helps us manage pull requests merging.<br>
+_[Read more about this](https://github.com/meilisearch/integration-guides/blob/main/resources/bors.md)._
+
+### How to Publish a new Release
+
+The full Meilisearch release process is described in [this guide](https://github.com/meilisearch/core-team/blob/main/resources/meilisearch-release.md). Please follow it carefully before doing any release.
+
+### Release assets
+
+For each release, the following assets are created:
+- Binaries for differents platforms (Linux, MacOS, Windows and ARM architectures) are attached to the GitHub release
+- Binaries are pushed to HomeBrew and APT (not published for RC)
+- Docker tags are created/updated:
+  - `vX.Y.Z`
+  - `vX.Y` (not published for RC)
+  - `latest` (not published for RC)
+
+<hr>
+
+Thank you again for reading this through, we can not wait to begin to work with you if you made your way through this contributing guide ❤️

Cargo.toml

@@ -1,11 +1,18 @@
 [workspace]
+resolver = "2"
 members = [
-    "meilidb-core",
-    "meilidb-http",
-    "meilidb-schema",
-    "meilidb-tokenizer",
-    "meilidb-types",
+    "meilisearch-http",
+    "meilisearch-types",
+    "meilisearch-lib",
+    "meilisearch-auth",
+    "permissive-json-pointer",
 ]
 
 [profile.release]
-debug = true
+codegen-units = 1
+
+[profile.dev.package.flate2]
+opt-level = 3
+
+[profile.dev.package.milli]
+opt-level = 3

Cross.toml

@@ -0,0 +1,7 @@
+[build.env]
+passthrough = [
+    "RUST_BACKTRACE",
+    "CARGO_TERM_COLOR",
+    "RUSTFLAGS",
+    "JEMALLOC_SYS_WITH_LG_PAGE"
+]

Dockerfile

@@ -0,0 +1,46 @@
+# Compile
+FROM    rust:alpine3.16 AS compiler
+
+RUN     apk add -q --update-cache --no-cache build-base openssl-dev
+
+WORKDIR /meilisearch
+
+ARG     COMMIT_SHA
+ARG     COMMIT_DATE
+ENV     COMMIT_SHA=${COMMIT_SHA} COMMIT_DATE=${COMMIT_DATE}
+ENV     RUSTFLAGS="-C target-feature=-crt-static"
+
+COPY    . .
+RUN     set -eux; \
+        apkArch="$(apk --print-arch)"; \
+        if [ "$apkArch" = "aarch64" ]; then \
+            export JEMALLOC_SYS_WITH_LG_PAGE=16; \
+        fi && \
+        cargo build --release
+
+# Run
+FROM    alpine:3.16
+
+ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
+ENV     MEILI_SERVER_PROVIDER docker
+
+RUN     apk update --quiet \
+        && apk add -q --no-cache libgcc tini curl
+
+# add meilisearch to the `/bin` so you can run it from anywhere and it's easy
+# to find.
+COPY    --from=compiler /meilisearch/target/release/meilisearch /bin/meilisearch
+# To stay compatible with the older version of the container (pre v0.27.0) we're
+# going to symlink the meilisearch binary in the path to `/meilisearch`
+RUN     ln -s /bin/meilisearch /meilisearch
+
+# This directory should hold all the data related to meilisearch so we're going
+# to move our PWD in there.
+# We don't want to put the meilisearch binary
+WORKDIR /meili_data
+
+
+EXPOSE  7700/tcp
+
+ENTRYPOINT ["tini", "--"]
+CMD     /bin/meilisearch

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) [year] [fullname]
+Copyright (c) 2019-2022 Meili SAS
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,160 +1,103 @@
-# MeiliDB
+<p align="center">
+  <img src="assets/meilisearch-logo-light.svg?sanitize=true#gh-light-mode-only">
+  <img src="assets/meilisearch-logo-dark.svg?sanitize=true#gh-dark-mode-only">
+</p>
 
-[![Build Status](https://dev.azure.com/thomas0884/thomas/_apis/build/status/meilisearch.MeiliDB?branchName=master)](https://dev.azure.com/thomas0884/thomas/_build/latest?definitionId=1&branchName=master)
-[![dependency status](https://deps.rs/repo/github/meilisearch/MeiliDB/status.svg)](https://deps.rs/repo/github/meilisearch/MeiliDB)
-[![License](https://img.shields.io/badge/license-commons%20clause-lightgrey)](https://commonsclause.com/)
+<h4 align="center">
+  <a href="https://www.meilisearch.com">Website</a> |
+  <a href="https://roadmap.meilisearch.com/tabs/1-under-consideration">Roadmap</a> |
+  <a href="https://blog.meilisearch.com">Blog</a> |
+  <a href="https://docs.meilisearch.com">Documentation</a> |
+  <a href="https://docs.meilisearch.com/faq/">FAQ</a> |
+  <a href="https://slack.meilisearch.com">Slack</a>
+</h4>
 
-Ultra relevant and instant full-text search API.
+<p align="center">
+  <a href="https://github.com/meilisearch/meilisearch/actions"><img src="https://github.com/meilisearch/meilisearch/workflows/Cargo%20test/badge.svg" alt="Build Status"></a>
+  <a href="https://deps.rs/repo/github/meilisearch/meilisearch"><img src="https://deps.rs/repo/github/meilisearch/meilisearch/status.svg" alt="Dependency status"></a>
+  <a href="https://github.com/meilisearch/meilisearch/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-informational" alt="License"></a>
+  <a href="https://app.bors.tech/repositories/26457"><img src="https://bors.tech/images/badge_small.svg" alt="Bors enabled"></a>
+</p>
 
-MeiliSearch is a powerful, fast, open-source, easy to use and deploy search engine. The search and indexation are fully customizable and handles features like typo-tolerance, filters, and ranking.
+<p align="center">⚡ A lightning-fast search engine that fits effortlessly into your apps, websites, and workflow 🔍</p>
 
-## Features
+Meilisearch helps you shape a delightful search experience in a snap, offering features that work out-of-the-box to speed up your workflow.
 
-- Provides [6 default ranking criteria](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-core/src/criterion/mod.rs#L107-L113) used to [bucket sort](https://en.wikipedia.org/wiki/Bucket_sort) documents
-- Accepts [custom criteria](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-core/src/criterion/mod.rs#L24-L33) and can apply them in any custom order
-- Support [ranged queries](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-core/src/query_builder.rs#L283), useful for paginating results
-- Can [distinct](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-core/src/query_builder.rs#L265-L270) and [filter](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-core/src/query_builder.rs#L246-L259) returned documents based on context defined rules
-- Searches for [concatenated](https://github.com/meilisearch/MeiliDB/pull/164) and [splitted query words](https://github.com/meilisearch/MeiliDB/pull/232) to improve the search quality.
-- Can store complete documents or only [user schema specified fields](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-schema/src/lib.rs#L265-L279)
-- The [default tokenizer](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-tokenizer/src/lib.rs) can index latin and kanji based languages
-- Returns [the matching text areas](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-core/src/lib.rs#L66-L88), useful to highlight matched words in results
-- Accepts query time search config like the [searchable attributes](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-core/src/query_builder.rs#L272-L275)
-- Supports [runtime incremental indexing](https://github.com/meilisearch/MeiliDB/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilidb-core/src/store/mod.rs#L143-L173)
+<p align="center" name="demo">
+  <a href="https://where2watch.meilisearch.com/#gh-light-mode-only" target="_blank">
+    <img src="assets/demo-light.gif#gh-light-mode-only" alt="A bright colored application for finding movies screening near the user">
+  </a>
+  <a href="https://where2watch.meilisearch.com/#gh-dark-mode-only" target="_blank">
+    <img src="assets/demo-dark.gif#gh-dark-mode-only" alt="A dark colored application for finding movies screening near the user">
+  </a>
+</p>
 
+🔥 [**Try it!**](https://where2watch.meilisearch.com/) 🔥
 
+## ✨ Features
 
-It uses [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database) as the internal key-value store. The key-value store allows us to handle updates and queries with small memory and CPU overheads. The whole ranking system is [data oriented](https://github.com/meilisearch/MeiliDB/issues/82) and provides great performances.
+- **Search-as-you-type:** find search results in less than 50 milliseconds
+- **[Typo tolerance](https://docs.meilisearch.com/learn/getting_started/customizing_relevancy.html#typo-tolerance):** get relevant matches even when queries contain typos and misspellings
+- **[Filtering and faceted search](https://docs.meilisearch.com/learn/advanced/filtering_and_faceted_search.html):** enhance your user's search experience with custom filters and build a faceted search interface in a few lines of code
+- **[Sorting](https://docs.meilisearch.com/learn/advanced/sorting.html):** sort results based on price, date, or pretty much anything else your users need
+- **[Synonym support](https://docs.meilisearch.com/learn/getting_started/customizing_relevancy.html#synonyms):** configure synonyms to include more relevant content in your search results
+- **[Geosearch](https://docs.meilisearch.com/learn/advanced/geosearch.html):** filter and sort documents based on geographic data
+- **[Extensive language support](https://docs.meilisearch.com/learn/what_is_meilisearch/language.html):** search datasets in any language, with optimized support for Chinese, Japanese, Hebrew, and languages using the Latin alphabet
+- **[Security management](https://docs.meilisearch.com/learn/security/master_api_keys.html):** control which users can access what data with API keys that allow fine-grained permissions handling
+- **[Multi-Tenancy](https://docs.meilisearch.com/learn/security/tenant_tokens.html):** personalize search results for any number of application tenants
+- **Highly Customizable:** customize Meilisearch to your specific needs or use our out-of-the-box and hassle-free presets
+- **[RESTful API](https://docs.meilisearch.com/reference/api/overview.html):** integrate Meilisearch in your technical stack with our plugins and SDKs
+- **Easy to install, deploy, and maintain**
 
-You can [read the deep dive](deep-dive.md) if you want more information on the engine, it describes the whole process of generating updates and handling queries or you can take a look at the [typos and ranking rules](typos-ranking-rules.md) if you want to know the default rules used to sort the documents.
+## 📖 Documentation
 
-We will be proud if you submit issues and pull requests. You can help to grow this project and start contributing by checking [issues tagged "good-first-issue"](https://github.com/meilisearch/MeiliDB/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22). It is a good start!
+You can consult Meilisearch's documentation at [https://docs.meilisearch.com](https://docs.meilisearch.com/).
 
-[![crates.io demo gif](misc/crates-io-demo.gif)](https://crates.meilisearch.com)
+## 🚀 Getting started
 
-> Meili helps the Rust community find crates on [crates.meilisearch.com](https://crates.meilisearch.com)
+For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [Quick Start](https://docs.meilisearch.com/learn/getting_started/quick_start.html) guide.
 
+You may also want to check out [Meilisearch 101](https://docs.meilisearch.com/learn/getting_started/filtering_and_sorting.html) for an introduction to some of Meilisearch's most popular features.
 
+## ☁️ Meilisearch cloud
 
-## Quick Start
+Join the closed beta for Meilisearch cloud by filling out [this form](https://meilisearch.typeform.com/to/VI2cI2rv).
 
-You can deploy your own instant, relevant and typo-tolerant MeiliDB search engine by yourself too.
-Something similar to the demo above can be achieve by following these little three steps first.
-You will need to create your own web front display to make it pretty though.
+## 🧰 SDKs & integration tools
 
-### Deploy the Server
+Install one of our SDKs in your project for seamless integration between Meilisearch and your favorite language or framework!
 
-If you have not installed Rust and its package manager `cargo` yet, go to [the installation page](https://www.rust-lang.org/tools/install).<br/>
-You can deploy the server on your own machine, it will listen to HTTP requests on the 8080 port by default.
+Take a look at the complete [Meilisearch integration list](https://docs.meilisearch.com/learn/what_is_meilisearch/sdks.html).
 
-```bash
-rustup override set nightly
-cargo run --release
-```
+![Logos belonging to different languages and frameworks supported by Meilisearch, including React, Ruby on Rails, Go, Rust, and PHP](assets/integrations.png)
 
-For more logs during the execution, run:
-```bash
-RUST_LOG=info cargo run --release
-```
+## ⚙️ Advanced usage
 
-### Create an Index and Upload Some Documents
+Experienced users will want to keep our [API Reference](https://docs.meilisearch.com/reference/api) close at hand.
 
-MeiliDB can serve multiple indexes, with different kinds of documents,
-therefore, it is required to create the index before sending documents to it.
+We also offer a wide range of dedicated guides to all Meilisearch features, such as [filtering](https://docs.meilisearch.com/learn/advanced/filtering_and_faceted_search.html), [sorting](https://docs.meilisearch.com/learn/advanced/sorting.html), [geosearch](https://docs.meilisearch.com/learn/advanced/geosearch.html), [API keys](https://docs.meilisearch.com/learn/security/master_api_keys.html), and [tenant tokens](https://docs.meilisearch.com/learn/security/tenant_tokens.html).
 
-```bash
-curl -i -X POST 'http://127.0.0.1:8080/indexes/movies'
-```
+Finally, for more in-depth information, refer to our articles explaining fundamental Meilisearch concepts such as [documents](https://docs.meilisearch.com/learn/core_concepts/documents.html) and [indexes](https://docs.meilisearch.com/learn/core_concepts/indexes.html).
 
-Now that the server knows about our brand new index, we can send it data.
-We provided you a little dataset, it is available in the `datasets/` directory.
+## 📊 Telemetry
 
-```bash
-curl -i -X POST 'http://127.0.0.1:8080/indexes/movies/documents' \
-  --header 'content-type: application/json' \
-  --data @datasets/movies/movies.json
-```
+Meilisearch collects **anonymized** data from users to help us improve our product. You can [deactivate this](https://docs.meilisearch.com/learn/what_is_meilisearch/telemetry.html#how-to-disable-data-collection) whenever you want.
 
-### Search for Documents
+To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Don't forget to include your `Instance UID` in the message, as this helps us quickly find and delete your data.
 
-The search engine is now aware of our documents and can serve those via our HTTP server again.
-The [`jq` command line tool](https://stedolan.github.io/jq/) can greatly help you read the server responses.
+If you want to know more about the kind of data we collect and what we use it for, check the [telemetry section](https://docs.meilisearch.com/learn/what_is_meilisearch/telemetry.html) of our documentation.
 
-```bash
-curl 'http://127.0.0.1:8080/indexes/movies/search?q=botman'
-```
+## 📫 Get in touch!
 
-```json
-{
-  "hits": [
-    {
-      "id": "29751",
-      "title": "Batman Unmasked: The Psychology of the Dark Knight",
-      "poster": "https://image.tmdb.org/t/p/w1280/jjHu128XLARc2k4cJrblAvZe0HE.jpg",
-      "overview": "Delve into the world of Batman and the vigilante justice tha",
-      "release_date": "2008-07-15"
-    },
-    {
-      "id": "471474",
-      "title": "Batman: Gotham by Gaslight",
-      "poster": "https://image.tmdb.org/t/p/w1280/7souLi5zqQCnpZVghaXv0Wowi0y.jpg",
-      "overview": "ve Victorian Age Gotham City, Batman begins his war on crime",
-      "release_date": "2018-01-12"
-    }
-  ],
-  "offset": 0,
-  "limit": 2,
-  "processingTimeMs": 1,
-  "query": "botman"
-}
-```
+Meilisearch is a search engine created by [Meili](https://www.welcometothejungle.com/en/companies/meilisearch), a software development company based in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/)
 
+🗞 [Subscribe to our newsletter](https://meilisearch.us2.list-manage.com/subscribe?u=27870f7b71c908a8b359599fb&id=79582d828e) if you don't want to miss any updates! We promise we won't clutter your mailbox: we only send one edition every two months.
 
+💌 Want to make a suggestion or give feedback? Here are some of the channels where you can reach us:
 
-## Performances
+- For feature requests, please visit our [product repository](https://github.com/meilisearch/product/discussions)
+- Found a bug? Open an [issue](https://github.com/meilisearch/meilisearch/issues)!
+- Want to be part of our Slack community? [Join us!](https://slack.meilisearch.com/)
+- For everything else, please check [this page listing some of the other places where you can find us](https://docs.meilisearch.com/learn/what_is_meilisearch/contact.html)
 
-With a dataset composed of _100 353_ documents with _352_ attributes each and _3_ of them indexed.
-So more than _300 000_ fields indexed for _35 million_ stored we can handle more than _2.8k req/sec_ with an average response time of _9 ms_ on an Intel i7-7700 (8) @ 4.2GHz.
-
-Requests are made using [wrk](https://github.com/wg/wrk) and scripted to simulate real users queries.
-
-```
-Running 10s test @ http://localhost:2230
-  2 threads and 25 connections
-  Thread Stats   Avg      Stdev     Max   +/- Stdev
-    Latency     9.52ms    7.61ms  99.25ms   84.58%
-    Req/Sec     1.41k   119.11     1.78k    64.50%
-  28080 requests in 10.01s, 7.42MB read
-Requests/sec:   2806.46
-Transfer/sec:    759.17KB
-```
-
-We also indexed a dataset containing something like _12 millions_ cities names in _24 minutes_ on a machine with _8 cores_, _64 GB of RAM_ and a _300 GB NMVe_ SSD.<br/>
-The resulting database was _16 GB_ and search results were between _30 ms_ and _4 seconds_ for short prefix queries.
-
-### Notes
-
-With Rust 1.32 the allocator has been [changed to use the system allocator](https://blog.rust-lang.org/2019/01/17/Rust-1.32.0.html#jemalloc-is-removed-by-default).
-We have seen much better performances when [using jemalloc as the global allocator](https://github.com/alexcrichton/jemallocator#documentation).
-
-## Usage and Examples
-
-MeiliDB also provides an example binary that is mostly used for features testing.
-Notice that the example binary is faster to index data as it does read direct CSV files and not JSON HTTP payloads.
-
-The _index_ subcommand has been made to create an index and inject documents into it. Using the command line below, the index will be named _movies_ and the _19 700_ movies of the `datasets/` will be injected in MeiliDB.
-
-```bash
-cargo run --release --example from_file -- \
-    index example.mdb datasets/movies/movies.csv \
-    --schema datasets/movies/schema.toml
-```
-
-Once the first command is done, you can query the freshly created _movies_ index using the _search_ subcomand. In this example we filtered the dataset to only show _non-adult_ movies using the non-definitive `!adult` syntax filter.
-
-```bash
-cargo run --release --example from_file -- \
-    search example.mdb \
-    --number-results 4 \
-    --filter '!adult' \
-    id popularity adult original_title
-```
+Thank you for your support!

SECURITY.md

@@ -0,0 +1,33 @@
+# Security
+
+Meilisearch takes the security of our software products and services seriously.
+
+If you believe you have found a security vulnerability in any Meilisearch-owned repository, please report it to us as described below.
+
+## Supported versions
+
+As long as we are pre-v1.0, only the latest version of Meilisearch will be supported with security updates.
+
+## Reporting security issues
+
+⚠️ Please do not report security vulnerabilities through public GitHub issues. ⚠️
+
+Instead, please kindly email us at security@meilisearch.com
+
+Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
+
+- Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+- Full paths of source file(s) related to the manifestation of the issue
+- The location of the affected source code (tag/branch/commit or direct URL)
+- Any special configuration required to reproduce the issue
+- Step-by-step instructions to reproduce the issue
+- Proof-of-concept or exploit code (if possible)
+- Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+You will receive a response from us within 72 hours. If the issue is confirmed, we will release a patch as soon as possible depending on complexity.
+
+## Preferred languages
+
+We prefer all communications to be in English.

assets/do-btn-blue.svg

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="200px" height="42px" viewBox="0 0 200 42" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 52.5 (67469) - http://www.bohemiancoding.com/sketch -->
+    <title>do-btn-blue</title>
+    <desc>Created with Sketch.</desc>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="Partner-welcome-kit-Copy-3" transform="translate(-651.000000, -762.000000)">
+            <g id="do-btn-blue" transform="translate(651.000000, 763.000000)">
+                <rect id="Rectangle-Copy" fill="#0069FF" x="0" y="0" width="200" height="40" rx="6"></rect>
+                <path d="M45,0 L45,40" id="Line-2" stroke="#FFFFFF" stroke-linecap="square"></path>
+                <g id="DO_Logo_horizontal_blue-Copy" transform="translate(13.000000, 11.000000)" fill="#FFFFFF">
+                    <path d="M10.0098493,20 L10.0098493,16.1262429 C14.12457,16.1262429 17.2897398,12.0548452 15.7269372,7.74627862 C15.1334679,6.14538921 13.8674,4.86072487 12.2650328,4.28756693 C7.952489,2.72620566 3.87733294,5.88845634 3.87733294,9.99938223 C3.87733294,9.99938223 3.87733294,9.99938223 3.87733294,9.99938223 L0,9.99938223 C0,3.45747613 6.3303395,-1.64165309 13.1948014,0.492866119 C16.2017127,1.42177726 18.57559,3.81322933 19.5053586,6.79760341 C21.6418482,13.6754986 16.5577943,20 10.0098493,20 Z" id="XMLID_49_"></path>
+                    <polygon id="XMLID_47_" points="9.56521739 15.6521739 6.08695652 15.6521739 6.08695652 12.173913 6.08695652 12.173913 9.56521739 12.173913 9.56521739 12.173913"></polygon>
+                    <polygon id="XMLID_46_" points="6.08695652 19.1304348 3.47826087 19.1304348 3.47826087 19.1304348 3.47826087 16.5217391 6.08695652 16.5217391"></polygon>
+                    <polygon id="XMLID_45_" points="3.47826087 16.5217391 0.869565217 16.5217391 0.869565217 16.5217391 0.869565217 13.9130435 0.869565217 13.9130435 3.47826087 13.9130435 3.47826087 13.9130435"></polygon>
+                </g>
+                <text id="Create-a-Droplet-Copy" font-family="Sailec-Medium, Sailec" font-size="16" font-weight="400" fill="#FFFFFF">
+                    <tspan x="58" y="26">Create a Droplet</tspan>
+                </text>
+            </g>
+        </g>
+    </g>
+</svg>

assets/logo.svg

@@ -0,0 +1,19 @@
+<svg width="300" height="300" viewBox="0 0 300 300" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M0 237L55.426 96.7678C63.2367 77.0063 82.499 64 103.955 64H137.371L81.9447 204.232C74.1341 223.993 54.8717 237 33.4156 237H0Z" fill="url(#paint0_linear_1_898)"/>
+<path d="M81.3123 237L136.738 96.7682C144.549 77.0067 163.811 64.0004 185.267 64.0004H218.683L163.257 204.232C155.446 223.994 136.184 237 114.728 237H81.3123Z" fill="url(#paint1_linear_1_898)"/>
+<path d="M162.629 237L218.055 96.7682C225.866 77.0067 245.128 64.0004 266.584 64.0004H300L244.574 204.232C236.763 223.994 217.501 237 196.045 237H162.629Z" fill="url(#paint2_linear_1_898)"/>
+<defs>
+<linearGradient id="paint0_linear_1_898" x1="300.001" y1="50.7858" x2="1.63474" y2="221.244" gradientUnits="userSpaceOnUse">
+<stop stop-color="#FF5CAA"/>
+<stop offset="1" stop-color="#FF4E62"/>
+</linearGradient>
+<linearGradient id="paint1_linear_1_898" x1="300.001" y1="50.7858" x2="1.63474" y2="221.244" gradientUnits="userSpaceOnUse">
+<stop stop-color="#FF5CAA"/>
+<stop offset="1" stop-color="#FF4E62"/>
+</linearGradient>
+<linearGradient id="paint2_linear_1_898" x1="300.001" y1="50.7858" x2="1.63474" y2="221.244" gradientUnits="userSpaceOnUse">
+<stop stop-color="#FF5CAA"/>
+<stop offset="1" stop-color="#FF4E62"/>
+</linearGradient>
+</defs>
+</svg>

assets/meilisearch-logo-dark.svg

@@ -0,0 +1,30 @@
+<svg width="495" height="74" viewBox="0 0 495 74" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M181.842 42.5349C181.842 37.6137 184.201 34.715 188.718 34.715C192.965 34.715 194.381 37.7486 194.381 41.6585V62.6238H203.953V40.5799C203.953 32.3556 199.639 26.4907 191.145 26.4907C186.089 26.4907 182.516 28.0412 179.415 31.4792C177.393 28.3782 173.955 26.4907 169.168 26.4907C164.112 26.4907 160.607 28.5805 158.989 31.614V27.2996H150.158V62.6238H159.731V42.3326C159.731 37.6137 162.157 34.715 166.607 34.715C170.854 34.715 172.269 37.7486 172.269 41.6585V62.6238H181.842V42.5349Z" fill="white"/>
+<path d="M243.245 47.7256C243.245 47.7256 243.379 46.4448 243.379 44.8943C243.379 34.4454 236.301 26.4907 225.852 26.4907C215.403 26.4907 208.123 34.4454 208.123 44.8943C208.123 55.7477 215.471 63.4327 225.92 63.4327C234.077 63.4327 240.548 58.5116 242.638 51.3659H232.998C231.852 53.9276 229.088 55.2084 226.189 55.2084C221.403 55.2084 218.302 52.5793 217.628 47.7256H243.245ZM225.785 34.1757C230.234 34.1757 233.133 36.8722 233.807 40.8495H217.763C218.572 36.8048 221.403 34.1757 225.785 34.1757Z" fill="white"/>
+<path d="M244.791 35.524H249.038V62.6238H258.61V27.2996H244.791V35.524ZM253.824 22.7156C257.195 22.7156 259.622 20.3561 259.622 16.9855C259.622 13.6149 257.195 11.188 253.824 11.188C250.454 11.188 248.027 13.6149 248.027 16.9855C248.027 20.3561 250.454 22.7156 253.824 22.7156Z" fill="white"/>
+<path d="M278.432 54.3995C278.163 54.3995 277.758 54.4669 277.152 54.4669C274.994 54.4669 274.725 53.4557 274.725 51.9726V12.0644H265.152V52.6467C265.152 59.6576 267.849 62.7586 275.466 62.7586C276.747 62.7586 277.96 62.6238 278.432 62.5564V54.3995Z" fill="white"/>
+<path d="M279.521 35.524H283.768V62.6238H293.341V27.2996H279.521V35.524ZM288.555 22.7156C291.925 22.7156 294.352 20.3561 294.352 16.9855C294.352 13.6149 291.925 11.188 288.555 11.188C285.184 11.188 282.757 13.6149 282.757 16.9855C282.757 20.3561 285.184 22.7156 288.555 22.7156Z" fill="white"/>
+<path d="M312.557 62.9937C321.86 62.9937 326.242 58.0726 326.242 52.8819C326.242 38.4556 305.007 46.4777 305.007 36.9725C305.007 33.8716 307.636 31.2425 312.962 31.2425C318.422 31.2425 320.984 34.2086 321.388 37.9163H326.175C325.77 33.2648 322.602 27.0629 313.097 27.0629C304.94 27.0629 300.356 31.9166 300.356 37.1748C300.356 51.264 321.591 43.1745 321.591 53.0167C321.591 56.4547 318.355 58.8142 312.557 58.8142C306.625 58.8142 303.659 55.848 303.322 51.4662H298.468C298.873 57.4659 302.648 62.9937 312.557 62.9937Z" fill="white"/>
+<path d="M364.256 46.4103C364.256 46.4103 364.324 45.3317 364.324 44.5901C364.324 34.8827 358.054 27.0629 347.808 27.0629C337.494 27.0629 330.955 35.4894 330.955 44.9946C330.955 54.6346 337.022 62.9937 347.875 62.9937C356.032 62.9937 361.695 58.0052 363.717 51.4662H358.729C357.245 55.6458 353.201 58.6794 347.943 58.6794C340.729 58.6794 336.213 53.3538 335.741 46.4103H364.256ZM347.808 31.3773C354.549 31.3773 358.931 35.8939 359.538 42.5004H335.876C336.685 36.1636 341.134 31.3773 347.808 31.3773Z" fill="white"/>
+<path d="M394.037 45.871V49.1068C394.037 54.9717 389.79 59.0164 381.634 59.0164C376.578 59.0164 373.814 56.9266 373.814 52.41C373.814 50.118 374.892 48.3652 376.578 47.4215C378.33 46.4777 380.69 45.871 394.037 45.871ZM381.094 62.9937C387.027 62.9937 391.813 61.1062 394.24 57.1963V62.1848H398.824V39.7364C398.824 32.1188 394.442 27.0629 384.532 27.0629C375.027 27.0629 370.848 31.8492 369.971 37.9837H374.623C375.566 33.13 379.274 31.1751 384.33 31.1751C390.802 31.1751 394.037 33.8716 394.037 39.669V41.8936C383.184 41.8936 378.667 42.0959 375.297 43.4441C371.387 44.9946 369.095 48.4327 369.095 52.5448C369.095 58.5445 372.937 62.9937 381.094 62.9937Z" fill="white"/>
+<path d="M424.991 27.6022C424.991 27.6022 424.182 27.5348 423.845 27.5348C417.509 27.5348 414.138 30.838 412.857 33.1974V27.8718H408.273V62.1848H413.059V42.7026C413.059 35.5569 417.441 32.0514 423.306 32.0514C424.182 32.0514 424.991 32.1188 424.991 32.1188V27.6022Z" fill="white"/>
+<path d="M425.809 45.062C425.809 54.4324 432.28 62.9937 442.729 62.9937C452.032 62.9937 457.425 56.7918 458.773 49.9831H453.92C452.504 55.3087 448.594 58.6794 442.729 58.6794C435.516 58.6794 430.662 52.9493 430.662 45.062C430.662 37.1073 435.516 31.3773 442.729 31.3773C448.594 31.3773 452.504 34.7479 453.92 40.0735H458.773C457.425 33.2648 452.032 27.0629 442.729 27.0629C432.28 27.0629 425.809 35.6243 425.809 45.062Z" fill="white"/>
+<path d="M470.041 11.6254H465.255V62.1848H470.041V41.8936C470.041 34.8827 474.558 31.2425 480.355 31.2425C486.49 31.2425 489.389 35.0176 489.389 41.2195V62.1848H494.175V40.2757C494.175 32.6581 489.658 27.0629 481.164 27.0629C474.76 27.0629 471.255 30.5683 470.041 32.6581V11.6254Z" fill="white"/>
+<path d="M0.825012 73.993L24.0688 14.5224C27.3443 6.14179 35.4223 0.625977 44.4203 0.625977H58.4336L35.1899 60.0966C31.9143 68.4772 23.8363 73.993 14.8384 73.993H0.825012Z" fill="url(#paint0_linear_0_3)"/>
+<path d="M34.9246 73.9932L58.1684 14.5226C61.444 6.14197 69.5219 0.626152 78.5199 0.626152H92.5333L69.2895 60.0968C66.014 68.4774 57.936 73.9932 48.938 73.9932H34.9246Z" fill="url(#paint1_linear_0_3)"/>
+<path d="M69.0262 73.9932L92.27 14.5226C95.5456 6.14197 103.624 0.626152 112.622 0.626152H126.635L103.391 60.0968C100.116 68.4774 92.0376 73.9932 83.0396 73.9932H69.0262Z" fill="url(#paint2_linear_0_3)"/>
+<defs>
+<linearGradient id="paint0_linear_0_3" x1="126.635" y1="-4.97799" x2="0.825008" y2="66.0978" gradientUnits="userSpaceOnUse">
+<stop stop-color="#FF5CAA"/>
+<stop offset="1" stop-color="#FF4E62"/>
+</linearGradient>
+<linearGradient id="paint1_linear_0_3" x1="126.635" y1="-4.97799" x2="0.825008" y2="66.0978" gradientUnits="userSpaceOnUse">
+<stop stop-color="#FF5CAA"/>
+<stop offset="1" stop-color="#FF4E62"/>
+</linearGradient>
+<linearGradient id="paint2_linear_0_3" x1="126.635" y1="-4.97799" x2="0.825008" y2="66.0978" gradientUnits="userSpaceOnUse">
+<stop stop-color="#FF5CAA"/>
+<stop offset="1" stop-color="#FF4E62"/>
+</linearGradient>
+</defs>
+</svg>

assets/meilisearch-logo-light.svg

@@ -0,0 +1,30 @@
+<svg width="495" height="74" viewBox="0 0 495 74" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M181.84 42.5347C181.84 37.6136 184.199 34.7149 188.716 34.7149C192.963 34.7149 194.378 37.7484 194.378 41.6584V62.6237H203.951V40.5798C203.951 32.3554 199.637 26.4906 191.143 26.4906C186.087 26.4906 182.514 28.041 179.413 31.4791C177.39 28.3781 173.952 26.4906 169.166 26.4906C164.11 26.4906 160.605 28.5804 158.987 31.6139V27.2995H150.156V62.6237H159.728V42.3325C159.728 37.6136 162.155 34.7149 166.604 34.7149C170.851 34.7149 172.267 37.7484 172.267 41.6584V62.6237H181.84V42.5347Z" fill="#21004B"/>
+<path d="M243.242 47.7255C243.242 47.7255 243.377 46.4447 243.377 44.8942C243.377 34.4452 236.299 26.4906 225.85 26.4906C215.401 26.4906 208.12 34.4452 208.12 44.8942C208.12 55.7476 215.468 63.4326 225.917 63.4326C234.074 63.4326 240.546 58.5115 242.636 51.3658H232.996C231.85 53.9274 229.086 55.2083 226.187 55.2083C221.401 55.2083 218.3 52.5792 217.626 47.7255H243.242ZM225.783 34.1756C230.232 34.1756 233.131 36.8721 233.805 40.8494H217.76C218.569 36.8047 221.401 34.1756 225.783 34.1756Z" fill="#21004B"/>
+<path d="M244.789 35.5238H249.036V62.6237H258.608V27.2995H244.789V35.5238ZM253.822 22.7155C257.193 22.7155 259.619 20.356 259.619 16.9854C259.619 13.6148 257.193 11.1879 253.822 11.1879C250.451 11.1879 248.024 13.6148 248.024 16.9854C248.024 20.356 250.451 22.7155 253.822 22.7155Z" fill="#21004B"/>
+<path d="M278.43 54.3993C278.16 54.3993 277.756 54.4667 277.149 54.4667C274.992 54.4667 274.722 53.4556 274.722 51.9725V12.0643H265.15V52.6466C265.15 59.6575 267.846 62.7585 275.464 62.7585C276.745 62.7585 277.958 62.6237 278.43 62.5562V54.3993Z" fill="#21004B"/>
+<path d="M279.519 35.5238H283.766V62.6237H293.339V27.2995H279.519V35.5238ZM288.553 22.7155C291.923 22.7155 294.35 20.356 294.35 16.9854C294.35 13.6148 291.923 11.1879 288.553 11.1879C285.182 11.1879 282.755 13.6148 282.755 16.9854C282.755 20.356 285.182 22.7155 288.553 22.7155Z" fill="#21004B"/>
+<path d="M312.557 62.9939C321.86 62.9939 326.242 58.0728 326.242 52.882C326.242 38.4557 305.007 46.4778 305.007 36.9726C305.007 33.8717 307.636 31.2426 312.962 31.2426C318.422 31.2426 320.984 34.2087 321.388 37.9164H326.175C325.77 33.265 322.602 27.063 313.097 27.063C304.94 27.063 300.356 31.9167 300.356 37.1749C300.356 51.2641 321.591 43.1746 321.591 53.0168C321.591 56.4548 318.355 58.8143 312.557 58.8143C306.625 58.8143 303.659 55.8481 303.322 51.4663H298.468C298.872 57.466 302.648 62.9939 312.557 62.9939Z" fill="#21004B"/>
+<path d="M364.256 46.4104C364.256 46.4104 364.324 45.3318 364.324 44.5903C364.324 34.8829 358.054 27.063 347.808 27.063C337.494 27.063 330.955 35.4896 330.955 44.9947C330.955 54.6347 337.022 62.9939 347.875 62.9939C356.032 62.9939 361.695 58.0053 363.717 51.4663H358.728C357.245 55.6459 353.201 58.6795 347.942 58.6795C340.729 58.6795 336.213 53.3539 335.741 46.4104H364.256ZM347.808 31.3774C354.549 31.3774 358.931 35.894 359.537 42.5005H335.876C336.685 36.1637 341.134 31.3774 347.808 31.3774Z" fill="#21004B"/>
+<path d="M394.037 45.8711V49.1069C394.037 54.9718 389.79 59.0165 381.633 59.0165C376.578 59.0165 373.814 56.9267 373.814 52.4101C373.814 50.1181 374.892 48.3654 376.578 47.4216C378.33 46.4778 380.69 45.8711 394.037 45.8711ZM381.094 62.9939C387.026 62.9939 391.813 61.1063 394.24 57.1964V62.1849H398.824V39.7366C398.824 32.1189 394.442 27.063 384.532 27.063C375.027 27.063 370.847 31.8493 369.971 37.9838H374.623C375.566 33.1301 379.274 31.1752 384.33 31.1752C390.802 31.1752 394.037 33.8717 394.037 39.6691V41.8938C383.184 41.8938 378.667 42.096 375.297 43.4442C371.387 44.9947 369.095 48.4328 369.095 52.5449C369.095 58.5446 372.937 62.9939 381.094 62.9939Z" fill="#21004B"/>
+<path d="M424.991 27.6023C424.991 27.6023 424.182 27.5349 423.845 27.5349C417.508 27.5349 414.138 30.8381 412.857 33.1975V27.872H408.273V62.1849H413.059V42.7027C413.059 35.557 417.441 32.0515 423.306 32.0515C424.182 32.0515 424.991 32.1189 424.991 32.1189V27.6023Z" fill="#21004B"/>
+<path d="M425.809 45.0621C425.809 54.4325 432.28 62.9939 442.729 62.9939C452.032 62.9939 457.425 56.7919 458.773 49.9832H453.92C452.504 55.3088 448.594 58.6795 442.729 58.6795C435.516 58.6795 430.662 52.9494 430.662 45.0621C430.662 37.1075 435.516 31.3774 442.729 31.3774C448.594 31.3774 452.504 34.748 453.92 40.0736H458.773C457.425 33.265 452.032 27.063 442.729 27.063C432.28 27.063 425.809 35.6244 425.809 45.0621Z" fill="#21004B"/>
+<path d="M470.041 11.6255H465.255V62.1849H470.041V41.8938C470.041 34.8829 474.558 31.2426 480.355 31.2426C486.49 31.2426 489.389 35.0177 489.389 41.2196V62.1849H494.175V40.2759C494.175 32.6582 489.658 27.063 481.164 27.063C474.76 27.063 471.255 30.5685 470.041 32.6582V11.6255Z" fill="#21004B"/>
+<path d="M0.824951 73.993L24.0688 14.5224C27.3443 6.14179 35.4223 0.625977 44.4202 0.625977H58.4336L35.1898 60.0966C31.9143 68.4772 23.8363 73.993 14.8383 73.993H0.824951Z" fill="url(#paint0_linear_0_15)"/>
+<path d="M34.9246 73.9932L58.1684 14.5226C61.4439 6.14197 69.5219 0.626152 78.5199 0.626152H92.5332L69.2894 60.0968C66.0139 68.4774 57.9359 73.9932 48.9379 73.9932H34.9246Z" fill="url(#paint1_linear_0_15)"/>
+<path d="M69.0262 73.9932L92.27 14.5226C95.5455 6.14197 103.623 0.626152 112.621 0.626152H126.635L103.391 60.0968C100.115 68.4774 92.0375 73.9932 83.0395 73.9932H69.0262Z" fill="url(#paint2_linear_0_15)"/>
+<defs>
+<linearGradient id="paint0_linear_0_15" x1="126.635" y1="-4.97799" x2="0.824952" y2="66.0978" gradientUnits="userSpaceOnUse">
+<stop stop-color="#FF5CAA"/>
+<stop offset="1" stop-color="#FF4E62"/>
+</linearGradient>
+<linearGradient id="paint1_linear_0_15" x1="126.635" y1="-4.97799" x2="0.824952" y2="66.0978" gradientUnits="userSpaceOnUse">
+<stop stop-color="#FF5CAA"/>
+<stop offset="1" stop-color="#FF4E62"/>
+</linearGradient>
+<linearGradient id="paint2_linear_0_15" x1="126.635" y1="-4.97799" x2="0.824952" y2="66.0978" gradientUnits="userSpaceOnUse">
+<stop stop-color="#FF5CAA"/>
+<stop offset="1" stop-color="#FF4E62"/>
+</linearGradient>
+</defs>
+</svg>

azure-pipelines.yml

@@ -1,52 +0,0 @@
----
-trigger:
-  branches:
-    include: [ master ]
-
-pr: [ master ]
-
-jobs:
-  - job: test
-    pool:
-      vmImage: 'Ubuntu 16.04'
-    container: tpayet/chiquitita:latest
-    steps:
-      - script: |
-          curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain nightly
-          $HOME/.cargo/bin/rustup component add rustfmt
-        displayName: 'Install rustc and components'
-      - script: |
-          $HOME/.cargo/bin/cargo check
-        displayName: 'Check MeiliDB'
-      - script: |
-          $HOME/.cargo/bin/cargo test
-        displayName: 'Test MeiliDB'
-      - script: |
-          $HOME/.cargo/bin/cargo fmt --all -- --check
-        displayName: 'Fmt MeiliDB'
-
-  - job: build
-    dependsOn:
-      - test
-    condition: succeeded()
-    pool:
-      vmImage: 'Ubuntu 16.04'
-    container: tpayet/chiquitita:latest
-    steps:
-      - script: |
-          curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain nightly
-          $HOME/.cargo/bin/rustup component add rustfmt
-        displayName: 'Install rustc and components'
-      - script: |
-          $HOME/.cargo/bin/cargo build --release
-        displayName: 'Build MeiliDB'
-      - task: CopyFiles@2
-        inputs:
-          contents: '$(System.DefaultWorkingDirectory)/target/release/meilidb-http'
-          targetFolder: $(Build.ArtifactStagingDirectory)
-        displayName: 'Copy build'
-      - task: PublishBuildArtifacts@1
-        inputs:
-          artifactName: meilidb
-        displayName: 'Upload artifacts'
-

bors.toml

@@ -0,0 +1,11 @@
+status = [
+    'Tests on ubuntu-18.04',
+    'Tests on macos-latest',
+    'Tests on windows-latest',
+    'Run Clippy',
+    'Run Rustfmt',
+    'Run tests in debug',
+]
+pr_status = ['Milestone Check']
+# 3 hours timeout
+timeout-sec = 10800

datasets/movies/README.md

@@ -1 +0,0 @@
-_datas in movies.csv are from https://www.themoviedb.org/_

datasets/movies/schema.toml

@@ -1,21 +0,0 @@
-# This schema has been generated ...
-# The order in which the attributes are declared is important,
-# it specify the attribute xxx...
-identifier = "id"
-
-[attributes.id]
-displayed = true
-
-[attributes.title]
-displayed = true
-indexed = true
-
-[attributes.overview]
-displayed = true
-indexed = true
-
-[attributes.release_date]
-displayed = true
-
-[attributes.poster]
-displayed = true

deep-dive.md

@@ -1,95 +0,0 @@
-# A deep dive in MeiliDB
-
-On the 15 of May 2019.
-
-MeiliDB is a full text search engine based on a final state transducer named [fst](https://github.com/BurntSushi/fst) and a key-value store named [sled](https://github.com/spacejam/sled). The goal of a search engine is to store data and to respond to queries as accurate and fast as possible. To achieve this it must save the matching words in an [inverted index](https://en.wikipedia.org/wiki/Inverted_index).
-
-<!-- MarkdownTOC autolink="true" -->
-
-- [Where is the data stored?](#where-is-the-data-stored)
-- [What does the key-value store contains?](#what-does-the-key-value-store-contains)
-    - [The inverted word index](#the-inverted-word-index)
-        - [A final state transducer](#a-final-state-transducer)
-        - [Document indexes](#document-indexes)
-    - [The schema](#the-schema)
-    - [Document attributes](#document-attributes)
-- [How is a request processed?](#how-is-a-request-processed)
-    - [Query lexemes](#query-lexemes)
-    - [Automatons and query index](#automatons-and-query-index)
-    - [Sort by criteria](#sort-by-criteria)
-
-<!-- /MarkdownTOC -->
-
-## Where is the data stored?
-
-MeiliDB is entirely backed by a key-value store like any good database (i.e. Postgres, MySQL). This brings a great flexibility in the way documents can be stored and updates handled along time.
-
-[sled will brings some](https://github.com/spacejam/sled/tree/434533332a3f485e6d2e467023be0a0b55d3a1af#plans) of the [A.C.I.D. properties](https://en.wikipedia.org/wiki/ACID_(computer_science)) to help us be sure the saved data is consistent.
-
-
-
-## What does the key-value store contains?
-
-It contain the inverted word index, the schema and the documents fields.
-
-### The inverted word index
-
-[The inverted word index](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-data/src/database/words_index.rs) is a sled Tree dedicated to store and give access to all documents that contains a specific word. The information stored under the word is simply a big ordered array of where in the document the word has been found. In other word, a big list of [`DocIndex`](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-core/src/lib.rs#L35-L51).
-
-#### A final state transducer
-
-_...also abbreviated fst_
-
-This is the first entry point of the engine, you can read more about how it work with the beautiful blog post of @BurntSushi, [Index 1,600,000,000 Keys with Automata and Rust](https://blog.burntsushi.net/transducers/).
-
-To make it short it is a powerful way to store all the words that are present in the indexed documents. You construct it by giving it all the words you want to index. When you want to search in it you can provide any automaton you want, in MeiliDB [a custom levenshtein automaton](https://github.com/tantivy-search/levenshtein-automata/) is used.
-
-#### Document indexes
-
-The `fst` will only return the words that match with the search automaton but the goal of the search engine is to retrieve all matches in all the documents when a query is made. You want it to return some sort of position in an attribute in a document, an information about where the given word matched.
-
-To make it possible we retrieve all of the `DocIndex` corresponding to all the matching words in the fst, we use the [`WordsIndex`](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-data/src/database/words_index.rs#L11-L21) Tree to get the `DocIndexes` corresponding the words.
-
-### The schema
-
-The schema is a data structure that represents which documents attributes should be stored and which should be indexed. It is stored under a the [`MainIndex`](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-data/src/database/main_index.rs#L12) Tree and given to MeiliDB only at the creation of an index.
-
-Each document attribute is associated to a unique 16 bit number named [`SchemaAttr`](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-data/src/schema.rs#L186).
-
-In the future, this schema type could be given along with updates, the database could be able to handled a new schema and reindex the database according to the new one.
-
-### Document attributes
-
-When the engine handle a query the result that the requester want is a document, not only the [`Matches`](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-core/src/lib.rs#L62-L88) associated to it, fields of the original document must be returned too.
-
-So MeiliDB again uses the power of the underlying key-value store and save the documents attributes marked as _STORE_ in the schema. The dedicated Tree for this information is the [`DocumentsIndex`](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-data/src/database/documents_index.rs#L11).
-
-When a document field is saved in the key-value store its value is binary encoded using [message pack](https://github.com/3Hren/msgpack-rust), so a document must be serializable using serde.
-
-
-
-## How is a request processed?
-
-Now that we have our inverted index we are able to return results based on a query. In the MeiliDB universe a query is a simple string containing words.
-
-### Query lexemes
-
-The first step to be able to call the underlying structures is to split the query in words, for that we use a [custom tokenizer](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-tokenizer/src/lib.rs#L82-L84). Note that a tokenizer is specialized for a human language, this is the hard part.
-
-### Automatons and query index
-
-So to query the fst we need an automaton, in MeiliDB we use a [levenshtein automaton](https://en.wikipedia.org/wiki/Levenshtein_automaton), this automaton is constructed using a string and a maximum distance. According to the [Algolia's blog post](https://blog.algolia.com/inside-the-algolia-engine-part-3-query-processing/#algolia%e2%80%99s-way-of-searching-for-alternatives) we [created the DFAs](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-core/src/automaton.rs#L59-L78) with different settings.
-
-Thanks to the power of the fst library [it is possible to union multiple automatons](https://docs.rs/fst/0.3.2/fst/map/struct.OpBuilder.html#method.union) on the same fst set. The `Stream` is able to return all the matching words. We use these words to find the whole list of `DocIndexes` associated.
-
-With all these informations it is possible [to reconstruct a list of all the `DocIndexes` associated](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-core/src/query_builder.rs#L103-L130) with the words queried.
-
-### Sort by criteria
-
-Now that we are able to get a big list of [DocIndexes](https://github.com/Kerollmops/MeiliDB/blob/550dc1e99224e386516877450320f694947332d4/src/lib.rs#L21-L36) it is not enough to sort them by criteria, we need more informations like the levenshtein distance or the fact that a query word match exactly the word stored in the fst. So [we stuff it a little bit](https://github.com/Kerollmops/MeiliDB/blob/550dc1e99224e386516877450320f694947332d4/src/rank/query_builder.rs#L86-L93), and aggregate all these [Matches](https://github.com/Kerollmops/MeiliDB/blob/550dc1e99224e386516877450320f694947332d4/src/lib.rs#L47-L74) for each document. This way it will be easy to sort a simple vector of document using a bunch of functions.
-
-With this big list of documents and associated matches [we are able to sort only the part of the slice that we want](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-core/src/query_builder.rs#L160-L188) using bucket sorting. [Each criterion](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-core/src/criterion/mod.rs#L95-L101) is evaluated on each subslice without copy, thanks to [GroupByMut](https://docs.rs/slice-group-by/0.2.4/slice_group_by/) which, I hope [will soon be merged](https://github.com/rust-lang/rfcs/pull/2477).
-
-Note that it is possible to customize the criteria used by using the `QueryBuilder::with_criteria` constructor, this way you can implement some custom ranking based on the document attributes using the appropriate structure and the [`document` method](https://github.com/meilisearch/MeiliDB/blob/3db823de002243004612e36a19b4578d800dab97/meilidb-data/src/database/index.rs#L86).
-
-At this point, MeiliDB work is over 🎉

download-latest.sh

@@ -0,0 +1,240 @@
+#!/bin/sh
+
+# COLORS
+RED='\033[31m'
+GREEN='\033[32m'
+DEFAULT='\033[0m'
+
+# GLOBALS
+GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number]
+
+# FUNCTIONS
+
+# semverParseInto and semverLT from https://github.com/cloudflare/semver_bash/blob/master/semver.sh
+
+# usage: semverParseInto version major minor patch special
+# version: the string version
+# major, minor, patch, special: will be assigned by the function
+semverParseInto() {
+    local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)'
+    #MAJOR
+    eval $2=`echo $1 | sed -e "s#$RE#\1#"`
+    #MINOR
+    eval $3=`echo $1 | sed -e "s#$RE#\2#"`
+    #PATCH
+    eval $4=`echo $1 | sed -e "s#$RE#\3#"`
+    #SPECIAL
+    eval $5=`echo $1 | sed -e "s#$RE#\4#"`
+}
+
+# usage: semverLT version1 version2
+semverLT() {
+    local MAJOR_A=0
+    local MINOR_A=0
+    local PATCH_A=0
+    local SPECIAL_A=0
+
+    local MAJOR_B=0
+    local MINOR_B=0
+    local PATCH_B=0
+    local SPECIAL_B=0
+
+    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
+    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
+
+    if [ $MAJOR_A -lt $MAJOR_B ]; then
+        return 0
+    fi
+    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -lt $MINOR_B ]; then
+        return 0
+    fi
+    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then
+        return 0
+    fi
+    if [ "_$SPECIAL_A"  == '_' ] && [ "_$SPECIAL_B"  == '_' ] ; then
+        return 1
+    fi
+    if [ "_$SPECIAL_A"  == '_' ] && [ "_$SPECIAL_B"  != '_' ] ; then
+        return 1
+    fi
+    if [ "_$SPECIAL_A"  != '_' ] && [ "_$SPECIAL_B"  == '_' ] ; then
+        return 0
+    fi
+    if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then
+        return 0
+    fi
+
+    return 1
+}
+
+# Get a token from https://github.com/settings/tokens to increase rate limit (from 60 to 5000), make sure the token scope is set to 'public_repo'
+# Create GITHUB_PAT environment variable once you acquired the token to start using it
+# Returns the tag of the latest stable release (in terms of semver and not of release date)
+get_latest() {
+    temp_file='temp_file' # temp_file needed because the grep would start before the download is over
+
+    if [ -z "$GITHUB_PAT" ]; then
+        curl -s 'https://api.github.com/repos/meilisearch/meilisearch/releases' > "$temp_file" || return 1
+    else
+        curl -H "Authorization: token $GITHUB_PAT" -s 'https://api.github.com/repos/meilisearch/meilisearch/releases' > "$temp_file" || return 1
+    fi
+
+    releases=$(cat "$temp_file" | \
+        grep -E '"tag_name":|"draft":|"prerelease":' \
+        | tr -d ',"' | cut -d ':' -f2 | tr -d ' ')
+        # Returns a list of [tag_name draft_boolean prerelease_boolean ...]
+        # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false...
+
+    i=0
+    latest=''
+    current_tag=''
+    for release_info in $releases; do
+        if [ $i -eq 0 ]; then # Checking tag_name
+            if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release
+                current_tag=$release_info
+            else
+                current_tag=''
+            fi
+            i=1
+        elif [ $i -eq 1 ]; then # Checking draft boolean
+            if [ "$release_info" = 'true' ]; then
+                current_tag=''
+            fi
+            i=2
+        elif [ $i -eq 2 ]; then # Checking prerelease boolean
+            if [ "$release_info" = 'true' ]; then
+                current_tag=''
+            fi
+            i=0
+            if [ "$current_tag" != '' ]; then # If the current_tag is valid
+                if [ "$latest" = '' ]; then # If there is no latest yet
+                    latest="$current_tag"
+                else
+                    semverLT $current_tag $latest # Comparing latest and the current tag
+                    if [ $? -eq 1 ]; then
+                        latest="$current_tag"
+                    fi
+                fi
+            fi
+        fi
+    done
+
+    rm -f "$temp_file"
+    return 0
+}
+
+# Gets the OS by setting the $os variable
+# Returns 0 in case of success, 1 otherwise.
+get_os() {
+    os_name=$(uname -s)
+    case "$os_name" in
+    'Darwin')
+        os='macos'
+        ;;
+    'Linux')
+        os='linux'
+        ;;
+	'MINGW'*)
+        os='windows'
+        ;;
+    *)
+        return 1
+    esac
+    return 0
+}
+
+# Gets the architecture by setting the $archi variable
+# Returns 0 in case of success, 1 otherwise.
+get_archi() {
+    architecture=$(uname -m)
+    case "$architecture" in
+    'x86_64' | 'amd64' )
+        archi='amd64'
+        ;;
+    'arm64')
+        if [ $os = 'macos' ]; then # MacOS M1
+            archi='amd64'
+        else
+            archi='aarch64'
+        fi
+        ;;
+    'aarch64')
+        archi='aarch64'
+        ;;
+    *)
+        return 1
+    esac
+    return 0
+}
+
+success_usage() {
+    printf "$GREEN%s\n$DEFAULT" "Meilisearch $latest binary successfully downloaded as '$binary_name' file."
+    echo ''
+    echo 'Run it:'
+    echo '    $ ./meilisearch'
+    echo 'Usage:'
+    echo '    $ ./meilisearch --help'
+}
+
+not_available_failure_usage() {
+    printf "$RED%s\n$DEFAULT" 'ERROR: Meilisearch binary is not available for your OS distribution or your architecture yet.'
+    echo ''
+    echo 'However, you can easily compile the binary from the source files.'
+    echo 'Follow the steps at the page ("Source" tab): https://docs.meilisearch.com/learn/getting_started/installation.html'
+}
+
+fetch_release_failure_usage() {
+    echo ''
+    printf "$RED%s\n$DEFAULT" 'ERROR: Impossible to get the latest stable version of Meilisearch.'
+    echo 'Please let us know about this issue: https://github.com/meilisearch/meilisearch/issues/new/choose'
+}
+
+# MAIN
+
+# Fill $latest variable
+if ! get_latest; then
+    fetch_release_failure_usage # TO CHANGE
+    exit 1
+fi
+
+if [ "$latest" = '' ]; then
+    fetch_release_failure_usage
+    exit 1
+fi
+
+# Fill $os variable
+if ! get_os; then
+    not_available_failure_usage
+    exit 1
+fi
+
+# Fill $archi variable
+if ! get_archi; then
+    not_available_failure_usage
+    exit 1
+fi
+
+echo "Downloading Meilisearch binary $latest for $os, architecture $archi..."
+case "$os" in
+    'windows')
+        release_file="meilisearch-$os-$archi.exe"
+        binary_name='meilisearch.exe'
+
+        ;;
+    *)
+        release_file="meilisearch-$os-$archi"
+        binary_name='meilisearch'
+
+esac
+
+# Fetch the Meilisearch binary
+link="https://github.com/meilisearch/meilisearch/releases/download/$latest/$release_file"
+curl --fail -OL "$link"
+if [ $? -ne 0 ]; then
+    fetch_release_failure_usage
+    exit 1
+fi
+
+mv "$release_file" "$binary_name"
+chmod 744 "$binary_name"
+success_usage

meilidb-core/Cargo.toml

@@ -1,40 +0,0 @@
-[package]
-name = "meilidb-core"
-version = "0.7.0"
-authors = ["Kerollmops <clement@meilisearch.com>"]
-edition = "2018"
-
-[dependencies]
-arc-swap = "0.4.3"
-bincode = "1.1.4"
-byteorder = "1.3.2"
-chrono = { version = "0.4.9", features = ["serde"] }
-crossbeam-channel = "0.4.0"
-deunicode = "1.0.0"
-env_logger = "0.7.0"
-fst = { version = "0.3.5", default-features = false }
-hashbrown = { version = "0.6.0", features = ["serde"] }
-heed = "0.5.0"
-levenshtein_automata = { version = "0.1.1", features = ["fst_automaton"] }
-log = "0.4.8"
-meilidb-schema = { path = "../meilidb-schema", version = "0.6.0" }
-meilidb-tokenizer = { path = "../meilidb-tokenizer", version = "0.6.0" }
-meilidb-types = { path = "../meilidb-types", version = "0.1.0" }
-once_cell = "1.2.0"
-ordered-float = { version = "1.0.2", features = ["serde"] }
-sdset = "0.3.3"
-serde = { version = "1.0.101", features = ["derive"] }
-serde_json = "1.0.41"
-siphasher = "0.3.0"
-slice-group-by = "0.2.6"
-zerocopy = "0.2.8"
-
-[dev-dependencies]
-assert_matches = "1.3"
-csv = "1.0.7"
-indexmap = { version = "1.2.0", features = ["serde-1"] }
-rustyline = { version = "5.0.0", default-features = false }
-structopt = "0.3.2"
-tempfile = "3.1.0"
-termcolor = "1.0.4"
-toml = "0.5.3"

meilidb-core/examples/from_file.rs

@@ -1,471 +0,0 @@
-use std::collections::btree_map::{BTreeMap, Entry};
-use std::collections::HashSet;
-use std::error::Error;
-use std::io::Write;
-use std::iter::FromIterator;
-use std::path::{Path, PathBuf};
-use std::time::{Duration, Instant};
-use std::{fs, io, sync::mpsc};
-
-use rustyline::{Config, Editor};
-use serde::{Deserialize, Serialize};
-use structopt::StructOpt;
-use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
-
-use meilidb_core::{Database, Highlight, ProcessedUpdateResult};
-use meilidb_schema::SchemaAttr;
-
-#[derive(Debug, StructOpt)]
-struct IndexCommand {
-    /// The destination where the database must be created.
-    #[structopt(parse(from_os_str))]
-    database_path: PathBuf,
-
-    #[structopt(long, default_value = "default")]
-    index_name: String,
-
-    /// The csv file to index.
-    #[structopt(parse(from_os_str))]
-    csv_data_path: PathBuf,
-
-    /// The path to the schema.
-    #[structopt(long, parse(from_os_str))]
-    schema: PathBuf,
-
-    #[structopt(long)]
-    update_group_size: Option<usize>,
-
-    #[structopt(long, parse(from_os_str))]
-    compact_to_path: Option<PathBuf>,
-}
-
-#[derive(Debug, StructOpt)]
-struct SearchCommand {
-    /// The path of the database to work with.
-    #[structopt(parse(from_os_str))]
-    database_path: PathBuf,
-
-    #[structopt(long, default_value = "default")]
-    index_name: String,
-
-    /// Timeout after which the search will return results.
-    #[structopt(long)]
-    fetch_timeout_ms: Option<u64>,
-
-    /// The number of returned results
-    #[structopt(short, long, default_value = "10")]
-    number_results: usize,
-
-    /// The number of characters before and after the first match
-    #[structopt(short = "C", long, default_value = "35")]
-    char_context: usize,
-
-    /// A filter string that can be `!adult` or `adult` to
-    /// filter documents on this specfied field
-    #[structopt(short, long)]
-    filter: Option<String>,
-
-    /// Fields that must be displayed.
-    displayed_fields: Vec<String>,
-}
-
-#[derive(Debug, StructOpt)]
-struct ShowUpdatesCommand {
-    /// The path of the database to work with.
-    #[structopt(parse(from_os_str))]
-    database_path: PathBuf,
-
-    #[structopt(long, default_value = "default")]
-    index_name: String,
-}
-
-#[derive(Debug, StructOpt)]
-enum Command {
-    Index(IndexCommand),
-    Search(SearchCommand),
-    ShowUpdates(ShowUpdatesCommand),
-}
-
-impl Command {
-    fn path(&self) -> &Path {
-        match self {
-            Command::Index(command) => &command.database_path,
-            Command::Search(command) => &command.database_path,
-            Command::ShowUpdates(command) => &command.database_path,
-        }
-    }
-}
-
-#[derive(Serialize, Deserialize)]
-#[serde(transparent)]
-struct Document(indexmap::IndexMap<String, String>);
-
-fn index_command(command: IndexCommand, database: Database) -> Result<(), Box<dyn Error>> {
-    let start = Instant::now();
-
-    let (sender, receiver) = mpsc::sync_channel(100);
-    let update_fn =
-        move |_name: &str, update: ProcessedUpdateResult| sender.send(update.update_id).unwrap();
-    let index = match database.open_index(&command.index_name) {
-        Some(index) => index,
-        None => database.create_index(&command.index_name).unwrap(),
-    };
-
-    database.set_update_callback(Box::new(update_fn));
-
-    let env = &database.env;
-
-    let schema = {
-        let string = fs::read_to_string(&command.schema)?;
-        toml::from_str(&string).unwrap()
-    };
-
-    let mut writer = env.write_txn().unwrap();
-    match index.main.schema(&writer)? {
-        Some(current_schema) => {
-            if current_schema != schema {
-                return Err(meilidb_core::Error::SchemaDiffer.into());
-            }
-            writer.abort();
-        }
-        None => {
-            index.schema_update(&mut writer, schema)?;
-            writer.commit().unwrap();
-        }
-    }
-
-    let mut rdr = csv::Reader::from_path(command.csv_data_path)?;
-    let mut raw_record = csv::StringRecord::new();
-    let headers = rdr.headers()?.clone();
-
-    let mut max_update_id = 0;
-    let mut i = 0;
-    let mut end_of_file = false;
-
-    while !end_of_file {
-        let mut additions = index.documents_addition();
-
-        loop {
-            end_of_file = !rdr.read_record(&mut raw_record)?;
-            if end_of_file {
-                break;
-            }
-
-            let document: Document = match raw_record.deserialize(Some(&headers)) {
-                Ok(document) => document,
-                Err(e) => {
-                    eprintln!("{:?}", e);
-                    continue;
-                }
-            };
-
-            additions.update_document(document);
-
-            print!("\rindexing document {}", i);
-            i += 1;
-
-            if let Some(group_size) = command.update_group_size {
-                if i % group_size == 0 {
-                    break;
-                }
-            }
-        }
-
-        println!();
-
-        let mut writer = env.write_txn().unwrap();
-        println!("committing update...");
-        let update_id = additions.finalize(&mut writer)?;
-        writer.commit().unwrap();
-        max_update_id = max_update_id.max(update_id);
-        println!("committed update {}", update_id);
-    }
-
-    println!("Waiting for update {}", max_update_id);
-    for id in receiver {
-        if id == max_update_id {
-            break;
-        }
-    }
-
-    println!(
-        "database created in {:.2?} at: {:?}",
-        start.elapsed(),
-        command.database_path
-    );
-
-    if let Some(path) = command.compact_to_path {
-        fs::create_dir_all(&path)?;
-        let start = Instant::now();
-        let _file = database.copy_and_compact_to_path(path.join("data.mdb"))?;
-        println!(
-            "database compacted in {:.2?} at: {:?}",
-            start.elapsed(),
-            path
-        );
-    }
-
-    Ok(())
-}
-
-fn display_highlights(text: &str, ranges: &[usize]) -> io::Result<()> {
-    let mut stdout = StandardStream::stdout(ColorChoice::Always);
-    let mut highlighted = false;
-
-    for range in ranges.windows(2) {
-        let [start, end] = match range {
-            [start, end] => [*start, *end],
-            _ => unreachable!(),
-        };
-        if highlighted {
-            stdout.set_color(
-                ColorSpec::new()
-                    .set_fg(Some(Color::Yellow))
-                    .set_underline(true),
-            )?;
-        }
-        write!(&mut stdout, "{}", &text[start..end])?;
-        stdout.reset()?;
-        highlighted = !highlighted;
-    }
-
-    Ok(())
-}
-
-fn char_to_byte_range(index: usize, length: usize, text: &str) -> (usize, usize) {
-    let mut byte_index = 0;
-    let mut byte_length = 0;
-
-    for (n, (i, c)) in text.char_indices().enumerate() {
-        if n == index {
-            byte_index = i;
-        }
-
-        if n + 1 == index + length {
-            byte_length = i - byte_index + c.len_utf8();
-            break;
-        }
-    }
-
-    (byte_index, byte_length)
-}
-
-fn create_highlight_areas(text: &str, highlights: &[Highlight]) -> Vec<usize> {
-    let mut byte_indexes = BTreeMap::new();
-
-    for highlight in highlights {
-        let char_index = highlight.char_index as usize;
-        let char_length = highlight.char_length as usize;
-        let (byte_index, byte_length) = char_to_byte_range(char_index, char_length, text);
-
-        match byte_indexes.entry(byte_index) {
-            Entry::Vacant(entry) => {
-                entry.insert(byte_length);
-            }
-            Entry::Occupied(mut entry) => {
-                if *entry.get() < byte_length {
-                    entry.insert(byte_length);
-                }
-            }
-        }
-    }
-
-    let mut title_areas = Vec::new();
-    title_areas.push(0);
-    for (byte_index, length) in byte_indexes {
-        title_areas.push(byte_index);
-        title_areas.push(byte_index + length);
-    }
-    title_areas.push(text.len());
-    title_areas.sort_unstable();
-    title_areas
-}
-
-/// note: matches must have been sorted by `char_index` and `char_length` before being passed.
-///
-/// ```no_run
-/// matches.sort_unstable_by_key(|m| (m.char_index, m.char_length));
-///
-/// let matches = matches.matches.iter().filter(|m| SchemaAttr::new(m.attribute) == attr).cloned();
-///
-/// let (text, matches) = crop_text(&text, matches, 35);
-/// ```
-fn crop_text(
-    text: &str,
-    highlights: impl IntoIterator<Item = Highlight>,
-    context: usize,
-) -> (String, Vec<Highlight>) {
-    let mut highlights = highlights.into_iter().peekable();
-
-    let char_index = highlights
-        .peek()
-        .map(|m| m.char_index as usize)
-        .unwrap_or(0);
-    let start = char_index.saturating_sub(context);
-    let text = text.chars().skip(start).take(context * 2).collect();
-
-    let highlights = highlights
-        .take_while(|m| (m.char_index as usize) + (m.char_length as usize) <= start + (context * 2))
-        .map(|highlight| Highlight {
-            char_index: highlight.char_index - start as u16,
-            ..highlight
-        })
-        .collect();
-
-    (text, highlights)
-}
-
-fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<dyn Error>> {
-    let env = &database.env;
-    let index = database
-        .open_index(&command.index_name)
-        .expect("Could not find index");
-
-    let reader = env.read_txn().unwrap();
-    let schema = index.main.schema(&reader)?;
-    reader.abort();
-
-    let schema = schema.ok_or(meilidb_core::Error::SchemaMissing)?;
-
-    let fields = command.displayed_fields.iter().map(String::as_str);
-    let fields = HashSet::from_iter(fields);
-
-    let config = Config::builder().auto_add_history(true).build();
-    let mut readline = Editor::<()>::with_config(config);
-    let _ = readline.load_history("query-history.txt");
-
-    for result in readline.iter("Searching for: ") {
-        match result {
-            Ok(query) => {
-                let start_total = Instant::now();
-
-                let reader = env.read_txn().unwrap();
-                let ref_index = &index;
-                let ref_reader = &reader;
-
-                let mut builder = index.query_builder();
-                if let Some(timeout) = command.fetch_timeout_ms {
-                    builder.with_fetch_timeout(Duration::from_millis(timeout));
-                }
-
-                if let Some(ref filter) = command.filter {
-                    let filter = filter.as_str();
-                    let (positive, filter) = if filter.chars().next() == Some('!') {
-                        (false, &filter[1..])
-                    } else {
-                        (true, filter)
-                    };
-
-                    let attr = schema
-                        .attribute(&filter)
-                        .expect("Could not find filtered attribute");
-
-                    builder.with_filter(move |document_id| {
-                        let string: String = ref_index
-                            .document_attribute(ref_reader, document_id, attr)
-                            .unwrap()
-                            .unwrap();
-                        (string == "true") == positive
-                    });
-                }
-
-                let documents = builder.query(ref_reader, &query, 0..command.number_results)?;
-
-                let mut retrieve_duration = Duration::default();
-
-                let number_of_documents = documents.len();
-                for mut doc in documents {
-                    doc.highlights
-                        .sort_unstable_by_key(|m| (m.char_index, m.char_length));
-
-                    let start_retrieve = Instant::now();
-                    let result = index.document::<Document>(&reader, Some(&fields), doc.id);
-                    retrieve_duration += start_retrieve.elapsed();
-
-                    match result {
-                        Ok(Some(document)) => {
-                            println!("raw-id: {:?}", doc.id);
-                            for (name, text) in document.0 {
-                                print!("{}: ", name);
-
-                                let attr = schema.attribute(&name).unwrap();
-                                let highlights = doc
-                                    .highlights
-                                    .iter()
-                                    .filter(|m| SchemaAttr::new(m.attribute) == attr)
-                                    .cloned();
-                                let (text, highlights) =
-                                    crop_text(&text, highlights, command.char_context);
-                                let areas = create_highlight_areas(&text, &highlights);
-                                display_highlights(&text, &areas)?;
-                                println!();
-                            }
-                        }
-                        Ok(None) => eprintln!("missing document"),
-                        Err(e) => eprintln!("{}", e),
-                    }
-
-                    let mut matching_attributes = HashSet::new();
-                    for highlight in doc.highlights {
-                        let attr = SchemaAttr::new(highlight.attribute);
-                        let name = schema.attribute_name(attr);
-                        matching_attributes.insert(name);
-                    }
-
-                    let matching_attributes = Vec::from_iter(matching_attributes);
-                    println!("matching in: {:?}", matching_attributes);
-
-                    println!();
-                }
-
-                eprintln!(
-                    "whole documents fields retrieve took {:.2?}",
-                    retrieve_duration
-                );
-                eprintln!(
-                    "===== Found {} results in {:.2?} =====",
-                    number_of_documents,
-                    start_total.elapsed()
-                );
-            }
-            Err(err) => {
-                println!("Error: {:?}", err);
-                break;
-            }
-        }
-    }
-
-    readline.save_history("query-history.txt").unwrap();
-
-    Ok(())
-}
-
-fn show_updates_command(
-    command: ShowUpdatesCommand,
-    database: Database,
-) -> Result<(), Box<dyn Error>> {
-    let env = &database.env;
-    let index = database
-        .open_index(&command.index_name)
-        .expect("Could not find index");
-
-    let reader = env.read_txn().unwrap();
-    let updates = index.all_updates_status(&reader)?;
-    println!("{:#?}", updates);
-    reader.abort();
-
-    Ok(())
-}
-
-fn main() -> Result<(), Box<dyn Error>> {
-    env_logger::init();
-
-    let opt = Command::from_args();
-    let database = Database::open_or_create(opt.path())?;
-
-    match opt {
-        Command::Index(command) => index_command(command, database),
-        Command::Search(command) => search_command(command, database),
-        Command::ShowUpdates(command) => show_updates_command(command, database),
-    }
-}

meilidb-core/src/automaton/dfa.rs

@@ -1,48 +0,0 @@
-use levenshtein_automata::{LevenshteinAutomatonBuilder as LevBuilder, DFA};
-use once_cell::sync::OnceCell;
-
-static LEVDIST0: OnceCell<LevBuilder> = OnceCell::new();
-static LEVDIST1: OnceCell<LevBuilder> = OnceCell::new();
-static LEVDIST2: OnceCell<LevBuilder> = OnceCell::new();
-
-#[derive(Copy, Clone)]
-enum PrefixSetting {
-    Prefix,
-    NoPrefix,
-}
-
-fn build_dfa_with_setting(query: &str, setting: PrefixSetting) -> DFA {
-    use PrefixSetting::{NoPrefix, Prefix};
-
-    match query.len() {
-        0..=4 => {
-            let builder = LEVDIST0.get_or_init(|| LevBuilder::new(0, true));
-            match setting {
-                Prefix => builder.build_prefix_dfa(query),
-                NoPrefix => builder.build_dfa(query),
-            }
-        }
-        5..=8 => {
-            let builder = LEVDIST1.get_or_init(|| LevBuilder::new(1, true));
-            match setting {
-                Prefix => builder.build_prefix_dfa(query),
-                NoPrefix => builder.build_dfa(query),
-            }
-        }
-        _ => {
-            let builder = LEVDIST2.get_or_init(|| LevBuilder::new(2, true));
-            match setting {
-                Prefix => builder.build_prefix_dfa(query),
-                NoPrefix => builder.build_dfa(query),
-            }
-        }
-    }
-}
-
-pub fn build_prefix_dfa(query: &str) -> DFA {
-    build_dfa_with_setting(query, PrefixSetting::Prefix)
-}
-
-pub fn build_dfa(query: &str) -> DFA {
-    build_dfa_with_setting(query, PrefixSetting::NoPrefix)
-}

meilidb-core/src/automaton/mod.rs

@@ -1,295 +0,0 @@
-mod dfa;
-mod query_enhancer;
-
-use std::cmp::Reverse;
-use std::{cmp, vec};
-
-use fst::{IntoStreamer, Streamer};
-use levenshtein_automata::DFA;
-use meilidb_tokenizer::{is_cjk, split_query_string};
-
-use crate::error::MResult;
-use crate::store;
-
-use self::dfa::{build_dfa, build_prefix_dfa};
-pub use self::query_enhancer::QueryEnhancer;
-use self::query_enhancer::QueryEnhancerBuilder;
-
-const NGRAMS: usize = 3;
-
-pub struct AutomatonProducer {
-    automatons: Vec<AutomatonGroup>,
-}
-
-impl AutomatonProducer {
-    pub fn new(
-        reader: &heed::RoTxn,
-        query: &str,
-        main_store: store::Main,
-        postings_list_store: store::PostingsLists,
-        synonyms_store: store::Synonyms,
-    ) -> MResult<(AutomatonProducer, QueryEnhancer)> {
-        let (automatons, query_enhancer) = generate_automatons(
-            reader,
-            query,
-            main_store,
-            postings_list_store,
-            synonyms_store,
-        )?;
-
-        Ok((AutomatonProducer { automatons }, query_enhancer))
-    }
-
-    pub fn into_iter(self) -> vec::IntoIter<AutomatonGroup> {
-        self.automatons.into_iter()
-    }
-}
-
-#[derive(Debug)]
-pub struct AutomatonGroup {
-    pub is_phrase_query: bool,
-    pub automatons: Vec<Automaton>,
-}
-
-impl AutomatonGroup {
-    fn normal(automatons: Vec<Automaton>) -> AutomatonGroup {
-        AutomatonGroup {
-            is_phrase_query: false,
-            automatons,
-        }
-    }
-
-    fn phrase_query(automatons: Vec<Automaton>) -> AutomatonGroup {
-        AutomatonGroup {
-            is_phrase_query: true,
-            automatons,
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct Automaton {
-    pub index: usize,
-    pub ngram: usize,
-    pub query_len: usize,
-    pub is_exact: bool,
-    pub is_prefix: bool,
-    pub query: String,
-}
-
-impl Automaton {
-    pub fn dfa(&self) -> DFA {
-        if self.is_prefix {
-            build_prefix_dfa(&self.query)
-        } else {
-            build_dfa(&self.query)
-        }
-    }
-
-    fn exact(index: usize, ngram: usize, query: &str) -> Automaton {
-        Automaton {
-            index,
-            ngram,
-            query_len: query.len(),
-            is_exact: true,
-            is_prefix: false,
-            query: query.to_string(),
-        }
-    }
-
-    fn prefix_exact(index: usize, ngram: usize, query: &str) -> Automaton {
-        Automaton {
-            index,
-            ngram,
-            query_len: query.len(),
-            is_exact: true,
-            is_prefix: true,
-            query: query.to_string(),
-        }
-    }
-
-    fn non_exact(index: usize, ngram: usize, query: &str) -> Automaton {
-        Automaton {
-            index,
-            ngram,
-            query_len: query.len(),
-            is_exact: false,
-            is_prefix: false,
-            query: query.to_string(),
-        }
-    }
-}
-
-pub fn normalize_str(string: &str) -> String {
-    let mut string = string.to_lowercase();
-
-    if !string.contains(is_cjk) {
-        string = deunicode::deunicode_with_tofu(&string, "");
-    }
-
-    string
-}
-
-fn split_best_frequency<'a>(
-    reader: &heed::RoTxn,
-    word: &'a str,
-    postings_lists_store: store::PostingsLists,
-) -> MResult<Option<(&'a str, &'a str)>> {
-    let chars = word.char_indices().skip(1);
-    let mut best = None;
-
-    for (i, _) in chars {
-        let (left, right) = word.split_at(i);
-
-        let left_freq = postings_lists_store
-            .postings_list(reader, left.as_ref())?
-            .map_or(0, |i| i.len());
-
-        let right_freq = postings_lists_store
-            .postings_list(reader, right.as_ref())?
-            .map_or(0, |i| i.len());
-
-        let min_freq = cmp::min(left_freq, right_freq);
-        if min_freq != 0 && best.map_or(true, |(old, _, _)| min_freq > old) {
-            best = Some((min_freq, left, right));
-        }
-    }
-
-    Ok(best.map(|(_, l, r)| (l, r)))
-}
-
-fn generate_automatons(
-    reader: &heed::RoTxn,
-    query: &str,
-    main_store: store::Main,
-    postings_lists_store: store::PostingsLists,
-    synonym_store: store::Synonyms,
-) -> MResult<(Vec<AutomatonGroup>, QueryEnhancer)> {
-    let has_end_whitespace = query.chars().last().map_or(false, char::is_whitespace);
-    let query_words: Vec<_> = split_query_string(query).map(str::to_lowercase).collect();
-    let synonyms = match main_store.synonyms_fst(reader)? {
-        Some(synonym) => synonym,
-        None => fst::Set::default(),
-    };
-
-    let mut automaton_index = 0;
-    let mut automatons = Vec::new();
-    let mut enhancer_builder = QueryEnhancerBuilder::new(&query_words);
-
-    // We must not declare the original words to the query enhancer
-    // *but* we need to push them in the automatons list first
-    let mut original_automatons = Vec::new();
-    let mut original_words = query_words.iter().peekable();
-    while let Some(word) = original_words.next() {
-        let has_following_word = original_words.peek().is_some();
-        let not_prefix_dfa = has_following_word || has_end_whitespace || word.chars().all(is_cjk);
-
-        let automaton = if not_prefix_dfa {
-            Automaton::exact(automaton_index, 1, word)
-        } else {
-            Automaton::prefix_exact(automaton_index, 1, word)
-        };
-        automaton_index += 1;
-        original_automatons.push(automaton);
-    }
-
-    automatons.push(AutomatonGroup::normal(original_automatons));
-
-    for n in 1..=NGRAMS {
-        let mut ngrams = query_words.windows(n).enumerate().peekable();
-        while let Some((query_index, ngram_slice)) = ngrams.next() {
-            let query_range = query_index..query_index + n;
-            let ngram_nb_words = ngram_slice.len();
-            let ngram = ngram_slice.join(" ");
-
-            let has_following_word = ngrams.peek().is_some();
-            let not_prefix_dfa =
-                has_following_word || has_end_whitespace || ngram.chars().all(is_cjk);
-
-            // automaton of synonyms of the ngrams
-            let normalized = normalize_str(&ngram);
-            let lev = if not_prefix_dfa {
-                build_dfa(&normalized)
-            } else {
-                build_prefix_dfa(&normalized)
-            };
-
-            let mut stream = synonyms.search(&lev).into_stream();
-            while let Some(base) = stream.next() {
-                // only trigger alternatives when the last word has been typed
-                // i.e. "new " do not but "new yo" triggers alternatives to "new york"
-                let base = std::str::from_utf8(base).unwrap();
-                let base_nb_words = split_query_string(base).count();
-                if ngram_nb_words != base_nb_words {
-                    continue;
-                }
-
-                if let Some(synonyms) = synonym_store.synonyms(reader, base.as_bytes())? {
-                    let mut stream = synonyms.into_stream();
-                    while let Some(synonyms) = stream.next() {
-                        let synonyms = std::str::from_utf8(synonyms).unwrap();
-                        let synonyms_words: Vec<_> = split_query_string(synonyms).collect();
-                        let nb_synonym_words = synonyms_words.len();
-
-                        let real_query_index = automaton_index;
-                        enhancer_builder.declare(
-                            query_range.clone(),
-                            real_query_index,
-                            &synonyms_words,
-                        );
-
-                        for synonym in synonyms_words {
-                            let automaton = if nb_synonym_words == 1 {
-                                Automaton::exact(automaton_index, n, synonym)
-                            } else {
-                                Automaton::non_exact(automaton_index, n, synonym)
-                            };
-                            automaton_index += 1;
-                            automatons.push(AutomatonGroup::normal(vec![automaton]));
-                        }
-                    }
-                }
-            }
-
-            if n == 1 {
-                if let Some((left, right)) =
-                    split_best_frequency(reader, &normalized, postings_lists_store)?
-                {
-                    let a = Automaton::exact(automaton_index, 1, left);
-                    enhancer_builder.declare(query_range.clone(), automaton_index, &[left]);
-                    automaton_index += 1;
-
-                    let b = Automaton::exact(automaton_index, 1, right);
-                    enhancer_builder.declare(query_range.clone(), automaton_index, &[left]);
-                    automaton_index += 1;
-
-                    automatons.push(AutomatonGroup::phrase_query(vec![a, b]));
-                }
-            } else {
-                // automaton of concatenation of query words
-                let concat = ngram_slice.concat();
-                let normalized = normalize_str(&concat);
-
-                let real_query_index = automaton_index;
-                enhancer_builder.declare(query_range.clone(), real_query_index, &[&normalized]);
-
-                let automaton = Automaton::exact(automaton_index, n, &normalized);
-                automaton_index += 1;
-                automatons.push(AutomatonGroup::normal(vec![automaton]));
-            }
-        }
-    }
-
-    // order automatons, the most important first,
-    // we keep the original automatons at the front.
-    automatons[1..].sort_by_key(|group| {
-        let a = group.automatons.first().unwrap();
-        (
-            Reverse(a.is_exact),
-            a.ngram,
-            Reverse(group.automatons.len()),
-        )
-    });
-
-    Ok((automatons, enhancer_builder.build()))
-}

meilidb-core/src/automaton/query_enhancer.rs

@@ -1,423 +0,0 @@
-use std::cmp::Ordering::{Equal, Greater, Less};
-use std::ops::Range;
-
-/// Return `true` if the specified range can accept the given replacements words.
-/// Returns `false` if the replacements words are already present in the original query
-/// or if there is fewer replacement words than the range to replace.
-//
-//
-// ## Ignored because already present in original
-//
-//     new york city subway
-//     -------- ^^^^
-//   /          \
-//  [new york city]
-//
-//
-// ## Ignored because smaller than the original
-//
-//   new york city subway
-//   -------------
-//   \          /
-//    [new york]
-//
-//
-// ## Accepted because bigger than the original
-//
-//        NYC subway
-//        ---
-//       /   \
-//      /     \
-//     /       \
-//    /         \
-//   /           \
-//  [new york city]
-//
-fn rewrite_range_with<S, T>(query: &[S], range: Range<usize>, words: &[T]) -> bool
-where
-    S: AsRef<str>,
-    T: AsRef<str>,
-{
-    if words.len() <= range.len() {
-        // there is fewer or equal replacement words
-        // than there is already in the replaced range
-        return false;
-    }
-
-    // retrieve the part to rewrite but with the length
-    // of the replacement part
-    let original = query.iter().skip(range.start).take(words.len());
-
-    // check if the original query doesn't already contain
-    // the replacement words
-    !original
-        .map(AsRef::as_ref)
-        .eq(words.iter().map(AsRef::as_ref))
-}
-
-type Origin = usize;
-type RealLength = usize;
-
-struct FakeIntervalTree {
-    intervals: Vec<(Range<usize>, (Origin, RealLength))>,
-}
-
-impl FakeIntervalTree {
-    fn new(mut intervals: Vec<(Range<usize>, (Origin, RealLength))>) -> FakeIntervalTree {
-        intervals.sort_unstable_by_key(|(r, _)| (r.start, r.end));
-        FakeIntervalTree { intervals }
-    }
-
-    fn query(&self, point: usize) -> Option<(Range<usize>, (Origin, RealLength))> {
-        let element = self.intervals.binary_search_by(|(r, _)| {
-            if point >= r.start {
-                if point < r.end {
-                    Equal
-                } else {
-                    Less
-                }
-            } else {
-                Greater
-            }
-        });
-
-        let n = match element {
-            Ok(n) => n,
-            Err(n) => n,
-        };
-
-        match self.intervals.get(n) {
-            Some((range, value)) if range.contains(&point) => Some((range.clone(), *value)),
-            _otherwise => None,
-        }
-    }
-}
-
-pub struct QueryEnhancerBuilder<'a, S> {
-    query: &'a [S],
-    origins: Vec<usize>,
-    real_to_origin: Vec<(Range<usize>, (Origin, RealLength))>,
-}
-
-impl<S: AsRef<str>> QueryEnhancerBuilder<'_, S> {
-    pub fn new(query: &[S]) -> QueryEnhancerBuilder<S> {
-        // we initialize origins query indices based on their positions
-        let origins: Vec<_> = (0..=query.len()).collect();
-        let real_to_origin = origins.iter().map(|&o| (o..o + 1, (o, 1))).collect();
-
-        QueryEnhancerBuilder {
-            query,
-            origins,
-            real_to_origin,
-        }
-    }
-
-    /// Update the final real to origin query indices mapping.
-    ///
-    /// `range` is the original words range that this `replacement` words replace
-    /// and `real` is the first real query index of these replacement words.
-    pub fn declare<T>(&mut self, range: Range<usize>, real: usize, replacement: &[T])
-    where
-        T: AsRef<str>,
-    {
-        // check if the range of original words
-        // can be rewritten with the replacement words
-        if rewrite_range_with(self.query, range.clone(), replacement) {
-            // this range can be replaced so we need to
-            // modify the origins accordingly
-            let offset = replacement.len() - range.len();
-
-            let previous_padding = self.origins[range.end - 1];
-            let current_offset = (self.origins[range.end] - 1) - previous_padding;
-            let diff = offset.saturating_sub(current_offset);
-            self.origins[range.end] += diff;
-
-            for r in &mut self.origins[range.end + 1..] {
-                *r += diff;
-            }
-        }
-
-        // we need to store the real number and origins relations
-        // this way it will be possible to know by how many
-        // we need to pad real query indices
-        let real_range = real..real + replacement.len().max(range.len());
-        let real_length = replacement.len();
-        self.real_to_origin
-            .push((real_range, (range.start, real_length)));
-    }
-
-    pub fn build(self) -> QueryEnhancer {
-        QueryEnhancer {
-            origins: self.origins,
-            real_to_origin: FakeIntervalTree::new(self.real_to_origin),
-        }
-    }
-}
-
-pub struct QueryEnhancer {
-    origins: Vec<usize>,
-    real_to_origin: FakeIntervalTree,
-}
-
-impl QueryEnhancer {
-    /// Returns the query indices to use to replace this real query index.
-    pub fn replacement(&self, real: u32) -> Range<u32> {
-        let real = real as usize;
-
-        // query the fake interval tree with the real query index
-        let (range, (origin, real_length)) = self
-            .real_to_origin
-            .query(real)
-            .expect("real has never been declared");
-
-        // if `real` is the end bound of the range
-        if (range.start + real_length - 1) == real {
-            let mut count = range.len();
-            let mut new_origin = origin;
-            for (i, slice) in self.origins[new_origin..].windows(2).enumerate() {
-                let len = slice[1] - slice[0];
-                count = count.saturating_sub(len);
-                if count == 0 {
-                    new_origin = origin + i;
-                    break;
-                }
-            }
-
-            let n = real - range.start;
-            let start = self.origins[origin];
-            let end = self.origins[new_origin + 1];
-            let remaining = (end - start) - n;
-
-            Range {
-                start: (start + n) as u32,
-                end: (start + n + remaining) as u32,
-            }
-        } else {
-            // just return the origin along with
-            // the real position of the word
-            let n = real as usize - range.start;
-            let origin = self.origins[origin];
-
-            Range {
-                start: (origin + n) as u32,
-                end: (origin + n + 1) as u32,
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn original_unmodified() {
-        let query = ["new", "york", "city", "subway"];
-        //             0       1       2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // new york = new york city
-        builder.declare(0..2, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // new
-        assert_eq!(enhancer.replacement(1), 1..2); // york
-        assert_eq!(enhancer.replacement(2), 2..3); // city
-        assert_eq!(enhancer.replacement(3), 3..4); // subway
-        assert_eq!(enhancer.replacement(4), 0..1); // new
-        assert_eq!(enhancer.replacement(5), 1..2); // york
-        assert_eq!(enhancer.replacement(6), 2..3); // city
-    }
-
-    #[test]
-    fn simple_growing() {
-        let query = ["new", "york", "subway"];
-        //             0       1        2
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // new york = new york city
-        builder.declare(0..2, 3, &["new", "york", "city"]);
-        //                    ^      3       4       5
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // new
-        assert_eq!(enhancer.replacement(1), 1..3); // york
-        assert_eq!(enhancer.replacement(2), 3..4); // subway
-        assert_eq!(enhancer.replacement(3), 0..1); // new
-        assert_eq!(enhancer.replacement(4), 1..2); // york
-        assert_eq!(enhancer.replacement(5), 2..3); // city
-    }
-
-    #[test]
-    fn same_place_growings() {
-        let query = ["NY", "subway"];
-        //             0       1
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NY = new york
-        builder.declare(0..1, 2, &["new", "york"]);
-        //                    ^      2       3
-
-        // NY = new york city
-        builder.declare(0..1, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // NY = NYC
-        builder.declare(0..1, 7, &["NYC"]);
-        //                    ^      7
-
-        // NY = new york city
-        builder.declare(0..1, 8, &["new", "york", "city"]);
-        //                    ^      8       9      10
-
-        // subway = underground train
-        builder.declare(1..2, 11, &["underground", "train"]);
-        //                    ^          11          12
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..3); // NY
-        assert_eq!(enhancer.replacement(1), 3..5); // subway
-        assert_eq!(enhancer.replacement(2), 0..1); // new
-        assert_eq!(enhancer.replacement(3), 1..3); // york
-        assert_eq!(enhancer.replacement(4), 0..1); // new
-        assert_eq!(enhancer.replacement(5), 1..2); // york
-        assert_eq!(enhancer.replacement(6), 2..3); // city
-        assert_eq!(enhancer.replacement(7), 0..3); // NYC
-        assert_eq!(enhancer.replacement(8), 0..1); // new
-        assert_eq!(enhancer.replacement(9), 1..2); // york
-        assert_eq!(enhancer.replacement(10), 2..3); // city
-        assert_eq!(enhancer.replacement(11), 3..4); // underground
-        assert_eq!(enhancer.replacement(12), 4..5); // train
-    }
-
-    #[test]
-    fn bigger_growing() {
-        let query = ["NYC", "subway"];
-        //             0        1
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(0..1, 2, &["new", "york", "city"]);
-        //                    ^      2       3       4
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..3); // NYC
-        assert_eq!(enhancer.replacement(1), 3..4); // subway
-        assert_eq!(enhancer.replacement(2), 0..1); // new
-        assert_eq!(enhancer.replacement(3), 1..2); // york
-        assert_eq!(enhancer.replacement(4), 2..3); // city
-    }
-
-    #[test]
-    fn middle_query_growing() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // great
-        assert_eq!(enhancer.replacement(1), 1..2); // awesome
-        assert_eq!(enhancer.replacement(2), 2..5); // NYC
-        assert_eq!(enhancer.replacement(3), 5..6); // subway
-        assert_eq!(enhancer.replacement(4), 2..3); // new
-        assert_eq!(enhancer.replacement(5), 3..4); // york
-        assert_eq!(enhancer.replacement(6), 4..5); // city
-    }
-
-    #[test]
-    fn end_query_growing() {
-        let query = ["NYC", "subway"];
-        //             0        1
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(1..2, 2, &["underground", "train"]);
-        //                    ^         2            3
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // NYC
-        assert_eq!(enhancer.replacement(1), 1..3); // subway
-        assert_eq!(enhancer.replacement(2), 1..2); // underground
-        assert_eq!(enhancer.replacement(3), 2..3); // train
-    }
-
-    #[test]
-    fn multiple_growings() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // subway = underground train
-        builder.declare(3..4, 7, &["underground", "train"]);
-        //                    ^          7           8
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // great
-        assert_eq!(enhancer.replacement(1), 1..2); // awesome
-        assert_eq!(enhancer.replacement(2), 2..5); // NYC
-        assert_eq!(enhancer.replacement(3), 5..7); // subway
-        assert_eq!(enhancer.replacement(4), 2..3); // new
-        assert_eq!(enhancer.replacement(5), 3..4); // york
-        assert_eq!(enhancer.replacement(6), 4..5); // city
-        assert_eq!(enhancer.replacement(7), 5..6); // underground
-        assert_eq!(enhancer.replacement(8), 6..7); // train
-    }
-
-    #[test]
-    fn multiple_probable_growings() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // subway = underground train
-        builder.declare(3..4, 7, &["underground", "train"]);
-        //                    ^          7           8
-
-        // great awesome = good
-        builder.declare(0..2, 9, &["good"]);
-        //                    ^       9
-
-        // awesome NYC = NY
-        builder.declare(1..3, 10, &["NY"]);
-        //                    ^^     10
-
-        // NYC subway = metro
-        builder.declare(2..4, 11, &["metro"]);
-        //                    ^^      11
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // great
-        assert_eq!(enhancer.replacement(1), 1..2); // awesome
-        assert_eq!(enhancer.replacement(2), 2..5); // NYC
-        assert_eq!(enhancer.replacement(3), 5..7); // subway
-        assert_eq!(enhancer.replacement(4), 2..3); // new
-        assert_eq!(enhancer.replacement(5), 3..4); // york
-        assert_eq!(enhancer.replacement(6), 4..5); // city
-        assert_eq!(enhancer.replacement(7), 5..6); // underground
-        assert_eq!(enhancer.replacement(8), 6..7); // train
-        assert_eq!(enhancer.replacement(9), 0..2); // good
-        assert_eq!(enhancer.replacement(10), 1..5); // NY
-        assert_eq!(enhancer.replacement(11), 2..5); // metro
-    }
-}

meilidb-core/src/criterion/document_id.rs

@@ -1,16 +0,0 @@
-use crate::criterion::Criterion;
-use crate::RawDocument;
-use std::cmp::Ordering;
-
-#[derive(Debug, Clone, Copy)]
-pub struct DocumentId;
-
-impl Criterion for DocumentId {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        lhs.id.cmp(&rhs.id)
-    }
-
-    fn name(&self) -> &str {
-        "DocumentId"
-    }
-}

meilidb-core/src/criterion/exact.rs

@@ -1,132 +0,0 @@
-use std::cmp::Ordering;
-
-use meilidb_schema::SchemaAttr;
-use sdset::Set;
-use slice_group_by::GroupBy;
-
-use crate::criterion::Criterion;
-use crate::RawDocument;
-
-#[inline]
-fn number_exact_matches(
-    query_index: &[u32],
-    attribute: &[u16],
-    is_exact: &[bool],
-    fields_counts: &Set<(SchemaAttr, u64)>,
-) -> usize {
-    let mut count = 0;
-    let mut index = 0;
-
-    for group in query_index.linear_group() {
-        let len = group.len();
-
-        let mut found_exact = false;
-        for (pos, is_exact) in is_exact[index..index + len].iter().enumerate() {
-            if *is_exact {
-                found_exact = true;
-                let attr = &attribute[index + pos];
-                if let Ok(pos) = fields_counts.binary_search_by_key(attr, |(a, _)| a.0) {
-                    let (_, count) = fields_counts[pos];
-                    if count == 1 {
-                        return usize::max_value();
-                    }
-                }
-            }
-        }
-
-        count += found_exact as usize;
-        index += len;
-    }
-
-    count
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct Exact;
-
-impl Criterion for Exact {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = {
-            let query_index = lhs.query_index();
-            let is_exact = lhs.is_exact();
-            let attribute = lhs.attribute();
-            let fields_counts = &lhs.fields_counts;
-
-            number_exact_matches(query_index, attribute, is_exact, fields_counts)
-        };
-
-        let rhs = {
-            let query_index = rhs.query_index();
-            let is_exact = rhs.is_exact();
-            let attribute = rhs.attribute();
-            let fields_counts = &rhs.fields_counts;
-
-            number_exact_matches(query_index, attribute, is_exact, fields_counts)
-        };
-
-        lhs.cmp(&rhs).reverse()
-    }
-
-    fn name(&self) -> &str {
-        "Exact"
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    // typing: "soulier"
-    //
-    // doc0: "Soulier bleu"
-    // doc1: "souliereres rouge"
-    #[test]
-    fn easy_case() {
-        let doc0 = {
-            let query_index = &[0];
-            let attribute = &[0];
-            let is_exact = &[true];
-            let fields_counts = Set::new(&[(SchemaAttr(0), 2)]).unwrap();
-
-            number_exact_matches(query_index, attribute, is_exact, fields_counts)
-        };
-
-        let doc1 = {
-            let query_index = &[0];
-            let attribute = &[0];
-            let is_exact = &[false];
-            let fields_counts = Set::new(&[(SchemaAttr(0), 2)]).unwrap();
-
-            number_exact_matches(query_index, attribute, is_exact, fields_counts)
-        };
-
-        assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
-    }
-
-    // typing: "soulier"
-    //
-    // doc0: { 0. "soulier" }
-    // doc1: { 0. "soulier bleu et blanc" }
-    #[test]
-    fn basic() {
-        let doc0 = {
-            let query_index = &[0];
-            let attribute = &[0];
-            let is_exact = &[true];
-            let fields_counts = Set::new(&[(SchemaAttr(0), 1)]).unwrap();
-
-            number_exact_matches(query_index, attribute, is_exact, fields_counts)
-        };
-
-        let doc1 = {
-            let query_index = &[0];
-            let attribute = &[0];
-            let is_exact = &[true];
-            let fields_counts = Set::new(&[(SchemaAttr(0), 4)]).unwrap();
-
-            number_exact_matches(query_index, attribute, is_exact, fields_counts)
-        };
-
-        assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
-    }
-}

meilidb-core/src/criterion/mod.rs

@@ -1,121 +0,0 @@
-mod document_id;
-mod exact;
-mod number_of_words;
-mod sort_by_attr;
-mod sum_of_typos;
-mod sum_of_words_attribute;
-mod sum_of_words_position;
-mod words_proximity;
-
-use crate::RawDocument;
-use std::cmp::Ordering;
-
-pub use self::{
-    document_id::DocumentId, exact::Exact, number_of_words::NumberOfWords,
-    sort_by_attr::SortByAttr, sum_of_typos::SumOfTypos,
-    sum_of_words_attribute::SumOfWordsAttribute, sum_of_words_position::SumOfWordsPosition,
-    words_proximity::WordsProximity,
-};
-
-pub trait Criterion: Send + Sync {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering;
-
-    fn name(&self) -> &str;
-
-    #[inline]
-    fn eq(&self, lhs: &RawDocument, rhs: &RawDocument) -> bool {
-        self.evaluate(lhs, rhs) == Ordering::Equal
-    }
-}
-
-impl<'a, T: Criterion + ?Sized + Send + Sync> Criterion for &'a T {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        (**self).evaluate(lhs, rhs)
-    }
-
-    fn name(&self) -> &str {
-        (**self).name()
-    }
-
-    fn eq(&self, lhs: &RawDocument, rhs: &RawDocument) -> bool {
-        (**self).eq(lhs, rhs)
-    }
-}
-
-impl<T: Criterion + ?Sized> Criterion for Box<T> {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        (**self).evaluate(lhs, rhs)
-    }
-
-    fn name(&self) -> &str {
-        (**self).name()
-    }
-
-    fn eq(&self, lhs: &RawDocument, rhs: &RawDocument) -> bool {
-        (**self).eq(lhs, rhs)
-    }
-}
-
-#[derive(Default)]
-pub struct CriteriaBuilder<'a> {
-    inner: Vec<Box<dyn Criterion + 'a>>,
-}
-
-impl<'a> CriteriaBuilder<'a> {
-    pub fn new() -> CriteriaBuilder<'a> {
-        CriteriaBuilder { inner: Vec::new() }
-    }
-
-    pub fn with_capacity(capacity: usize) -> CriteriaBuilder<'a> {
-        CriteriaBuilder {
-            inner: Vec::with_capacity(capacity),
-        }
-    }
-
-    pub fn reserve(&mut self, additional: usize) {
-        self.inner.reserve(additional)
-    }
-
-    pub fn add<C: 'a>(mut self, criterion: C) -> CriteriaBuilder<'a>
-    where
-        C: Criterion,
-    {
-        self.push(criterion);
-        self
-    }
-
-    pub fn push<C: 'a>(&mut self, criterion: C)
-    where
-        C: Criterion,
-    {
-        self.inner.push(Box::new(criterion));
-    }
-
-    pub fn build(self) -> Criteria<'a> {
-        Criteria { inner: self.inner }
-    }
-}
-
-pub struct Criteria<'a> {
-    inner: Vec<Box<dyn Criterion + 'a>>,
-}
-
-impl<'a> Default for Criteria<'a> {
-    fn default() -> Self {
-        CriteriaBuilder::with_capacity(7)
-            .add(SumOfTypos)
-            .add(NumberOfWords)
-            .add(WordsProximity)
-            .add(SumOfWordsAttribute)
-            .add(SumOfWordsPosition)
-            .add(Exact)
-            .add(DocumentId)
-            .build()
-    }
-}
-
-impl<'a> AsRef<[Box<dyn Criterion + 'a>]> for Criteria<'a> {
-    fn as_ref(&self) -> &[Box<dyn Criterion + 'a>] {
-        &self.inner
-    }
-}

meilidb-core/src/criterion/number_of_words.rs

@@ -1,31 +0,0 @@
-use crate::criterion::Criterion;
-use crate::RawDocument;
-use slice_group_by::GroupBy;
-use std::cmp::Ordering;
-
-#[inline]
-fn number_of_query_words(query_index: &[u32]) -> usize {
-    query_index.linear_group().count()
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct NumberOfWords;
-
-impl Criterion for NumberOfWords {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = {
-            let query_index = lhs.query_index();
-            number_of_query_words(query_index)
-        };
-        let rhs = {
-            let query_index = rhs.query_index();
-            number_of_query_words(query_index)
-        };
-
-        lhs.cmp(&rhs).reverse()
-    }
-
-    fn name(&self) -> &str {
-        "NumberOfWords"
-    }
-}

meilidb-core/src/criterion/sort_by_attr.rs

@@ -1,130 +0,0 @@
-use std::cmp::Ordering;
-use std::error::Error;
-use std::fmt;
-
-use crate::criterion::Criterion;
-use crate::{RankedMap, RawDocument};
-use meilidb_schema::{Schema, SchemaAttr};
-
-/// An helper struct that permit to sort documents by
-/// some of their stored attributes.
-///
-/// # Note
-///
-/// If a document cannot be deserialized it will be considered [`None`][].
-///
-/// Deserialized documents are compared like `Some(doc0).cmp(&Some(doc1))`,
-/// so you must check the [`Ord`] of `Option` implementation.
-///
-/// [`None`]: https://doc.rust-lang.org/std/option/enum.Option.html#variant.None
-/// [`Ord`]: https://doc.rust-lang.org/std/option/enum.Option.html#impl-Ord
-///
-/// # Example
-///
-/// ```ignore
-/// use serde_derive::Deserialize;
-/// use meilidb::rank::criterion::*;
-///
-/// let custom_ranking = SortByAttr::lower_is_better(&ranked_map, &schema, "published_at")?;
-///
-/// let builder = CriteriaBuilder::with_capacity(8)
-///        .add(SumOfTypos)
-///        .add(NumberOfWords)
-///        .add(WordsProximity)
-///        .add(SumOfWordsAttribute)
-///        .add(SumOfWordsPosition)
-///        .add(Exact)
-///        .add(custom_ranking)
-///        .add(DocumentId);
-///
-/// let criterion = builder.build();
-///
-/// ```
-pub struct SortByAttr<'a> {
-    ranked_map: &'a RankedMap,
-    attr: SchemaAttr,
-    reversed: bool,
-}
-
-impl<'a> SortByAttr<'a> {
-    pub fn lower_is_better(
-        ranked_map: &'a RankedMap,
-        schema: &Schema,
-        attr_name: &str,
-    ) -> Result<SortByAttr<'a>, SortByAttrError> {
-        SortByAttr::new(ranked_map, schema, attr_name, false)
-    }
-
-    pub fn higher_is_better(
-        ranked_map: &'a RankedMap,
-        schema: &Schema,
-        attr_name: &str,
-    ) -> Result<SortByAttr<'a>, SortByAttrError> {
-        SortByAttr::new(ranked_map, schema, attr_name, true)
-    }
-
-    fn new(
-        ranked_map: &'a RankedMap,
-        schema: &Schema,
-        attr_name: &str,
-        reversed: bool,
-    ) -> Result<SortByAttr<'a>, SortByAttrError> {
-        let attr = match schema.attribute(attr_name) {
-            Some(attr) => attr,
-            None => return Err(SortByAttrError::AttributeNotFound),
-        };
-
-        if !schema.props(attr).is_ranked() {
-            return Err(SortByAttrError::AttributeNotRegisteredForRanking);
-        }
-
-        Ok(SortByAttr {
-            ranked_map,
-            attr,
-            reversed,
-        })
-    }
-}
-
-impl<'a> Criterion for SortByAttr<'a> {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = self.ranked_map.get(lhs.id, self.attr);
-        let rhs = self.ranked_map.get(rhs.id, self.attr);
-
-        match (lhs, rhs) {
-            (Some(lhs), Some(rhs)) => {
-                let order = lhs.cmp(&rhs);
-                if self.reversed {
-                    order.reverse()
-                } else {
-                    order
-                }
-            }
-            (None, Some(_)) => Ordering::Greater,
-            (Some(_), None) => Ordering::Less,
-            (None, None) => Ordering::Equal,
-        }
-    }
-
-    fn name(&self) -> &str {
-        "SortByAttr"
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum SortByAttrError {
-    AttributeNotFound,
-    AttributeNotRegisteredForRanking,
-}
-
-impl fmt::Display for SortByAttrError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        use SortByAttrError::*;
-        match self {
-            AttributeNotFound => f.write_str("attribute not found in the schema"),
-            AttributeNotRegisteredForRanking => f.write_str("attribute not registered for ranking"),
-        }
-    }
-}
-
-impl Error for SortByAttrError {}

meilidb-core/src/criterion/sum_of_typos.rs

@@ -1,116 +0,0 @@
-use std::cmp::Ordering;
-
-use slice_group_by::GroupBy;
-
-use crate::criterion::Criterion;
-use crate::RawDocument;
-
-// This function is a wrong logarithmic 10 function.
-// It is safe to panic on input number higher than 3,
-// the number of typos is never bigger than that.
-#[inline]
-fn custom_log10(n: u8) -> f32 {
-    match n {
-        0 => 0.0,     // log(1)
-        1 => 0.30102, // log(2)
-        2 => 0.47712, // log(3)
-        3 => 0.60205, // log(4)
-        _ => panic!("invalid number"),
-    }
-}
-
-#[inline]
-fn sum_matches_typos(query_index: &[u32], distance: &[u8]) -> usize {
-    let mut number_words: usize = 0;
-    let mut sum_typos = 0.0;
-    let mut index = 0;
-
-    for group in query_index.linear_group() {
-        sum_typos += custom_log10(distance[index]);
-        number_words += 1;
-        index += group.len();
-    }
-
-    (number_words as f32 / (sum_typos + 1.0) * 1000.0) as usize
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct SumOfTypos;
-
-impl Criterion for SumOfTypos {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = {
-            let query_index = lhs.query_index();
-            let distance = lhs.distance();
-            sum_matches_typos(query_index, distance)
-        };
-
-        let rhs = {
-            let query_index = rhs.query_index();
-            let distance = rhs.distance();
-            sum_matches_typos(query_index, distance)
-        };
-
-        lhs.cmp(&rhs).reverse()
-    }
-
-    fn name(&self) -> &str {
-        "SumOfTypos"
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    // typing: "Geox CEO"
-    //
-    // doc0: "Geox SpA: CEO and Executive"
-    // doc1: "Mt. Gox CEO Resigns From Bitcoin Foundation"
-    #[test]
-    fn one_typo_reference() {
-        let query_index0 = &[0, 1];
-        let distance0 = &[0, 0];
-
-        let query_index1 = &[0, 1];
-        let distance1 = &[1, 0];
-
-        let doc0 = sum_matches_typos(query_index0, distance0);
-        let doc1 = sum_matches_typos(query_index1, distance1);
-        assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
-    }
-
-    // typing: "bouton manchette"
-    //
-    // doc0: "bouton manchette"
-    // doc1: "bouton"
-    #[test]
-    fn no_typo() {
-        let query_index0 = &[0, 1];
-        let distance0 = &[0, 0];
-
-        let query_index1 = &[0];
-        let distance1 = &[0];
-
-        let doc0 = sum_matches_typos(query_index0, distance0);
-        let doc1 = sum_matches_typos(query_index1, distance1);
-        assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
-    }
-
-    // typing: "bouton manchztte"
-    //
-    // doc0: "bouton manchette"
-    // doc1: "bouton"
-    #[test]
-    fn one_typo() {
-        let query_index0 = &[0, 1];
-        let distance0 = &[0, 1];
-
-        let query_index1 = &[0];
-        let distance1 = &[0];
-
-        let doc0 = sum_matches_typos(query_index0, distance0);
-        let doc1 = sum_matches_typos(query_index1, distance1);
-        assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
-    }
-}

meilidb-core/src/criterion/sum_of_words_attribute.rs

@@ -1,64 +0,0 @@
-use crate::criterion::Criterion;
-use crate::RawDocument;
-use slice_group_by::GroupBy;
-use std::cmp::Ordering;
-
-#[inline]
-fn sum_matches_attributes(query_index: &[u32], attribute: &[u16]) -> usize {
-    let mut sum_attributes = 0;
-    let mut index = 0;
-
-    for group in query_index.linear_group() {
-        sum_attributes += attribute[index] as usize;
-        index += group.len();
-    }
-
-    sum_attributes
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct SumOfWordsAttribute;
-
-impl Criterion for SumOfWordsAttribute {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = {
-            let query_index = lhs.query_index();
-            let attribute = lhs.attribute();
-            sum_matches_attributes(query_index, attribute)
-        };
-
-        let rhs = {
-            let query_index = rhs.query_index();
-            let attribute = rhs.attribute();
-            sum_matches_attributes(query_index, attribute)
-        };
-
-        lhs.cmp(&rhs)
-    }
-
-    fn name(&self) -> &str {
-        "SumOfWordsAttribute"
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    // typing: "soulier"
-    //
-    // doc0: { 0. "Soulier bleu", 1. "bla bla bla" }
-    // doc1: { 0. "Botte rouge", 1. "Soulier en cuir" }
-    #[test]
-    fn title_vs_description() {
-        let query_index0 = &[0];
-        let attribute0 = &[0];
-
-        let query_index1 = &[0];
-        let attribute1 = &[1];
-
-        let doc0 = sum_matches_attributes(query_index0, attribute0);
-        let doc1 = sum_matches_attributes(query_index1, attribute1);
-        assert_eq!(doc0.cmp(&doc1), Ordering::Less);
-    }
-}

meilidb-core/src/criterion/sum_of_words_position.rs

@@ -1,64 +0,0 @@
-use crate::criterion::Criterion;
-use crate::RawDocument;
-use slice_group_by::GroupBy;
-use std::cmp::Ordering;
-
-#[inline]
-fn sum_matches_attribute_index(query_index: &[u32], word_index: &[u16]) -> usize {
-    let mut sum_word_index = 0;
-    let mut index = 0;
-
-    for group in query_index.linear_group() {
-        sum_word_index += word_index[index] as usize;
-        index += group.len();
-    }
-
-    sum_word_index
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct SumOfWordsPosition;
-
-impl Criterion for SumOfWordsPosition {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = {
-            let query_index = lhs.query_index();
-            let word_index = lhs.word_index();
-            sum_matches_attribute_index(query_index, word_index)
-        };
-
-        let rhs = {
-            let query_index = rhs.query_index();
-            let word_index = rhs.word_index();
-            sum_matches_attribute_index(query_index, word_index)
-        };
-
-        lhs.cmp(&rhs)
-    }
-
-    fn name(&self) -> &str {
-        "SumOfWordsPosition"
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    // typing: "soulier"
-    //
-    // doc0: "Soulier bleu"
-    // doc1: "Botte rouge et soulier noir"
-    #[test]
-    fn easy_case() {
-        let query_index0 = &[0];
-        let word_index0 = &[0];
-
-        let query_index1 = &[0];
-        let word_index1 = &[3];
-
-        let doc0 = sum_matches_attribute_index(query_index0, word_index0);
-        let doc1 = sum_matches_attribute_index(query_index1, word_index1);
-        assert_eq!(doc0.cmp(&doc1), Ordering::Less);
-    }
-}

meilidb-core/src/criterion/words_proximity.rs

@@ -1,164 +0,0 @@
-use crate::criterion::Criterion;
-use crate::RawDocument;
-use slice_group_by::GroupBy;
-use std::cmp::{self, Ordering};
-
-const MAX_DISTANCE: u16 = 8;
-
-#[inline]
-fn clone_tuple<T: Clone, U: Clone>((a, b): (&T, &U)) -> (T, U) {
-    (a.clone(), b.clone())
-}
-
-fn index_proximity(lhs: u16, rhs: u16) -> u16 {
-    if lhs < rhs {
-        cmp::min(rhs - lhs, MAX_DISTANCE)
-    } else {
-        cmp::min(lhs - rhs, MAX_DISTANCE) + 1
-    }
-}
-
-fn attribute_proximity((lattr, lwi): (u16, u16), (rattr, rwi): (u16, u16)) -> u16 {
-    if lattr != rattr {
-        return MAX_DISTANCE;
-    }
-    index_proximity(lwi, rwi)
-}
-
-fn min_proximity((lattr, lwi): (&[u16], &[u16]), (rattr, rwi): (&[u16], &[u16])) -> u16 {
-    let mut min_prox = u16::max_value();
-
-    for a in lattr.iter().zip(lwi) {
-        for b in rattr.iter().zip(rwi) {
-            let a = clone_tuple(a);
-            let b = clone_tuple(b);
-            min_prox = cmp::min(min_prox, attribute_proximity(a, b));
-        }
-    }
-
-    min_prox
-}
-
-fn matches_proximity(
-    query_index: &[u32],
-    distance: &[u8],
-    attribute: &[u16],
-    word_index: &[u16],
-) -> u16 {
-    let mut query_index_groups = query_index.linear_group();
-    let mut proximity = 0;
-    let mut index = 0;
-
-    let get_attr_wi = |index: usize, group_len: usize| {
-        // retrieve the first distance group (with the lowest values)
-        let len = distance[index..index + group_len]
-            .linear_group()
-            .next()
-            .unwrap()
-            .len();
-
-        let rattr = &attribute[index..index + len];
-        let rwi = &word_index[index..index + len];
-
-        (rattr, rwi)
-    };
-
-    let mut last = query_index_groups.next().map(|group| {
-        let attr_wi = get_attr_wi(index, group.len());
-        index += group.len();
-        attr_wi
-    });
-
-    // iter by windows of size 2
-    while let (Some(lhs), Some(rhs)) = (last, query_index_groups.next()) {
-        let attr_wi = get_attr_wi(index, rhs.len());
-        proximity += min_proximity(lhs, attr_wi);
-        last = Some(attr_wi);
-        index += rhs.len();
-    }
-
-    proximity
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct WordsProximity;
-
-impl Criterion for WordsProximity {
-    fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
-        let lhs = {
-            let query_index = lhs.query_index();
-            let distance = lhs.distance();
-            let attribute = lhs.attribute();
-            let word_index = lhs.word_index();
-            matches_proximity(query_index, distance, attribute, word_index)
-        };
-
-        let rhs = {
-            let query_index = rhs.query_index();
-            let distance = rhs.distance();
-            let attribute = rhs.attribute();
-            let word_index = rhs.word_index();
-            matches_proximity(query_index, distance, attribute, word_index)
-        };
-
-        lhs.cmp(&rhs)
-    }
-
-    fn name(&self) -> &str {
-        "WordsProximity"
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn three_different_attributes() {
-        // "soup" "of the" "the day"
-        //
-        // { id: 0, attr: 0, attr_index: 0 }
-        // { id: 1, attr: 1, attr_index: 0 }
-        // { id: 2, attr: 1, attr_index: 1 }
-        // { id: 2, attr: 2, attr_index: 0 }
-        // { id: 3, attr: 3, attr_index: 1 }
-
-        let query_index = &[0, 1, 2, 2, 3];
-        let distance = &[0, 0, 0, 0, 0];
-        let attribute = &[0, 1, 1, 2, 3];
-        let word_index = &[0, 0, 1, 0, 1];
-
-        //   soup -> of = 8
-        // + of -> the  = 1
-        // + the -> day = 8 (not 1)
-        assert_eq!(
-            matches_proximity(query_index, distance, attribute, word_index),
-            17
-        );
-    }
-
-    #[test]
-    fn two_different_attributes() {
-        // "soup day" "soup of the day"
-        //
-        // { id: 0, attr: 0, attr_index: 0 }
-        // { id: 0, attr: 1, attr_index: 0 }
-        // { id: 1, attr: 1, attr_index: 1 }
-        // { id: 2, attr: 1, attr_index: 2 }
-        // { id: 3, attr: 0, attr_index: 1 }
-        // { id: 3, attr: 1, attr_index: 3 }
-
-        let query_index = &[0, 0, 1, 2, 3, 3];
-        let distance = &[0, 0, 0, 0, 0, 0];
-        let attribute = &[0, 1, 1, 1, 0, 1];
-        let word_index = &[0, 0, 1, 2, 1, 3];
-
-        //   soup -> of = 1
-        // + of -> the  = 1
-        // + the -> day = 1
-        assert_eq!(
-            matches_proximity(query_index, distance, attribute, word_index),
-            3
-        );
-    }
-}

meilidb-core/src/database.rs

@@ -1,844 +0,0 @@
-use std::collections::hash_map::{Entry, HashMap};
-use std::fs::File;
-use std::path::Path;
-use std::sync::{Arc, RwLock};
-use std::{fs, thread};
-
-use crossbeam_channel::{Receiver, Sender};
-use heed::types::{Str, Unit};
-use heed::{CompactionOption, Result as ZResult};
-use log::debug;
-
-use crate::{store, update, Index, MResult};
-
-pub type BoxUpdateFn = Box<dyn Fn(&str, update::ProcessedUpdateResult) + Send + Sync + 'static>;
-type ArcSwapFn = arc_swap::ArcSwapOption<BoxUpdateFn>;
-
-pub struct Database {
-    pub env: heed::Env,
-    common_store: heed::PolyDatabase,
-    indexes_store: heed::Database<Str, Unit>,
-    indexes: RwLock<HashMap<String, (Index, thread::JoinHandle<()>)>>,
-    update_fn: Arc<ArcSwapFn>,
-}
-
-macro_rules! r#break_try {
-    ($expr:expr, $msg:tt) => {
-        match $expr {
-            core::result::Result::Ok(val) => val,
-            core::result::Result::Err(err) => {
-                log::error!(concat!($msg, ": {}"), err);
-                break;
-            }
-        }
-    };
-}
-
-pub enum UpdateEvent {
-    NewUpdate,
-    MustStop,
-}
-
-pub type UpdateEvents = Receiver<UpdateEvent>;
-pub type UpdateEventsEmitter = Sender<UpdateEvent>;
-
-fn update_awaiter(
-    receiver: UpdateEvents,
-    env: heed::Env,
-    index_name: &str,
-    update_fn: Arc<ArcSwapFn>,
-    index: Index,
-) {
-    let mut receiver = receiver.into_iter();
-    while let Some(UpdateEvent::NewUpdate) = receiver.next() {
-        loop {
-            // instantiate a main/parent transaction
-            let mut writer = break_try!(env.write_txn(), "LMDB write transaction begin failed");
-
-            // retrieve the update that needs to be processed
-            let result = index.updates.pop_front(&mut writer);
-            let (update_id, update) = match break_try!(result, "pop front update failed") {
-                Some(value) => value,
-                None => {
-                    debug!("no more updates");
-                    writer.abort();
-                    break;
-                }
-            };
-
-            // instantiate a nested transaction
-            let result = env.nested_write_txn(&mut writer);
-            let mut nested_writer = break_try!(result, "LMDB nested write transaction failed");
-
-            // try to apply the update to the database using the nested transaction
-            let result = update::update_task(&mut nested_writer, index.clone(), update_id, update);
-            let status = break_try!(result, "update task failed");
-
-            // commit the nested transaction if the update was successful, abort it otherwise
-            if status.error.is_none() {
-                break_try!(nested_writer.commit(), "commit nested transaction failed");
-            } else {
-                nested_writer.abort()
-            }
-
-            // write the result of the update in the updates-results store
-            let updates_results = index.updates_results;
-            let result = updates_results.put_update_result(&mut writer, update_id, &status);
-
-            // always commit the main/parent transaction, even if the update was unsuccessful
-            break_try!(result, "update result store commit failed");
-            break_try!(writer.commit(), "update parent transaction failed");
-
-            // call the user callback when the update and the result are written consistently
-            if let Some(ref callback) = *update_fn.load() {
-                (callback)(index_name, status);
-            }
-        }
-    }
-
-    debug!("update loop system stopped");
-}
-
-impl Database {
-    pub fn open_or_create(path: impl AsRef<Path>) -> MResult<Database> {
-        fs::create_dir_all(path.as_ref())?;
-
-        let env = heed::EnvOpenOptions::new()
-            .map_size(10 * 1024 * 1024 * 1024) // 10GB
-            .max_dbs(3000)
-            .open(path)?;
-
-        let common_store = env.create_poly_database(Some("common"))?;
-        let indexes_store = env.create_database::<Str, Unit>(Some("indexes"))?;
-        let update_fn = Arc::new(ArcSwapFn::empty());
-
-        // list all indexes that needs to be opened
-        let mut must_open = Vec::new();
-        let reader = env.read_txn()?;
-        for result in indexes_store.iter(&reader)? {
-            let (index_name, _) = result?;
-            must_open.push(index_name.to_owned());
-        }
-
-        reader.abort();
-
-        // open the previously aggregated indexes
-        let mut indexes = HashMap::new();
-        for index_name in must_open {
-            let (sender, receiver) = crossbeam_channel::bounded(100);
-            let index = match store::open(&env, &index_name, sender.clone())? {
-                Some(index) => index,
-                None => {
-                    log::warn!(
-                        "the index {} doesn't exist or has not all the databases",
-                        index_name
-                    );
-                    continue;
-                }
-            };
-
-            let env_clone = env.clone();
-            let index_clone = index.clone();
-            let name_clone = index_name.clone();
-            let update_fn_clone = update_fn.clone();
-
-            let handle = thread::spawn(move || {
-                update_awaiter(
-                    receiver,
-                    env_clone,
-                    &name_clone,
-                    update_fn_clone,
-                    index_clone,
-                )
-            });
-
-            // send an update notification to make sure that
-            // possible pre-boot updates are consumed
-            sender.send(UpdateEvent::NewUpdate).unwrap();
-
-            let result = indexes.insert(index_name, (index, handle));
-            assert!(
-                result.is_none(),
-                "The index should not have been already open"
-            );
-        }
-
-        Ok(Database {
-            env,
-            common_store,
-            indexes_store,
-            indexes: RwLock::new(indexes),
-            update_fn,
-        })
-    }
-
-    pub fn open_index(&self, name: impl AsRef<str>) -> Option<Index> {
-        let indexes_lock = self.indexes.read().unwrap();
-        match indexes_lock.get(name.as_ref()) {
-            Some((index, ..)) => Some(index.clone()),
-            None => None,
-        }
-    }
-
-    pub fn create_index(&self, name: impl AsRef<str>) -> MResult<Index> {
-        let name = name.as_ref();
-        let mut indexes_lock = self.indexes.write().unwrap();
-
-        match indexes_lock.entry(name.to_owned()) {
-            Entry::Occupied(_) => Err(crate::Error::IndexAlreadyExists),
-            Entry::Vacant(entry) => {
-                let (sender, receiver) = crossbeam_channel::bounded(100);
-                let index = store::create(&self.env, name, sender)?;
-
-                let mut writer = self.env.write_txn()?;
-                self.indexes_store.put(&mut writer, name, &())?;
-
-                let env_clone = self.env.clone();
-                let index_clone = index.clone();
-                let name_clone = name.to_owned();
-                let update_fn_clone = self.update_fn.clone();
-
-                let handle = thread::spawn(move || {
-                    update_awaiter(
-                        receiver,
-                        env_clone,
-                        &name_clone,
-                        update_fn_clone,
-                        index_clone,
-                    )
-                });
-
-                writer.commit()?;
-                entry.insert((index.clone(), handle));
-
-                Ok(index)
-            }
-        }
-    }
-
-    pub fn delete_index(&self, name: impl AsRef<str>) -> MResult<bool> {
-        let name = name.as_ref();
-        let mut indexes_lock = self.indexes.write().unwrap();
-
-        match indexes_lock.remove_entry(name) {
-            Some((name, (index, handle))) => {
-                // remove the index name from the list of indexes
-                // and clear all the LMDB dbi
-                let mut writer = self.env.write_txn()?;
-                self.indexes_store.delete(&mut writer, &name)?;
-                store::clear(&mut writer, &index)?;
-                writer.commit()?;
-
-                // join the update loop thread to ensure it is stopped
-                handle.join().unwrap();
-
-                Ok(true)
-            }
-            None => Ok(false),
-        }
-    }
-
-    pub fn set_update_callback(&self, update_fn: BoxUpdateFn) {
-        let update_fn = Some(Arc::new(update_fn));
-        self.update_fn.swap(update_fn);
-    }
-
-    pub fn unset_update_callback(&self) {
-        self.update_fn.swap(None);
-    }
-
-    pub fn copy_and_compact_to_path<P: AsRef<Path>>(&self, path: P) -> ZResult<File> {
-        self.env.copy_to_path(path, CompactionOption::Enabled)
-    }
-
-    pub fn indexes_names(&self) -> MResult<Vec<String>> {
-        let indexes = self.indexes.read().unwrap();
-        Ok(indexes.keys().cloned().collect())
-    }
-
-    pub fn common_store(&self) -> heed::PolyDatabase {
-        self.common_store
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    use crate::update::{ProcessedUpdateResult, UpdateStatus};
-    use crate::DocumentId;
-    use serde::de::IgnoredAny;
-    use std::sync::mpsc;
-
-    #[test]
-    fn valid_updates() {
-        let dir = tempfile::tempdir().unwrap();
-
-        let database = Database::open_or_create(dir.path()).unwrap();
-        let env = &database.env;
-
-        let (sender, receiver) = mpsc::sync_channel(100);
-        let update_fn = move |_name: &str, update: ProcessedUpdateResult| {
-            sender.send(update.update_id).unwrap()
-        };
-        let index = database.create_index("test").unwrap();
-
-        database.set_update_callback(Box::new(update_fn));
-
-        let schema = {
-            let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
-            "#;
-            toml::from_str(data).unwrap()
-        };
-
-        let mut writer = env.write_txn().unwrap();
-        let _update_id = index.schema_update(&mut writer, schema).unwrap();
-        writer.commit().unwrap();
-
-        let mut additions = index.documents_addition();
-
-        let doc1 = serde_json::json!({
-            "id": 123,
-            "name": "Marvin",
-            "description": "My name is Marvin",
-        });
-
-        let doc2 = serde_json::json!({
-            "id": 234,
-            "name": "Kevin",
-            "description": "My name is Kevin",
-        });
-
-        additions.update_document(doc1);
-        additions.update_document(doc2);
-
-        let mut writer = env.write_txn().unwrap();
-        let update_id = additions.finalize(&mut writer).unwrap();
-        writer.commit().unwrap();
-
-        // block until the transaction is processed
-        let _ = receiver.into_iter().find(|id| *id == update_id);
-
-        let reader = env.read_txn().unwrap();
-        let result = index.update_status(&reader, update_id).unwrap();
-        assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-    }
-
-    #[test]
-    fn invalid_updates() {
-        let dir = tempfile::tempdir().unwrap();
-
-        let database = Database::open_or_create(dir.path()).unwrap();
-        let env = &database.env;
-
-        let (sender, receiver) = mpsc::sync_channel(100);
-        let update_fn = move |_name: &str, update: ProcessedUpdateResult| {
-            sender.send(update.update_id).unwrap()
-        };
-        let index = database.create_index("test").unwrap();
-
-        database.set_update_callback(Box::new(update_fn));
-
-        let schema = {
-            let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
-            "#;
-            toml::from_str(data).unwrap()
-        };
-
-        let mut writer = env.write_txn().unwrap();
-        let _update_id = index.schema_update(&mut writer, schema).unwrap();
-        writer.commit().unwrap();
-
-        let mut additions = index.documents_addition();
-
-        let doc1 = serde_json::json!({
-            "id": 123,
-            "name": "Marvin",
-            "description": "My name is Marvin",
-        });
-
-        let doc2 = serde_json::json!({
-            "name": "Kevin",
-            "description": "My name is Kevin",
-        });
-
-        additions.update_document(doc1);
-        additions.update_document(doc2);
-
-        let mut writer = env.write_txn().unwrap();
-        let update_id = additions.finalize(&mut writer).unwrap();
-        writer.commit().unwrap();
-
-        // block until the transaction is processed
-        let _ = receiver.into_iter().find(|id| *id == update_id);
-
-        let reader = env.read_txn().unwrap();
-        let result = index.update_status(&reader, update_id).unwrap();
-        assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_some());
-    }
-
-    #[test]
-    fn ignored_words_too_long() {
-        let dir = tempfile::tempdir().unwrap();
-
-        let database = Database::open_or_create(dir.path()).unwrap();
-        let env = &database.env;
-
-        let (sender, receiver) = mpsc::sync_channel(100);
-        let update_fn = move |_name: &str, update: ProcessedUpdateResult| {
-            sender.send(update.update_id).unwrap()
-        };
-        let index = database.create_index("test").unwrap();
-
-        database.set_update_callback(Box::new(update_fn));
-
-        let schema = {
-            let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-            "#;
-            toml::from_str(data).unwrap()
-        };
-
-        let mut writer = env.write_txn().unwrap();
-        let _update_id = index.schema_update(&mut writer, schema).unwrap();
-        writer.commit().unwrap();
-
-        let mut additions = index.documents_addition();
-
-        let doc1 = serde_json::json!({
-            "id": 123,
-            "name": "s̷̡̢̡̧̺̜̞͕͉͉͕̜͔̟̼̥̝͍̟̖͔͔̪͉̲̹̝̣̖͎̞̤̥͓͎̭̩͕̙̩̿̀̋̅̈́̌́̏̍̄̽͂̆̾̀̿̕̚̚͜͠͠ͅͅļ̵̨̨̨̰̦̻̳̖̳͚̬̫͚̦͖͈̲̫̣̩̥̻̙̦̱̼̠̖̻̼̘̖͉̪̜̠̙͖̙̩͔̖̯̩̲̿̽͋̔̿̍̓͂̍̿͊͆̃͗̔̎͐͌̾̆͗́̆̒̔̾̅̚̚͜͜ͅͅī̵̛̦̅̔̓͂͌̾́͂͛̎̋͐͆̽̂̋̋́̾̀̉̓̏̽́̑̀͒̇͋͛̈́̃̉̏͊̌̄̽̿̏̇͘̕̚̕p̶̧̛̛̖̯̗͕̝̗̭̱͙̖̗̟̟̐͆̊̂͐̋̓̂̈́̓͊̆͌̾̾͐͋͗͌̆̿̅͆̈́̈́̉͋̍͊͗̌̓̅̈̎̇̃̎̈́̉̐̋͑̃͘̕͘d̴̢̨̛͕̘̯͖̭̮̝̝̐̊̈̅̐̀͒̀́̈́̀͌̽͛͆͑̀̽̿͛̃̋̇̎̀́̂́͘͠͝ǫ̵̨̛̮̩̘͚̬̯̖̱͍̼͑͑̓̐́̑̿̈́̔͌̂̄͐͝ģ̶̧̜͇̣̭̺̪̺̖̻͖̮̭̣̙̻͒͊͗̓̓͒̀̀ͅ",
-        });
-
-        additions.update_document(doc1);
-
-        let mut writer = env.write_txn().unwrap();
-        let update_id = additions.finalize(&mut writer).unwrap();
-        writer.commit().unwrap();
-
-        // block until the transaction is processed
-        let _ = receiver.into_iter().find(|id| *id == update_id);
-
-        let reader = env.read_txn().unwrap();
-        let result = index.update_status(&reader, update_id).unwrap();
-        assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-    }
-
-    #[test]
-    fn add_schema_attributes_at_end() {
-        let dir = tempfile::tempdir().unwrap();
-
-        let database = Database::open_or_create(dir.path()).unwrap();
-        let env = &database.env;
-
-        let (sender, receiver) = mpsc::sync_channel(100);
-        let update_fn = move |_name: &str, update: ProcessedUpdateResult| {
-            sender.send(update.update_id).unwrap()
-        };
-        let index = database.create_index("test").unwrap();
-
-        database.set_update_callback(Box::new(update_fn));
-
-        let schema = {
-            let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
-            "#;
-            toml::from_str(data).unwrap()
-        };
-
-        let mut writer = env.write_txn().unwrap();
-        let _update_id = index.schema_update(&mut writer, schema).unwrap();
-        writer.commit().unwrap();
-
-        let mut additions = index.documents_addition();
-
-        let doc1 = serde_json::json!({
-            "id": 123,
-            "name": "Marvin",
-            "description": "My name is Marvin",
-        });
-
-        let doc2 = serde_json::json!({
-            "id": 234,
-            "name": "Kevin",
-            "description": "My name is Kevin",
-        });
-
-        additions.update_document(doc1);
-        additions.update_document(doc2);
-
-        let mut writer = env.write_txn().unwrap();
-        let _update_id = additions.finalize(&mut writer).unwrap();
-        writer.commit().unwrap();
-
-        let schema = {
-            let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
-
-                [attributes."age"]
-                displayed = true
-                indexed = true
-
-                [attributes."sex"]
-                displayed = true
-                indexed = true
-            "#;
-            toml::from_str(data).unwrap()
-        };
-
-        let mut writer = env.write_txn().unwrap();
-        let update_id = index.schema_update(&mut writer, schema).unwrap();
-        writer.commit().unwrap();
-
-        // block until the transaction is processed
-        let _ = receiver.iter().find(|id| *id == update_id);
-
-        // check if it has been accepted
-        let reader = env.read_txn().unwrap();
-        let result = index.update_status(&reader, update_id).unwrap();
-        assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-        reader.abort();
-
-        let mut additions = index.documents_addition();
-
-        let doc1 = serde_json::json!({
-            "id": 123,
-            "name": "Marvin",
-            "description": "My name is Marvin",
-            "age": 21,
-            "sex": "Male",
-        });
-
-        let doc2 = serde_json::json!({
-            "id": 234,
-            "name": "Kevin",
-            "description": "My name is Kevin",
-            "age": 23,
-            "sex": "Male",
-        });
-
-        additions.update_document(doc1);
-        additions.update_document(doc2);
-
-        let mut writer = env.write_txn().unwrap();
-        let update_id = additions.finalize(&mut writer).unwrap();
-        writer.commit().unwrap();
-
-        // block until the transaction is processed
-        let _ = receiver.iter().find(|id| *id == update_id);
-
-        // check if it has been accepted
-        let reader = env.read_txn().unwrap();
-        let result = index.update_status(&reader, update_id).unwrap();
-        assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-
-        // even try to search for a document
-        let results = index.query_builder().query(&reader, "21 ", 0..20).unwrap();
-        assert_matches!(results.len(), 1);
-
-        reader.abort();
-
-        // try to introduce attributes in the middle of the schema
-        let schema = {
-            let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
-
-                [attributes."city"]
-                displayed = true
-                indexed = true
-
-                [attributes."age"]
-                displayed = true
-                indexed = true
-
-                [attributes."sex"]
-                displayed = true
-                indexed = true
-            "#;
-            toml::from_str(data).unwrap()
-        };
-
-        let mut writer = env.write_txn().unwrap();
-        let update_id = index.schema_update(&mut writer, schema).unwrap();
-        writer.commit().unwrap();
-
-        // block until the transaction is processed
-        let _ = receiver.iter().find(|id| *id == update_id);
-
-        // check if it has been accepted
-        let reader = env.read_txn().unwrap();
-        let result = index.update_status(&reader, update_id).unwrap();
-        assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_some());
-    }
-
-    #[test]
-    fn deserialize_documents() {
-        let dir = tempfile::tempdir().unwrap();
-
-        let database = Database::open_or_create(dir.path()).unwrap();
-        let env = &database.env;
-
-        let (sender, receiver) = mpsc::sync_channel(100);
-        let update_fn = move |_name: &str, update: ProcessedUpdateResult| {
-            sender.send(update.update_id).unwrap()
-        };
-        let index = database.create_index("test").unwrap();
-
-        database.set_update_callback(Box::new(update_fn));
-
-        let schema = {
-            let data = r#"
-                identifier = "id"
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
-            "#;
-            toml::from_str(data).unwrap()
-        };
-
-        let mut writer = env.write_txn().unwrap();
-        let _update_id = index.schema_update(&mut writer, schema).unwrap();
-        writer.commit().unwrap();
-
-        let mut additions = index.documents_addition();
-
-        // DocumentId(7900334843754999545)
-        let doc1 = serde_json::json!({
-            "id": 123,
-            "name": "Marvin",
-            "description": "My name is Marvin",
-        });
-
-        // DocumentId(8367468610878465872)
-        let doc2 = serde_json::json!({
-            "id": 234,
-            "name": "Kevin",
-            "description": "My name is Kevin",
-        });
-
-        additions.update_document(doc1);
-        additions.update_document(doc2);
-
-        let mut writer = env.write_txn().unwrap();
-        let update_id = additions.finalize(&mut writer).unwrap();
-        writer.commit().unwrap();
-
-        // block until the transaction is processed
-        let _ = receiver.into_iter().find(|id| *id == update_id);
-
-        let reader = env.read_txn().unwrap();
-        let result = index.update_status(&reader, update_id).unwrap();
-        assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-
-        let document: Option<IgnoredAny> = index.document(&reader, None, DocumentId(25)).unwrap();
-        assert!(document.is_none());
-
-        let document: Option<IgnoredAny> = index
-            .document(&reader, None, DocumentId(7900334843754999545))
-            .unwrap();
-        assert!(document.is_some());
-
-        let document: Option<IgnoredAny> = index
-            .document(&reader, None, DocumentId(8367468610878465872))
-            .unwrap();
-        assert!(document.is_some());
-    }
-
-    #[test]
-    fn partial_document_update() {
-        let dir = tempfile::tempdir().unwrap();
-
-        let database = Database::open_or_create(dir.path()).unwrap();
-        let env = &database.env;
-
-        let (sender, receiver) = mpsc::sync_channel(100);
-        let update_fn = move |_name: &str, update: ProcessedUpdateResult| {
-            sender.send(update.update_id).unwrap()
-        };
-        let index = database.create_index("test").unwrap();
-
-        database.set_update_callback(Box::new(update_fn));
-
-        let schema = {
-            let data = r#"
-                identifier = "id"
-
-                [attributes."id"]
-                displayed = true
-
-                [attributes."name"]
-                displayed = true
-                indexed = true
-
-                [attributes."description"]
-                displayed = true
-                indexed = true
-            "#;
-            toml::from_str(data).unwrap()
-        };
-
-        let mut writer = env.write_txn().unwrap();
-        let _update_id = index.schema_update(&mut writer, schema).unwrap();
-        writer.commit().unwrap();
-
-        let mut additions = index.documents_addition();
-
-        // DocumentId(7900334843754999545)
-        let doc1 = serde_json::json!({
-            "id": 123,
-            "name": "Marvin",
-            "description": "My name is Marvin",
-        });
-
-        // DocumentId(8367468610878465872)
-        let doc2 = serde_json::json!({
-            "id": 234,
-            "name": "Kevin",
-            "description": "My name is Kevin",
-        });
-
-        additions.update_document(doc1);
-        additions.update_document(doc2);
-
-        let mut writer = env.write_txn().unwrap();
-        let update_id = additions.finalize(&mut writer).unwrap();
-        writer.commit().unwrap();
-
-        // block until the transaction is processed
-        let _ = receiver.iter().find(|id| *id == update_id);
-
-        let reader = env.read_txn().unwrap();
-        let result = index.update_status(&reader, update_id).unwrap();
-        assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-
-        let document: Option<IgnoredAny> = index.document(&reader, None, DocumentId(25)).unwrap();
-        assert!(document.is_none());
-
-        let document: Option<IgnoredAny> = index
-            .document(&reader, None, DocumentId(7900334843754999545))
-            .unwrap();
-        assert!(document.is_some());
-
-        let document: Option<IgnoredAny> = index
-            .document(&reader, None, DocumentId(8367468610878465872))
-            .unwrap();
-        assert!(document.is_some());
-
-        reader.abort();
-
-        let mut partial_additions = index.documents_partial_addition();
-
-        // DocumentId(7900334843754999545)
-        let partial_doc1 = serde_json::json!({
-            "id": 123,
-            "description": "I am the new Marvin",
-        });
-
-        // DocumentId(8367468610878465872)
-        let partial_doc2 = serde_json::json!({
-            "id": 234,
-            "description": "I am the new Kevin",
-        });
-
-        partial_additions.update_document(partial_doc1);
-        partial_additions.update_document(partial_doc2);
-
-        let mut writer = env.write_txn().unwrap();
-        let update_id = partial_additions.finalize(&mut writer).unwrap();
-        writer.commit().unwrap();
-
-        // block until the transaction is processed
-        let _ = receiver.iter().find(|id| *id == update_id);
-
-        let reader = env.read_txn().unwrap();
-        let result = index.update_status(&reader, update_id).unwrap();
-        assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-
-        let document: Option<serde_json::Value> = index
-            .document(&reader, None, DocumentId(7900334843754999545))
-            .unwrap();
-
-        let new_doc1 = serde_json::json!({
-            "id": 123,
-            "name": "Marvin",
-            "description": "I am the new Marvin",
-        });
-        assert_eq!(document, Some(new_doc1));
-
-        let document: Option<serde_json::Value> = index
-            .document(&reader, None, DocumentId(8367468610878465872))
-            .unwrap();
-
-        let new_doc2 = serde_json::json!({
-            "id": 234,
-            "name": "Kevin",
-            "description": "I am the new Kevin",
-        });
-        assert_eq!(document, Some(new_doc2));
-    }
-
-    #[test]
-    fn delete_index() {
-        let dir = tempfile::tempdir().unwrap();
-
-        let database = Database::open_or_create(dir.path()).unwrap();
-        let _index = database.create_index("test").unwrap();
-
-        let deleted = database.delete_index("test").unwrap();
-        assert!(deleted);
-
-        let result = database.open_index("test");
-        assert!(result.is_none());
-    }
-}

meilidb-core/src/distinct_map.rs

@@ -1,103 +0,0 @@
-use hashbrown::HashMap;
-use std::hash::Hash;
-
-pub struct DistinctMap<K> {
-    inner: HashMap<K, usize>,
-    limit: usize,
-    len: usize,
-}
-
-impl<K: Hash + Eq> DistinctMap<K> {
-    pub fn new(limit: usize) -> Self {
-        DistinctMap {
-            inner: HashMap::new(),
-            limit,
-            len: 0,
-        }
-    }
-
-    pub fn len(&self) -> usize {
-        self.len
-    }
-}
-
-pub struct BufferedDistinctMap<'a, K> {
-    internal: &'a mut DistinctMap<K>,
-    inner: HashMap<K, usize>,
-    len: usize,
-}
-
-impl<'a, K: Hash + Eq> BufferedDistinctMap<'a, K> {
-    pub fn new(internal: &'a mut DistinctMap<K>) -> BufferedDistinctMap<'a, K> {
-        BufferedDistinctMap {
-            internal,
-            inner: HashMap::new(),
-            len: 0,
-        }
-    }
-
-    pub fn register(&mut self, key: K) -> bool {
-        let internal_seen = self.internal.inner.get(&key).unwrap_or(&0);
-        let inner_seen = self.inner.entry(key).or_insert(0);
-        let seen = *internal_seen + *inner_seen;
-
-        if seen < self.internal.limit {
-            *inner_seen += 1;
-            self.len += 1;
-            true
-        } else {
-            false
-        }
-    }
-
-    pub fn register_without_key(&mut self) -> bool {
-        self.len += 1;
-        true
-    }
-
-    pub fn transfert_to_internal(&mut self) {
-        for (k, v) in self.inner.drain() {
-            let value = self.internal.inner.entry(k).or_insert(0);
-            *value += v;
-        }
-
-        self.internal.len += self.len;
-        self.len = 0;
-    }
-
-    pub fn len(&self) -> usize {
-        self.internal.len() + self.len
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn easy_distinct_map() {
-        let mut map = DistinctMap::new(2);
-        let mut buffered = BufferedDistinctMap::new(&mut map);
-
-        for x in &[1, 1, 1, 2, 3, 4, 5, 6, 6, 6, 6, 6] {
-            buffered.register(x);
-        }
-        buffered.transfert_to_internal();
-        assert_eq!(map.len(), 8);
-
-        let mut map = DistinctMap::new(2);
-        let mut buffered = BufferedDistinctMap::new(&mut map);
-        assert_eq!(buffered.register(1), true);
-        assert_eq!(buffered.register(1), true);
-        assert_eq!(buffered.register(1), false);
-        assert_eq!(buffered.register(1), false);
-
-        assert_eq!(buffered.register(2), true);
-        assert_eq!(buffered.register(3), true);
-        assert_eq!(buffered.register(2), true);
-        assert_eq!(buffered.register(2), false);
-
-        buffered.transfert_to_internal();
-        assert_eq!(map.len(), 5);
-    }
-}

meilidb-core/src/error.rs

@@ -1,117 +0,0 @@
-use crate::serde::{DeserializerError, SerializerError};
-use serde_json::Error as SerdeJsonError;
-use std::{error, fmt, io};
-
-pub type MResult<T> = Result<T, Error>;
-
-#[derive(Debug)]
-pub enum Error {
-    Io(io::Error),
-    IndexAlreadyExists,
-    SchemaDiffer,
-    SchemaMissing,
-    WordIndexMissing,
-    MissingDocumentId,
-    Zlmdb(heed::Error),
-    Fst(fst::Error),
-    SerdeJson(SerdeJsonError),
-    Bincode(bincode::Error),
-    Serializer(SerializerError),
-    Deserializer(DeserializerError),
-    UnsupportedOperation(UnsupportedOperation),
-}
-
-impl From<io::Error> for Error {
-    fn from(error: io::Error) -> Error {
-        Error::Io(error)
-    }
-}
-
-impl From<heed::Error> for Error {
-    fn from(error: heed::Error) -> Error {
-        Error::Zlmdb(error)
-    }
-}
-
-impl From<fst::Error> for Error {
-    fn from(error: fst::Error) -> Error {
-        Error::Fst(error)
-    }
-}
-
-impl From<SerdeJsonError> for Error {
-    fn from(error: SerdeJsonError) -> Error {
-        Error::SerdeJson(error)
-    }
-}
-
-impl From<bincode::Error> for Error {
-    fn from(error: bincode::Error) -> Error {
-        Error::Bincode(error)
-    }
-}
-
-impl From<SerializerError> for Error {
-    fn from(error: SerializerError) -> Error {
-        Error::Serializer(error)
-    }
-}
-
-impl From<DeserializerError> for Error {
-    fn from(error: DeserializerError) -> Error {
-        Error::Deserializer(error)
-    }
-}
-
-impl From<UnsupportedOperation> for Error {
-    fn from(op: UnsupportedOperation) -> Error {
-        Error::UnsupportedOperation(op)
-    }
-}
-
-impl fmt::Display for Error {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        use self::Error::*;
-        match self {
-            Io(e) => write!(f, "{}", e),
-            IndexAlreadyExists => write!(f, "index already exists"),
-            SchemaDiffer => write!(f, "schemas differ"),
-            SchemaMissing => write!(f, "this index does not have a schema"),
-            WordIndexMissing => write!(f, "this index does not have a word index"),
-            MissingDocumentId => write!(f, "document id is missing"),
-            Zlmdb(e) => write!(f, "heed error; {}", e),
-            Fst(e) => write!(f, "fst error; {}", e),
-            SerdeJson(e) => write!(f, "serde json error; {}", e),
-            Bincode(e) => write!(f, "bincode error; {}", e),
-            Serializer(e) => write!(f, "serializer error; {}", e),
-            Deserializer(e) => write!(f, "deserializer error; {}", e),
-            UnsupportedOperation(op) => write!(f, "unsupported operation; {}", op),
-        }
-    }
-}
-
-impl error::Error for Error {}
-
-#[derive(Debug)]
-pub enum UnsupportedOperation {
-    SchemaAlreadyExists,
-    CannotUpdateSchemaIdentifier,
-    CannotReorderSchemaAttribute,
-    CanOnlyIntroduceNewSchemaAttributesAtEnd,
-    CannotRemoveSchemaAttribute,
-}
-
-impl fmt::Display for UnsupportedOperation {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        use self::UnsupportedOperation::*;
-        match self {
-            SchemaAlreadyExists => write!(f, "Cannot update index which already have a schema"),
-            CannotUpdateSchemaIdentifier => write!(f, "Cannot update the identifier of a schema"),
-            CannotReorderSchemaAttribute => write!(f, "Cannot reorder the attributes of a schema"),
-            CanOnlyIntroduceNewSchemaAttributesAtEnd => {
-                write!(f, "Can only introduce new attributes at end of a schema")
-            }
-            CannotRemoveSchemaAttribute => write!(f, "Cannot remove attributes from a schema"),
-        }
-    }
-}

meilidb-core/src/levenshtein.rs

@@ -1,134 +0,0 @@
-use std::cmp::min;
-use std::collections::BTreeMap;
-use std::ops::{Index, IndexMut};
-
-// A simple wrapper around vec so we can get contiguous but index it like it's 2D array.
-struct N2Array<T> {
-    y_size: usize,
-    buf: Vec<T>,
-}
-
-impl<T: Clone> N2Array<T> {
-    fn new(x: usize, y: usize, value: T) -> N2Array<T> {
-        N2Array {
-            y_size: y,
-            buf: vec![value; x * y],
-        }
-    }
-}
-
-impl<T> Index<(usize, usize)> for N2Array<T> {
-    type Output = T;
-
-    #[inline]
-    fn index(&self, (x, y): (usize, usize)) -> &T {
-        &self.buf[(x * self.y_size) + y]
-    }
-}
-
-impl<T> IndexMut<(usize, usize)> for N2Array<T> {
-    #[inline]
-    fn index_mut(&mut self, (x, y): (usize, usize)) -> &mut T {
-        &mut self.buf[(x * self.y_size) + y]
-    }
-}
-
-pub fn prefix_damerau_levenshtein(source: &[u8], target: &[u8]) -> (u32, usize) {
-    let (n, m) = (source.len(), target.len());
-
-    assert!(
-        n <= m,
-        "the source string must be shorter than the target one"
-    );
-
-    if n == 0 {
-        return (m as u32, 0);
-    }
-    if m == 0 {
-        return (n as u32, 0);
-    }
-
-    if n == m && source == target {
-        return (0, m);
-    }
-
-    let inf = n + m;
-    let mut matrix = N2Array::new(n + 2, m + 2, 0);
-
-    matrix[(0, 0)] = inf;
-    for i in 0..n + 1 {
-        matrix[(i + 1, 0)] = inf;
-        matrix[(i + 1, 1)] = i;
-    }
-    for j in 0..m + 1 {
-        matrix[(0, j + 1)] = inf;
-        matrix[(1, j + 1)] = j;
-    }
-
-    let mut last_row = BTreeMap::new();
-
-    for (row, char_s) in source.iter().enumerate() {
-        let mut last_match_col = 0;
-        let row = row + 1;
-
-        for (col, char_t) in target.iter().enumerate() {
-            let col = col + 1;
-            let last_match_row = *last_row.get(&char_t).unwrap_or(&0);
-            let cost = if char_s == char_t { 0 } else { 1 };
-
-            let dist_add = matrix[(row, col + 1)] + 1;
-            let dist_del = matrix[(row + 1, col)] + 1;
-            let dist_sub = matrix[(row, col)] + cost;
-            let dist_trans = matrix[(last_match_row, last_match_col)]
-                + (row - last_match_row - 1)
-                + 1
-                + (col - last_match_col - 1);
-
-            let dist = min(min(dist_add, dist_del), min(dist_sub, dist_trans));
-
-            matrix[(row + 1, col + 1)] = dist;
-
-            if cost == 0 {
-                last_match_col = col;
-            }
-        }
-
-        last_row.insert(char_s, row);
-    }
-
-    let mut minimum = (u32::max_value(), 0);
-
-    for x in n..=m {
-        let dist = matrix[(n + 1, x + 1)] as u32;
-        if dist < minimum.0 {
-            minimum = (dist, x)
-        }
-    }
-
-    minimum
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn matched_length() {
-        let query = "Levenste";
-        let text = "Levenshtein";
-
-        let (dist, length) = prefix_damerau_levenshtein(query.as_bytes(), text.as_bytes());
-        assert_eq!(dist, 1);
-        assert_eq!(&text[..length], "Levenshte");
-    }
-
-    #[test]
-    #[should_panic]
-    fn matched_length_panic() {
-        let query = "Levenshtein";
-        let text = "Levenste";
-
-        // this function will panic if source if longer than target
-        prefix_damerau_levenshtein(query.as_bytes(), text.as_bytes());
-    }
-}

meilidb-core/src/lib.rs

@@ -1,97 +0,0 @@
-#[cfg(test)]
-#[macro_use]
-extern crate assert_matches;
-
-mod automaton;
-pub mod criterion;
-mod database;
-mod distinct_map;
-mod error;
-mod levenshtein;
-mod number;
-mod query_builder;
-mod ranked_map;
-mod raw_document;
-pub mod raw_indexer;
-mod reordered_attrs;
-pub mod serde;
-pub mod store;
-mod update;
-
-pub use self::database::{BoxUpdateFn, Database};
-pub use self::error::{Error, MResult};
-pub use self::number::{Number, ParseNumberError};
-pub use self::ranked_map::RankedMap;
-pub use self::raw_document::RawDocument;
-pub use self::store::Index;
-pub use self::update::{EnqueuedUpdateResult, ProcessedUpdateResult, UpdateStatus, UpdateType};
-pub use meilidb_types::{DocIndex, DocumentId, Highlight};
-
-#[doc(hidden)]
-#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct TmpMatch {
-    pub query_index: u32,
-    pub distance: u8,
-    pub attribute: u16,
-    pub word_index: u16,
-    pub is_exact: bool,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct Document {
-    pub id: DocumentId,
-    pub highlights: Vec<Highlight>,
-
-    #[cfg(test)]
-    pub matches: Vec<TmpMatch>,
-}
-
-impl Document {
-    #[cfg(not(test))]
-    fn from_raw(raw: RawDocument) -> Document {
-        Document {
-            id: raw.id,
-            highlights: raw.highlights,
-        }
-    }
-
-    #[cfg(test)]
-    fn from_raw(raw: RawDocument) -> Document {
-        let len = raw.query_index().len();
-        let mut matches = Vec::with_capacity(len);
-
-        let query_index = raw.query_index();
-        let distance = raw.distance();
-        let attribute = raw.attribute();
-        let word_index = raw.word_index();
-        let is_exact = raw.is_exact();
-
-        for i in 0..len {
-            let match_ = TmpMatch {
-                query_index: query_index[i],
-                distance: distance[i],
-                attribute: attribute[i],
-                word_index: word_index[i],
-                is_exact: is_exact[i],
-            };
-            matches.push(match_);
-        }
-
-        Document {
-            id: raw.id,
-            matches,
-            highlights: raw.highlights,
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::mem;
-
-    #[test]
-    fn docindex_mem_size() {
-        assert_eq!(mem::size_of::<DocIndex>(), 16);
-    }
-}

meilidb-core/src/number.rs

@@ -1,65 +0,0 @@
-use std::fmt;
-use std::num::{ParseFloatError, ParseIntError};
-use std::str::FromStr;
-
-use ordered_float::OrderedFloat;
-use serde::{Deserialize, Serialize};
-
-#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub enum Number {
-    Unsigned(u64),
-    Signed(i64),
-    Float(OrderedFloat<f64>),
-}
-
-impl FromStr for Number {
-    type Err = ParseNumberError;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        let uint_error = match u64::from_str(s) {
-            Ok(unsigned) => return Ok(Number::Unsigned(unsigned)),
-            Err(error) => error,
-        };
-
-        let int_error = match i64::from_str(s) {
-            Ok(signed) => return Ok(Number::Signed(signed)),
-            Err(error) => error,
-        };
-
-        let float_error = match f64::from_str(s) {
-            Ok(float) => return Ok(Number::Float(OrderedFloat(float))),
-            Err(error) => error,
-        };
-
-        Err(ParseNumberError {
-            uint_error,
-            int_error,
-            float_error,
-        })
-    }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ParseNumberError {
-    uint_error: ParseIntError,
-    int_error: ParseIntError,
-    float_error: ParseFloatError,
-}
-
-impl fmt::Display for ParseNumberError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        if self.uint_error == self.int_error {
-            write!(
-                f,
-                "can not parse number: {}, {}",
-                self.uint_error, self.float_error
-            )
-        } else {
-            write!(
-                f,
-                "can not parse number: {}, {}, {}",
-                self.uint_error, self.int_error, self.float_error
-            )
-        }
-    }
-}

meilidb-core/src/query_enhancer.rs

@@ -1,398 +0,0 @@
-use std::ops::Range;
-use std::cmp::Ordering::{Less, Greater, Equal};
-
-/// Return `true` if the specified range can accept the given replacements words.
-/// Returns `false` if the replacements words are already present in the original query
-/// or if there is fewer replacement words than the range to replace.
-//
-//
-// ## Ignored because already present in original
-//
-//     new york city subway
-//     -------- ^^^^
-//   /          \
-//  [new york city]
-//
-//
-// ## Ignored because smaller than the original
-//
-//   new york city subway
-//   -------------
-//   \          /
-//    [new york]
-//
-//
-// ## Accepted because bigger than the original
-//
-//        NYC subway
-//        ---
-//       /   \
-//      /     \
-//     /       \
-//    /         \
-//   /           \
-//  [new york city]
-//
-fn rewrite_range_with<S, T>(query: &[S], range: Range<usize>, words: &[T]) -> bool
-where S: AsRef<str>,
-      T: AsRef<str>,
-{
-    if words.len() <= range.len() {
-        // there is fewer or equal replacement words
-        // than there is already in the replaced range
-        return false
-    }
-
-    // retrieve the part to rewrite but with the length
-    // of the replacement part
-    let original = query.iter().skip(range.start).take(words.len());
-
-    // check if the original query doesn't already contain
-    // the replacement words
-    !original.map(AsRef::as_ref).eq(words.iter().map(AsRef::as_ref))
-}
-
-type Origin = usize;
-type RealLength = usize;
-
-struct FakeIntervalTree {
-    intervals: Vec<(Range<usize>, (Origin, RealLength))>,
-}
-
-impl FakeIntervalTree {
-    fn new(mut intervals: Vec<(Range<usize>, (Origin, RealLength))>) -> FakeIntervalTree {
-        intervals.sort_unstable_by_key(|(r, _)| (r.start, r.end));
-        FakeIntervalTree { intervals }
-    }
-
-    fn query(&self, point: usize) -> Option<(Range<usize>, (Origin, RealLength))> {
-        let element = self.intervals.binary_search_by(|(r, _)| {
-            if point >= r.start {
-                if point < r.end { Equal } else { Less }
-            } else { Greater }
-        });
-
-        let n = match element { Ok(n) => n, Err(n) => n };
-
-        match self.intervals.get(n) {
-            Some((range, value)) if range.contains(&point) => Some((range.clone(), *value)),
-            _otherwise => None,
-        }
-    }
-}
-
-pub struct QueryEnhancerBuilder<'a, S> {
-    query: &'a [S],
-    origins: Vec<usize>,
-    real_to_origin: Vec<(Range<usize>, (Origin, RealLength))>,
-}
-
-impl<S: AsRef<str>> QueryEnhancerBuilder<'_, S> {
-    pub fn new(query: &[S]) -> QueryEnhancerBuilder<S> {
-        // we initialize origins query indices based on their positions
-        let origins: Vec<_> = (0..query.len() + 1).collect();
-        let real_to_origin = origins.iter().map(|&o| (o..o+1, (o, 1))).collect();
-
-        QueryEnhancerBuilder { query, origins, real_to_origin }
-    }
-
-    /// Update the final real to origin query indices mapping.
-    ///
-    /// `range` is the original words range that this `replacement` words replace
-    /// and `real` is the first real query index of these replacement words.
-    pub fn declare<T>(&mut self, range: Range<usize>, real: usize, replacement: &[T])
-    where T: AsRef<str>,
-    {
-        // check if the range of original words
-        // can be rewritten with the replacement words
-        if rewrite_range_with(self.query, range.clone(), replacement) {
-
-            // this range can be replaced so we need to
-            // modify the origins accordingly
-            let offset = replacement.len() - range.len();
-
-            let previous_padding = self.origins[range.end - 1];
-            let current_offset = (self.origins[range.end] - 1) - previous_padding;
-            let diff = offset.saturating_sub(current_offset);
-            self.origins[range.end] += diff;
-
-            for r in &mut self.origins[range.end + 1..] {
-                *r += diff;
-            }
-        }
-
-        // we need to store the real number and origins relations
-        // this way it will be possible to know by how many
-        // we need to pad real query indices
-        let real_range = real..real + replacement.len().max(range.len());
-        let real_length = replacement.len();
-        self.real_to_origin.push((real_range, (range.start, real_length)));
-    }
-
-    pub fn build(self) -> QueryEnhancer {
-        QueryEnhancer {
-            origins: self.origins,
-            real_to_origin: FakeIntervalTree::new(self.real_to_origin),
-        }
-    }
-}
-
-pub struct QueryEnhancer {
-    origins: Vec<usize>,
-    real_to_origin: FakeIntervalTree,
-}
-
-impl QueryEnhancer {
-    /// Returns the query indices to use to replace this real query index.
-    pub fn replacement(&self, real: u32) -> Range<u32> {
-        let real = real as usize;
-
-        // query the fake interval tree with the real query index
-        let (range, (origin, real_length)) =
-            self.real_to_origin
-                .query(real)
-                .expect("real has never been declared");
-
-        // if `real` is the end bound of the range
-        if (range.start + real_length - 1) == real {
-            let mut count = range.len();
-            let mut new_origin = origin;
-            for (i, slice) in self.origins[new_origin..].windows(2).enumerate() {
-                let len = slice[1] - slice[0];
-                count = count.saturating_sub(len);
-                if count == 0 { new_origin = origin + i; break }
-            }
-
-            let n = real - range.start;
-            let start = self.origins[origin];
-            let end = self.origins[new_origin + 1];
-            let remaining = (end - start) - n;
-
-            Range { start: (start + n) as u32, end: (start + n + remaining) as u32 }
-
-        } else {
-            // just return the origin along with
-            // the real position of the word
-            let n = real as usize - range.start;
-            let origin = self.origins[origin];
-
-            Range { start: (origin + n) as u32, end: (origin + n + 1) as u32 }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn original_unmodified() {
-        let query = ["new", "york", "city", "subway"];
-        //             0       1       2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // new york = new york city
-        builder.declare(0..2, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // new
-        assert_eq!(enhancer.replacement(1), 1..2); // york
-        assert_eq!(enhancer.replacement(2), 2..3); // city
-        assert_eq!(enhancer.replacement(3), 3..4); // subway
-        assert_eq!(enhancer.replacement(4), 0..1); // new
-        assert_eq!(enhancer.replacement(5), 1..2); // york
-        assert_eq!(enhancer.replacement(6), 2..3); // city
-    }
-
-    #[test]
-    fn simple_growing() {
-        let query = ["new", "york", "subway"];
-        //             0       1        2
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // new york = new york city
-        builder.declare(0..2, 3, &["new", "york", "city"]);
-        //                    ^      3       4       5
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // new
-        assert_eq!(enhancer.replacement(1), 1..3); // york
-        assert_eq!(enhancer.replacement(2), 3..4); // subway
-        assert_eq!(enhancer.replacement(3), 0..1); // new
-        assert_eq!(enhancer.replacement(4), 1..2); // york
-        assert_eq!(enhancer.replacement(5), 2..3); // city
-    }
-
-    #[test]
-    fn same_place_growings() {
-        let query = ["NY", "subway"];
-        //             0       1
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NY = new york
-        builder.declare(0..1, 2, &["new", "york"]);
-        //                    ^      2       3
-
-        // NY = new york city
-        builder.declare(0..1, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // NY = NYC
-        builder.declare(0..1, 7, &["NYC"]);
-        //                    ^      7
-
-        // NY = new york city
-        builder.declare(0..1, 8, &["new", "york", "city"]);
-        //                    ^      8       9      10
-
-        // subway = underground train
-        builder.declare(1..2, 11, &["underground", "train"]);
-        //                    ^          11          12
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..3); // NY
-        assert_eq!(enhancer.replacement(1), 3..5); // subway
-        assert_eq!(enhancer.replacement(2), 0..1); // new
-        assert_eq!(enhancer.replacement(3), 1..3); // york
-        assert_eq!(enhancer.replacement(4), 0..1); // new
-        assert_eq!(enhancer.replacement(5), 1..2); // york
-        assert_eq!(enhancer.replacement(6), 2..3); // city
-        assert_eq!(enhancer.replacement(7), 0..3); // NYC
-        assert_eq!(enhancer.replacement(8), 0..1); // new
-        assert_eq!(enhancer.replacement(9), 1..2); // york
-        assert_eq!(enhancer.replacement(10), 2..3); // city
-        assert_eq!(enhancer.replacement(11), 3..4); // underground
-        assert_eq!(enhancer.replacement(12), 4..5); // train
-    }
-
-    #[test]
-    fn bigger_growing() {
-        let query = ["NYC", "subway"];
-        //             0        1
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(0..1, 2, &["new", "york", "city"]);
-        //                    ^      2       3       4
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..3); // NYC
-        assert_eq!(enhancer.replacement(1), 3..4); // subway
-        assert_eq!(enhancer.replacement(2), 0..1); // new
-        assert_eq!(enhancer.replacement(3), 1..2); // york
-        assert_eq!(enhancer.replacement(4), 2..3); // city
-    }
-
-    #[test]
-    fn middle_query_growing() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // great
-        assert_eq!(enhancer.replacement(1), 1..2); // awesome
-        assert_eq!(enhancer.replacement(2), 2..5); // NYC
-        assert_eq!(enhancer.replacement(3), 5..6); // subway
-        assert_eq!(enhancer.replacement(4), 2..3); // new
-        assert_eq!(enhancer.replacement(5), 3..4); // york
-        assert_eq!(enhancer.replacement(6), 4..5); // city
-    }
-
-    #[test]
-    fn end_query_growing() {
-        let query = ["NYC", "subway"];
-        //             0        1
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(1..2, 2, &["underground", "train"]);
-        //                    ^         2            3
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // NYC
-        assert_eq!(enhancer.replacement(1), 1..3); // subway
-        assert_eq!(enhancer.replacement(2), 1..2); // underground
-        assert_eq!(enhancer.replacement(3), 2..3); // train
-    }
-
-    #[test]
-    fn multiple_growings() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // subway = underground train
-        builder.declare(3..4, 7, &["underground", "train"]);
-        //                    ^          7           8
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0), 0..1); // great
-        assert_eq!(enhancer.replacement(1), 1..2); // awesome
-        assert_eq!(enhancer.replacement(2), 2..5); // NYC
-        assert_eq!(enhancer.replacement(3), 5..7); // subway
-        assert_eq!(enhancer.replacement(4), 2..3); // new
-        assert_eq!(enhancer.replacement(5), 3..4); // york
-        assert_eq!(enhancer.replacement(6), 4..5); // city
-        assert_eq!(enhancer.replacement(7), 5..6); // underground
-        assert_eq!(enhancer.replacement(8), 6..7); // train
-    }
-
-    #[test]
-    fn multiple_probable_growings() {
-        let query = ["great", "awesome", "NYC", "subway"];
-        //              0         1        2        3
-        let mut builder = QueryEnhancerBuilder::new(&query);
-
-        // NYC = new york city
-        builder.declare(2..3, 4, &["new", "york", "city"]);
-        //                    ^      4       5       6
-
-        // subway = underground train
-        builder.declare(3..4, 7, &["underground", "train"]);
-        //                    ^          7           8
-
-        // great awesome = good
-        builder.declare(0..2, 9, &["good"]);
-        //                    ^       9
-
-        // awesome NYC = NY
-        builder.declare(1..3, 10, &["NY"]);
-        //                    ^^     10
-
-        // NYC subway = metro
-        builder.declare(2..4, 11, &["metro"]);
-        //                    ^^      11
-
-        let enhancer = builder.build();
-
-        assert_eq!(enhancer.replacement(0),  0..1); // great
-        assert_eq!(enhancer.replacement(1),  1..2); // awesome
-        assert_eq!(enhancer.replacement(2),  2..5); // NYC
-        assert_eq!(enhancer.replacement(3),  5..7); // subway
-        assert_eq!(enhancer.replacement(4),  2..3); // new
-        assert_eq!(enhancer.replacement(5),  3..4); // york
-        assert_eq!(enhancer.replacement(6),  4..5); // city
-        assert_eq!(enhancer.replacement(7),  5..6); // underground
-        assert_eq!(enhancer.replacement(8),  6..7); // train
-        assert_eq!(enhancer.replacement(9),  0..2); // good
-        assert_eq!(enhancer.replacement(10), 1..5); // NY
-        assert_eq!(enhancer.replacement(11), 2..5); // metro
-    }
-}

meilidb-core/src/ranked_map.rs

@@ -1,41 +0,0 @@
-use std::io::{Read, Write};
-
-use hashbrown::HashMap;
-use meilidb_schema::SchemaAttr;
-use serde::{Deserialize, Serialize};
-
-use crate::{DocumentId, Number};
-
-#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(transparent)]
-pub struct RankedMap(HashMap<(DocumentId, SchemaAttr), Number>);
-
-impl RankedMap {
-    pub fn len(&self) -> usize {
-        self.0.len()
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.0.is_empty()
-    }
-
-    pub fn insert(&mut self, document: DocumentId, attribute: SchemaAttr, number: Number) {
-        self.0.insert((document, attribute), number);
-    }
-
-    pub fn remove(&mut self, document: DocumentId, attribute: SchemaAttr) {
-        self.0.remove(&(document, attribute));
-    }
-
-    pub fn get(&self, document: DocumentId, attribute: SchemaAttr) -> Option<Number> {
-        self.0.get(&(document, attribute)).cloned()
-    }
-
-    pub fn read_from_bin<R: Read>(reader: R) -> bincode::Result<RankedMap> {
-        bincode::deserialize_from(reader).map(RankedMap)
-    }
-
-    pub fn write_to_bin<W: Write>(&self, writer: W) -> bincode::Result<()> {
-        bincode::serialize_into(writer, &self.0)
-    }
-}

meilidb-core/src/raw_document.rs

@@ -1,186 +0,0 @@
-use std::fmt;
-use std::sync::Arc;
-
-use meilidb_schema::SchemaAttr;
-use sdset::SetBuf;
-use slice_group_by::GroupBy;
-
-use crate::{DocumentId, Highlight, TmpMatch};
-
-#[derive(Clone)]
-pub struct RawDocument {
-    pub id: DocumentId,
-    pub matches: SharedMatches,
-    pub highlights: Vec<Highlight>,
-    pub fields_counts: SetBuf<(SchemaAttr, u64)>,
-}
-
-impl RawDocument {
-    pub fn query_index(&self) -> &[u32] {
-        let r = self.matches.range;
-        // it is safe because construction/modifications
-        // can only be done in this module
-        unsafe {
-            &self
-                .matches
-                .matches
-                .query_index
-                .get_unchecked(r.start..r.end)
-        }
-    }
-
-    pub fn distance(&self) -> &[u8] {
-        let r = self.matches.range;
-        // it is safe because construction/modifications
-        // can only be done in this module
-        unsafe { &self.matches.matches.distance.get_unchecked(r.start..r.end) }
-    }
-
-    pub fn attribute(&self) -> &[u16] {
-        let r = self.matches.range;
-        // it is safe because construction/modifications
-        // can only be done in this module
-        unsafe { &self.matches.matches.attribute.get_unchecked(r.start..r.end) }
-    }
-
-    pub fn word_index(&self) -> &[u16] {
-        let r = self.matches.range;
-        // it is safe because construction/modifications
-        // can only be done in this module
-        unsafe {
-            &self
-                .matches
-                .matches
-                .word_index
-                .get_unchecked(r.start..r.end)
-        }
-    }
-
-    pub fn is_exact(&self) -> &[bool] {
-        let r = self.matches.range;
-        // it is safe because construction/modifications
-        // can only be done in this module
-        unsafe { &self.matches.matches.is_exact.get_unchecked(r.start..r.end) }
-    }
-}
-
-impl fmt::Debug for RawDocument {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        f.write_str("RawDocument {\r\n")?;
-        f.write_fmt(format_args!("{:>15}: {:?},\r\n", "id", self.id))?;
-        f.write_fmt(format_args!(
-            "{:>15}: {:^5?},\r\n",
-            "query_index",
-            self.query_index()
-        ))?;
-        f.write_fmt(format_args!(
-            "{:>15}: {:^5?},\r\n",
-            "distance",
-            self.distance()
-        ))?;
-        f.write_fmt(format_args!(
-            "{:>15}: {:^5?},\r\n",
-            "attribute",
-            self.attribute()
-        ))?;
-        f.write_fmt(format_args!(
-            "{:>15}: {:^5?},\r\n",
-            "word_index",
-            self.word_index()
-        ))?;
-        f.write_fmt(format_args!(
-            "{:>15}: {:^5?},\r\n",
-            "is_exact",
-            self.is_exact()
-        ))?;
-        f.write_str("}")?;
-        Ok(())
-    }
-}
-
-pub fn raw_documents_from(
-    matches: SetBuf<(DocumentId, TmpMatch)>,
-    highlights: SetBuf<(DocumentId, Highlight)>,
-    fields_counts: SetBuf<(DocumentId, SchemaAttr, u64)>,
-) -> Vec<RawDocument> {
-    let mut docs_ranges: Vec<(_, Range, _, _)> = Vec::new();
-    let mut matches2 = Matches::with_capacity(matches.len());
-
-    let matches = matches.linear_group_by_key(|(id, _)| *id);
-    let highlights = highlights.linear_group_by_key(|(id, _)| *id);
-    let fields_counts = fields_counts.linear_group_by_key(|(id, _, _)| *id);
-
-    for ((mgroup, hgroup), fgroup) in matches.zip(highlights).zip(fields_counts) {
-        debug_assert_eq!(mgroup[0].0, hgroup[0].0);
-        debug_assert_eq!(mgroup[0].0, fgroup[0].0);
-
-        let document_id = mgroup[0].0;
-        let start = docs_ranges.last().map(|(_, r, _, _)| r.end).unwrap_or(0);
-        let end = start + mgroup.len();
-        let highlights = hgroup.iter().map(|(_, h)| *h).collect();
-        let fields_counts = SetBuf::new(fgroup.iter().map(|(_, a, c)| (*a, *c)).collect()).unwrap();
-
-        docs_ranges.push((document_id, Range { start, end }, highlights, fields_counts));
-        matches2.extend_from_slice(mgroup);
-    }
-
-    let matches = Arc::new(matches2);
-    docs_ranges
-        .into_iter()
-        .map(|(id, range, highlights, fields_counts)| {
-            let matches = SharedMatches {
-                range,
-                matches: matches.clone(),
-            };
-            RawDocument {
-                id,
-                matches,
-                highlights,
-                fields_counts,
-            }
-        })
-        .collect()
-}
-
-#[derive(Debug, Copy, Clone)]
-struct Range {
-    start: usize,
-    end: usize,
-}
-
-#[derive(Clone)]
-pub struct SharedMatches {
-    range: Range,
-    matches: Arc<Matches>,
-}
-
-#[derive(Clone)]
-struct Matches {
-    query_index: Vec<u32>,
-    distance: Vec<u8>,
-    attribute: Vec<u16>,
-    word_index: Vec<u16>,
-    is_exact: Vec<bool>,
-}
-
-impl Matches {
-    fn with_capacity(cap: usize) -> Matches {
-        Matches {
-            query_index: Vec::with_capacity(cap),
-            distance: Vec::with_capacity(cap),
-            attribute: Vec::with_capacity(cap),
-            word_index: Vec::with_capacity(cap),
-            is_exact: Vec::with_capacity(cap),
-        }
-    }
-
-    fn extend_from_slice(&mut self, matches: &[(DocumentId, TmpMatch)]) {
-        for (_, match_) in matches {
-            self.query_index.push(match_.query_index);
-            self.distance.push(match_.distance);
-            self.attribute.push(match_.attribute);
-            self.word_index.push(match_.word_index);
-            self.is_exact.push(match_.is_exact);
-        }
-    }
-}

meilidb-core/src/raw_indexer.rs

@@ -1,271 +0,0 @@
-use std::collections::{BTreeMap, HashMap};
-use std::convert::TryFrom;
-
-use crate::{DocIndex, DocumentId};
-use deunicode::deunicode_with_tofu;
-use meilidb_schema::SchemaAttr;
-use meilidb_tokenizer::{is_cjk, SeqTokenizer, Token, Tokenizer};
-use sdset::SetBuf;
-
-const WORD_LENGTH_LIMIT: usize = 80;
-
-type Word = Vec<u8>; // TODO make it be a SmallVec
-
-pub struct RawIndexer {
-    word_limit: usize, // the maximum number of indexed words
-    stop_words: fst::Set,
-    words_doc_indexes: BTreeMap<Word, Vec<DocIndex>>,
-    docs_words: HashMap<DocumentId, Vec<Word>>,
-}
-
-pub struct Indexed {
-    pub words_doc_indexes: BTreeMap<Word, SetBuf<DocIndex>>,
-    pub docs_words: HashMap<DocumentId, fst::Set>,
-}
-
-impl RawIndexer {
-    pub fn new(stop_words: fst::Set) -> RawIndexer {
-        RawIndexer::with_word_limit(stop_words, 1000)
-    }
-
-    pub fn with_word_limit(stop_words: fst::Set, limit: usize) -> RawIndexer {
-        RawIndexer {
-            word_limit: limit,
-            stop_words,
-            words_doc_indexes: BTreeMap::new(),
-            docs_words: HashMap::new(),
-        }
-    }
-
-    pub fn index_text(&mut self, id: DocumentId, attr: SchemaAttr, text: &str) -> usize {
-        let mut number_of_words = 0;
-
-        for token in Tokenizer::new(text) {
-            let must_continue = index_token(
-                token,
-                id,
-                attr,
-                self.word_limit,
-                &self.stop_words,
-                &mut self.words_doc_indexes,
-                &mut self.docs_words,
-            );
-
-            number_of_words += 1;
-
-            if !must_continue {
-                break;
-            }
-        }
-
-        number_of_words
-    }
-
-    pub fn index_text_seq<'a, I>(&mut self, id: DocumentId, attr: SchemaAttr, iter: I)
-    where
-        I: IntoIterator<Item = &'a str>,
-    {
-        let iter = iter.into_iter();
-        for token in SeqTokenizer::new(iter) {
-            let must_continue = index_token(
-                token,
-                id,
-                attr,
-                self.word_limit,
-                &self.stop_words,
-                &mut self.words_doc_indexes,
-                &mut self.docs_words,
-            );
-
-            if !must_continue {
-                break;
-            }
-        }
-    }
-
-    pub fn build(self) -> Indexed {
-        let words_doc_indexes = self
-            .words_doc_indexes
-            .into_iter()
-            .map(|(word, indexes)| (word, SetBuf::from_dirty(indexes)))
-            .collect();
-
-        let docs_words = self
-            .docs_words
-            .into_iter()
-            .map(|(id, mut words)| {
-                words.sort_unstable();
-                words.dedup();
-                (id, fst::Set::from_iter(words).unwrap())
-            })
-            .collect();
-
-        Indexed {
-            words_doc_indexes,
-            docs_words,
-        }
-    }
-}
-
-fn index_token(
-    token: Token,
-    id: DocumentId,
-    attr: SchemaAttr,
-    word_limit: usize,
-    stop_words: &fst::Set,
-    words_doc_indexes: &mut BTreeMap<Word, Vec<DocIndex>>,
-    docs_words: &mut HashMap<DocumentId, Vec<Word>>,
-) -> bool {
-    if token.word_index >= word_limit {
-        return false;
-    }
-
-    let lower = token.word.to_lowercase();
-    let token = Token {
-        word: &lower,
-        ..token
-    };
-
-    if !stop_words.contains(&token.word) {
-        match token_to_docindex(id, attr, token) {
-            Some(docindex) => {
-                let word = Vec::from(token.word);
-
-                if word.len() <= WORD_LENGTH_LIMIT {
-                    words_doc_indexes
-                        .entry(word.clone())
-                        .or_insert_with(Vec::new)
-                        .push(docindex);
-                    docs_words.entry(id).or_insert_with(Vec::new).push(word);
-
-                    if !lower.contains(is_cjk) {
-                        let unidecoded = deunicode_with_tofu(&lower, "");
-                        if unidecoded != lower && !unidecoded.is_empty() {
-                            let word = Vec::from(unidecoded);
-                            if word.len() <= WORD_LENGTH_LIMIT {
-                                words_doc_indexes
-                                    .entry(word.clone())
-                                    .or_insert_with(Vec::new)
-                                    .push(docindex);
-                                docs_words.entry(id).or_insert_with(Vec::new).push(word);
-                            }
-                        }
-                    }
-                }
-            }
-            None => return false,
-        }
-    }
-
-    true
-}
-
-fn token_to_docindex(id: DocumentId, attr: SchemaAttr, token: Token) -> Option<DocIndex> {
-    let word_index = u16::try_from(token.word_index).ok()?;
-    let char_index = u16::try_from(token.char_index).ok()?;
-    let char_length = u16::try_from(token.word.chars().count()).ok()?;
-
-    let docindex = DocIndex {
-        document_id: id,
-        attribute: attr.0,
-        word_index,
-        char_index,
-        char_length,
-    };
-
-    Some(docindex)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn strange_apostrophe() {
-        let mut indexer = RawIndexer::new(fst::Set::default());
-
-        let docid = DocumentId(0);
-        let attr = SchemaAttr(0);
-        let text = "Zut, l’aspirateur, j’ai oublié de l’éteindre !";
-        indexer.index_text(docid, attr, text);
-
-        let Indexed {
-            words_doc_indexes, ..
-        } = indexer.build();
-
-        assert!(words_doc_indexes.get(&b"l"[..]).is_some());
-        assert!(words_doc_indexes.get(&b"aspirateur"[..]).is_some());
-        assert!(words_doc_indexes.get(&b"ai"[..]).is_some());
-        assert!(words_doc_indexes.get(&b"eteindre"[..]).is_some());
-        assert!(words_doc_indexes
-            .get(&"éteindre".to_owned().into_bytes())
-            .is_some());
-    }
-
-    #[test]
-    fn strange_apostrophe_in_sequence() {
-        let mut indexer = RawIndexer::new(fst::Set::default());
-
-        let docid = DocumentId(0);
-        let attr = SchemaAttr(0);
-        let text = vec!["Zut, l’aspirateur, j’ai oublié de l’éteindre !"];
-        indexer.index_text_seq(docid, attr, text);
-
-        let Indexed {
-            words_doc_indexes, ..
-        } = indexer.build();
-
-        assert!(words_doc_indexes.get(&b"l"[..]).is_some());
-        assert!(words_doc_indexes.get(&b"aspirateur"[..]).is_some());
-        assert!(words_doc_indexes.get(&b"ai"[..]).is_some());
-        assert!(words_doc_indexes.get(&b"eteindre"[..]).is_some());
-        assert!(words_doc_indexes
-            .get(&"éteindre".to_owned().into_bytes())
-            .is_some());
-    }
-
-    #[test]
-    fn basic_stop_words() {
-        let stop_words = sdset::SetBuf::from_dirty(vec!["l", "j", "ai", "de"]);
-        let stop_words = fst::Set::from_iter(stop_words).unwrap();
-
-        let mut indexer = RawIndexer::new(stop_words);
-
-        let docid = DocumentId(0);
-        let attr = SchemaAttr(0);
-        let text = "Zut, l’aspirateur, j’ai oublié de l’éteindre !";
-        indexer.index_text(docid, attr, text);
-
-        let Indexed {
-            words_doc_indexes, ..
-        } = indexer.build();
-
-        assert!(words_doc_indexes.get(&b"l"[..]).is_none());
-        assert!(words_doc_indexes.get(&b"aspirateur"[..]).is_some());
-        assert!(words_doc_indexes.get(&b"j"[..]).is_none());
-        assert!(words_doc_indexes.get(&b"ai"[..]).is_none());
-        assert!(words_doc_indexes.get(&b"de"[..]).is_none());
-        assert!(words_doc_indexes.get(&b"eteindre"[..]).is_some());
-        assert!(words_doc_indexes
-            .get(&"éteindre".to_owned().into_bytes())
-            .is_some());
-    }
-
-    #[test]
-    fn no_empty_unidecode() {
-        let mut indexer = RawIndexer::new(fst::Set::default());
-
-        let docid = DocumentId(0);
-        let attr = SchemaAttr(0);
-        let text = "🇯🇵";
-        indexer.index_text(docid, attr, text);
-
-        let Indexed {
-            words_doc_indexes, ..
-        } = indexer.build();
-
-        assert!(words_doc_indexes
-            .get(&"🇯🇵".to_owned().into_bytes())
-            .is_some());
-    }
-}

meilidb-core/src/reordered_attrs.rs

@@ -1,27 +0,0 @@
-#[derive(Default, Clone)]
-pub struct ReorderedAttrs {
-    count: usize,
-    reorders: Vec<Option<u16>>,
-}
-
-impl ReorderedAttrs {
-    pub fn new() -> ReorderedAttrs {
-        ReorderedAttrs {
-            count: 0,
-            reorders: Vec::new(),
-        }
-    }
-
-    pub fn insert_attribute(&mut self, attribute: u16) {
-        self.reorders.resize(attribute as usize + 1, None);
-        self.reorders[attribute as usize] = Some(self.count as u16);
-        self.count += 1;
-    }
-
-    pub fn get(&self, attribute: u16) -> Option<u16> {
-        match self.reorders.get(attribute as usize) {
-            Some(Some(attribute)) => Some(*attribute),
-            _ => None,
-        }
-    }
-}

meilidb-core/src/serde/convert_to_number.rs

@@ -1,198 +0,0 @@
-use std::str::FromStr;
-
-use ordered_float::OrderedFloat;
-use serde::ser;
-use serde::Serialize;
-
-use super::SerializerError;
-use crate::Number;
-
-pub struct ConvertToNumber;
-
-impl ser::Serializer for ConvertToNumber {
-    type Ok = Number;
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    fn serialize_bool(self, value: bool) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_char(self, _value: char) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "char" })
-    }
-
-    fn serialize_i8(self, value: i8) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(i64::from(value)))
-    }
-
-    fn serialize_i16(self, value: i16) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(i64::from(value)))
-    }
-
-    fn serialize_i32(self, value: i32) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(i64::from(value)))
-    }
-
-    fn serialize_i64(self, value: i64) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(value))
-    }
-
-    fn serialize_u8(self, value: u8) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_u16(self, value: u16) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_u32(self, value: u32) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_u64(self, value: u64) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(value))
-    }
-
-    fn serialize_f32(self, value: f32) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Float(OrderedFloat(f64::from(value))))
-    }
-
-    fn serialize_f64(self, value: f64) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Float(OrderedFloat(value)))
-    }
-
-    fn serialize_str(self, value: &str) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::from_str(value)?)
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnrankableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnrankableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "map" })
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "struct",
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "struct variant",
-        })
-    }
-}

meilidb-core/src/serde/convert_to_string.rs

@@ -1,258 +0,0 @@
-use serde::ser;
-use serde::Serialize;
-
-use super::SerializerError;
-
-pub struct ConvertToString;
-
-impl ser::Serializer for ConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = MapConvertToString;
-    type SerializeStruct = StructConvertToString;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    fn serialize_bool(self, _value: bool) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "boolean",
-        })
-    }
-
-    fn serialize_char(self, value: char) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i8(self, value: i8) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i16(self, value: i16) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i32(self, value: i32) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i64(self, value: i64) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u8(self, value: u8) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u16(self, value: u16) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u32(self, value: u32) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u64(self, value: u64) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_f32(self, value: f32) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_f64(self, value: f64) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_str(self, value: &str) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        Ok(MapConvertToString {
-            text: String::new(),
-        })
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Ok(StructConvertToString {
-            text: String::new(),
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct MapConvertToString {
-    text: String,
-}
-
-impl ser::SerializeMap for MapConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = key.serialize(ConvertToString)?;
-        self.text.push_str(&text);
-        self.text.push_str(" ");
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.text.push_str(&text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.text)
-    }
-}
-
-pub struct StructConvertToString {
-    text: String,
-}
-
-impl ser::SerializeStruct for StructConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let value = value.serialize(ConvertToString)?;
-        self.text.push_str(key);
-        self.text.push_str(" ");
-        self.text.push_str(&value);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.text)
-    }
-}

meilidb-core/src/serde/deserializer.rs

@@ -1,158 +0,0 @@
-use std::collections::HashSet;
-use std::io::Cursor;
-use std::{error::Error, fmt};
-
-use meilidb_schema::{Schema, SchemaAttr};
-use serde::{de, forward_to_deserialize_any};
-use serde_json::de::IoRead as SerdeJsonIoRead;
-use serde_json::Deserializer as SerdeJsonDeserializer;
-use serde_json::Error as SerdeJsonError;
-
-use crate::store::DocumentsFields;
-use crate::DocumentId;
-
-#[derive(Debug)]
-pub enum DeserializerError {
-    SerdeJson(SerdeJsonError),
-    Zlmdb(heed::Error),
-    Custom(String),
-}
-
-impl de::Error for DeserializerError {
-    fn custom<T: fmt::Display>(msg: T) -> Self {
-        DeserializerError::Custom(msg.to_string())
-    }
-}
-
-impl fmt::Display for DeserializerError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            DeserializerError::SerdeJson(e) => write!(f, "serde json related error: {}", e),
-            DeserializerError::Zlmdb(e) => write!(f, "heed related error: {}", e),
-            DeserializerError::Custom(s) => f.write_str(s),
-        }
-    }
-}
-
-impl Error for DeserializerError {}
-
-impl From<SerdeJsonError> for DeserializerError {
-    fn from(error: SerdeJsonError) -> DeserializerError {
-        DeserializerError::SerdeJson(error)
-    }
-}
-
-impl From<heed::Error> for DeserializerError {
-    fn from(error: heed::Error) -> DeserializerError {
-        DeserializerError::Zlmdb(error)
-    }
-}
-
-pub struct Deserializer<'a> {
-    pub document_id: DocumentId,
-    pub reader: &'a heed::RoTxn,
-    pub documents_fields: DocumentsFields,
-    pub schema: &'a Schema,
-    pub attributes: Option<&'a HashSet<SchemaAttr>>,
-}
-
-impl<'de, 'a, 'b> de::Deserializer<'de> for &'b mut Deserializer<'a> {
-    type Error = DeserializerError;
-
-    fn deserialize_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>
-    where
-        V: de::Visitor<'de>,
-    {
-        self.deserialize_option(visitor)
-    }
-
-    fn deserialize_option<V>(self, visitor: V) -> Result<V::Value, Self::Error>
-    where
-        V: de::Visitor<'de>,
-    {
-        self.deserialize_map(visitor)
-    }
-
-    fn deserialize_map<V>(self, visitor: V) -> Result<V::Value, Self::Error>
-    where
-        V: de::Visitor<'de>,
-    {
-        let mut error = None;
-
-        let iter = self
-            .documents_fields
-            .document_fields(self.reader, self.document_id)?
-            .filter_map(|result| {
-                let (attr, value) = match result {
-                    Ok(value) => value,
-                    Err(e) => {
-                        error = Some(e);
-                        return None;
-                    }
-                };
-
-                let is_displayed = self.schema.props(attr).is_displayed();
-                if is_displayed && self.attributes.map_or(true, |f| f.contains(&attr)) {
-                    let attribute_name = self.schema.attribute_name(attr);
-
-                    let cursor = Cursor::new(value.to_owned());
-                    let ioread = SerdeJsonIoRead::new(cursor);
-                    let value = Value(SerdeJsonDeserializer::new(ioread));
-
-                    Some((attribute_name, value))
-                } else {
-                    None
-                }
-            });
-
-        let mut iter = iter.peekable();
-
-        let result = match iter.peek() {
-            Some(_) => {
-                let map_deserializer = de::value::MapDeserializer::new(iter);
-                visitor
-                    .visit_some(map_deserializer)
-                    .map_err(DeserializerError::from)
-            }
-            None => visitor.visit_none(),
-        };
-
-        match error.take() {
-            Some(error) => Err(error.into()),
-            None => result,
-        }
-    }
-
-    forward_to_deserialize_any! {
-        bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string
-        bytes byte_buf unit unit_struct newtype_struct seq tuple
-        tuple_struct struct enum identifier ignored_any
-    }
-}
-
-struct Value(SerdeJsonDeserializer<SerdeJsonIoRead<Cursor<Vec<u8>>>>);
-
-impl<'de> de::IntoDeserializer<'de, SerdeJsonError> for Value {
-    type Deserializer = Self;
-
-    fn into_deserializer(self) -> Self::Deserializer {
-        self
-    }
-}
-
-impl<'de> de::Deserializer<'de> for Value {
-    type Error = SerdeJsonError;
-
-    fn deserialize_any<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
-    where
-        V: de::Visitor<'de>,
-    {
-        self.0.deserialize_any(visitor)
-    }
-
-    forward_to_deserialize_any! {
-        bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string
-        bytes byte_buf option unit unit_struct newtype_struct seq tuple
-        tuple_struct map struct enum identifier ignored_any
-    }
-}

meilidb-core/src/serde/extract_document_id.rs

@@ -1,295 +0,0 @@
-use std::hash::{Hash, Hasher};
-
-use crate::DocumentId;
-use serde::{ser, Serialize};
-use serde_json::Value;
-use siphasher::sip::SipHasher;
-
-use super::{ConvertToString, SerializerError};
-
-pub fn extract_document_id<D>(
-    identifier: &str,
-    document: &D,
-) -> Result<Option<DocumentId>, SerializerError>
-where
-    D: serde::Serialize,
-{
-    let serializer = ExtractDocumentId { identifier };
-    document.serialize(serializer)
-}
-
-pub fn value_to_string(value: &Value) -> Option<String> {
-    match value {
-        Value::Null => None,
-        Value::Bool(_) => None,
-        Value::Number(value) => Some(value.to_string()),
-        Value::String(value) => Some(value.to_string()),
-        Value::Array(_) => None,
-        Value::Object(_) => None,
-    }
-}
-
-pub fn compute_document_id<H: Hash>(t: H) -> DocumentId {
-    let mut s = SipHasher::new();
-    t.hash(&mut s);
-    let hash = s.finish();
-    DocumentId(hash)
-}
-
-struct ExtractDocumentId<'a> {
-    identifier: &'a str,
-}
-
-impl<'a> ser::Serializer for ExtractDocumentId<'a> {
-    type Ok = Option<DocumentId>;
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = ExtractDocumentIdMapSerializer<'a>;
-    type SerializeStruct = ExtractDocumentIdStructSerializer<'a>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    forward_to_unserializable_type! {
-        bool => serialize_bool,
-        char => serialize_char,
-
-        i8  => serialize_i8,
-        i16 => serialize_i16,
-        i32 => serialize_i32,
-        i64 => serialize_i64,
-
-        u8  => serialize_u8,
-        u16 => serialize_u16,
-        u32 => serialize_u32,
-        u64 => serialize_u64,
-
-        f32 => serialize_f32,
-        f64 => serialize_f64,
-    }
-
-    fn serialize_str(self, _value: &str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "str" })
-    }
-
-    fn serialize_bytes(self, _value: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        let serializer = ExtractDocumentIdMapSerializer {
-            identifier: self.identifier,
-            document_id: None,
-            current_key_name: None,
-        };
-
-        Ok(serializer)
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        let serializer = ExtractDocumentIdStructSerializer {
-            identifier: self.identifier,
-            document_id: None,
-        };
-
-        Ok(serializer)
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct ExtractDocumentIdMapSerializer<'a> {
-    identifier: &'a str,
-    document_id: Option<DocumentId>,
-    current_key_name: Option<String>,
-}
-
-impl<'a> ser::SerializeMap for ExtractDocumentIdMapSerializer<'a> {
-    type Ok = Option<DocumentId>;
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-        self.current_key_name = Some(key);
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        let key = self.current_key_name.take().unwrap();
-        self.serialize_entry(&key, value)
-    }
-
-    fn serialize_entry<K: ?Sized, V: ?Sized>(
-        &mut self,
-        key: &K,
-        value: &V,
-    ) -> Result<(), Self::Error>
-    where
-        K: Serialize,
-        V: Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-
-        if self.identifier == key {
-            let value = serde_json::to_string(value).and_then(|s| serde_json::from_str(&s))?;
-            match value_to_string(&value).map(|s| compute_document_id(&s)) {
-                Some(document_id) => self.document_id = Some(document_id),
-                None => return Err(SerializerError::InvalidDocumentIdType),
-            }
-        }
-
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.document_id)
-    }
-}
-
-pub struct ExtractDocumentIdStructSerializer<'a> {
-    identifier: &'a str,
-    document_id: Option<DocumentId>,
-}
-
-impl<'a> ser::SerializeStruct for ExtractDocumentIdStructSerializer<'a> {
-    type Ok = Option<DocumentId>;
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        if self.identifier == key {
-            let value = serde_json::to_string(value).and_then(|s| serde_json::from_str(&s))?;
-            match value_to_string(&value).map(compute_document_id) {
-                Some(document_id) => self.document_id = Some(document_id),
-                None => return Err(SerializerError::InvalidDocumentIdType),
-            }
-        }
-
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.document_id)
-    }
-}

meilidb-core/src/serde/indexer.rs

@@ -1,365 +0,0 @@
-use meilidb_schema::SchemaAttr;
-use serde::ser;
-use serde::Serialize;
-
-use super::{ConvertToString, SerializerError};
-use crate::raw_indexer::RawIndexer;
-use crate::DocumentId;
-
-pub struct Indexer<'a> {
-    pub attribute: SchemaAttr,
-    pub indexer: &'a mut RawIndexer,
-    pub document_id: DocumentId,
-}
-
-impl<'a> ser::Serializer for Indexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-    type SerializeSeq = SeqIndexer<'a>;
-    type SerializeTuple = TupleIndexer<'a>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = MapIndexer<'a>;
-    type SerializeStruct = StructIndexer<'a>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    fn serialize_bool(self, _value: bool) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "boolean",
-        })
-    }
-
-    fn serialize_char(self, value: char) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i8(self, value: i8) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i16(self, value: i16) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i32(self, value: i32) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i64(self, value: i64) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u8(self, value: u8) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u16(self, value: u16) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u32(self, value: u32) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u64(self, value: u64) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_f32(self, value: f32) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_f64(self, value: f64) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_str(self, text: &str) -> Result<Self::Ok, Self::Error> {
-        let number_of_words = self
-            .indexer
-            .index_text(self.document_id, self.attribute, text);
-        Ok(Some(number_of_words))
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        let number_of_words = self
-            .indexer
-            .index_text(self.document_id, self.attribute, &text);
-        Ok(Some(number_of_words))
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        Err(SerializerError::UnindexableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        let indexer = SeqIndexer {
-            attribute: self.attribute,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        let indexer = TupleIndexer {
-            attribute: self.attribute,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        let indexer = MapIndexer {
-            attribute: self.attribute,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "struct",
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct SeqIndexer<'a> {
-    attribute: SchemaAttr,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeSeq for SeqIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_element<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.attribute, texts);
-        Ok(None)
-    }
-}
-
-pub struct MapIndexer<'a> {
-    attribute: SchemaAttr,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeMap for MapIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = key.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.attribute, texts);
-        Ok(None)
-    }
-}
-
-pub struct StructIndexer<'a> {
-    attribute: SchemaAttr,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeStruct for StructIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let key_text = key.to_owned();
-        let value_text = value.serialize(ConvertToString)?;
-        self.texts.push(key_text);
-        self.texts.push(value_text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.attribute, texts);
-        Ok(None)
-    }
-}
-
-pub struct TupleIndexer<'a> {
-    attribute: SchemaAttr,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeTuple for TupleIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_element<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.attribute, texts);
-        Ok(None)
-    }
-}

meilidb-core/src/serde/mod.rs

@@ -1,103 +0,0 @@
-macro_rules! forward_to_unserializable_type {
-    ($($ty:ident => $se_method:ident,)*) => {
-        $(
-            fn $se_method(self, _v: $ty) -> Result<Self::Ok, Self::Error> {
-                Err(SerializerError::UnserializableType { type_name: "$ty" })
-            }
-        )*
-    }
-}
-
-mod convert_to_number;
-mod convert_to_string;
-mod deserializer;
-mod extract_document_id;
-mod indexer;
-mod serializer;
-
-pub use self::convert_to_number::ConvertToNumber;
-pub use self::convert_to_string::ConvertToString;
-pub use self::deserializer::{Deserializer, DeserializerError};
-pub use self::extract_document_id::{compute_document_id, extract_document_id, value_to_string};
-pub use self::indexer::Indexer;
-pub use self::serializer::{serialize_value, Serializer};
-
-use std::{error::Error, fmt};
-
-use serde::ser;
-use serde_json::Error as SerdeJsonError;
-
-use crate::ParseNumberError;
-
-#[derive(Debug)]
-pub enum SerializerError {
-    DocumentIdNotFound,
-    InvalidDocumentIdType,
-    Zlmdb(heed::Error),
-    SerdeJson(SerdeJsonError),
-    ParseNumber(ParseNumberError),
-    UnserializableType { type_name: &'static str },
-    UnindexableType { type_name: &'static str },
-    UnrankableType { type_name: &'static str },
-    Custom(String),
-}
-
-impl ser::Error for SerializerError {
-    fn custom<T: fmt::Display>(msg: T) -> Self {
-        SerializerError::Custom(msg.to_string())
-    }
-}
-
-impl fmt::Display for SerializerError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            SerializerError::DocumentIdNotFound => {
-                f.write_str("serialized document does not have an id according to the schema")
-            }
-            SerializerError::InvalidDocumentIdType => {
-                f.write_str("document identifier can only be of type string or number")
-            }
-            SerializerError::Zlmdb(e) => write!(f, "heed related error: {}", e),
-            SerializerError::SerdeJson(e) => write!(f, "serde json error: {}", e),
-            SerializerError::ParseNumber(e) => {
-                write!(f, "error while trying to parse a number: {}", e)
-            }
-            SerializerError::UnserializableType { type_name } => {
-                write!(f, "{} is not a serializable type", type_name)
-            }
-            SerializerError::UnindexableType { type_name } => {
-                write!(f, "{} is not an indexable type", type_name)
-            }
-            SerializerError::UnrankableType { type_name } => {
-                write!(f, "{} types can not be used for ranking", type_name)
-            }
-            SerializerError::Custom(s) => f.write_str(s),
-        }
-    }
-}
-
-impl Error for SerializerError {}
-
-impl From<String> for SerializerError {
-    fn from(value: String) -> SerializerError {
-        SerializerError::Custom(value)
-    }
-}
-
-impl From<SerdeJsonError> for SerializerError {
-    fn from(error: SerdeJsonError) -> SerializerError {
-        SerializerError::SerdeJson(error)
-    }
-}
-
-impl From<heed::Error> for SerializerError {
-    fn from(error: heed::Error) -> SerializerError {
-        SerializerError::Zlmdb(error)
-    }
-}
-
-impl From<ParseNumberError> for SerializerError {
-    fn from(error: ParseNumberError) -> SerializerError {
-        SerializerError::ParseNumber(error)
-    }
-}

meilidb-core/src/serde/serializer.rs

@@ -1,338 +0,0 @@
-use meilidb_schema::{Schema, SchemaAttr, SchemaProps};
-use serde::ser;
-
-use crate::raw_indexer::RawIndexer;
-use crate::store::{DocumentsFields, DocumentsFieldsCounts};
-use crate::{DocumentId, RankedMap};
-
-use super::{ConvertToNumber, ConvertToString, Indexer, SerializerError};
-
-pub struct Serializer<'a, 'b> {
-    pub txn: &'a mut heed::RwTxn<'b>,
-    pub schema: &'a Schema,
-    pub document_store: DocumentsFields,
-    pub document_fields_counts: DocumentsFieldsCounts,
-    pub indexer: &'a mut RawIndexer,
-    pub ranked_map: &'a mut RankedMap,
-    pub document_id: DocumentId,
-}
-
-impl<'a, 'b> ser::Serializer for Serializer<'a, 'b> {
-    type Ok = ();
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = MapSerializer<'a, 'b>;
-    type SerializeStruct = StructSerializer<'a, 'b>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    forward_to_unserializable_type! {
-        bool => serialize_bool,
-        char => serialize_char,
-
-        i8  => serialize_i8,
-        i16 => serialize_i16,
-        i32 => serialize_i32,
-        i64 => serialize_i64,
-
-        u8  => serialize_u8,
-        u16 => serialize_u16,
-        u32 => serialize_u32,
-        u64 => serialize_u64,
-
-        f32 => serialize_f32,
-        f64 => serialize_f64,
-    }
-
-    fn serialize_str(self, _v: &str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "str" })
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        Ok(MapSerializer {
-            txn: self.txn,
-            schema: self.schema,
-            document_id: self.document_id,
-            document_store: self.document_store,
-            document_fields_counts: self.document_fields_counts,
-            indexer: self.indexer,
-            ranked_map: self.ranked_map,
-            current_key_name: None,
-        })
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Ok(StructSerializer {
-            txn: self.txn,
-            schema: self.schema,
-            document_id: self.document_id,
-            document_store: self.document_store,
-            document_fields_counts: self.document_fields_counts,
-            indexer: self.indexer,
-            ranked_map: self.ranked_map,
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct MapSerializer<'a, 'b> {
-    txn: &'a mut heed::RwTxn<'b>,
-    schema: &'a Schema,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    document_fields_counts: DocumentsFieldsCounts,
-    indexer: &'a mut RawIndexer,
-    ranked_map: &'a mut RankedMap,
-    current_key_name: Option<String>,
-}
-
-impl<'a, 'b> ser::SerializeMap for MapSerializer<'a, 'b> {
-    type Ok = ();
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-        self.current_key_name = Some(key);
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let key = self.current_key_name.take().unwrap();
-        self.serialize_entry(&key, value)
-    }
-
-    fn serialize_entry<K: ?Sized, V: ?Sized>(
-        &mut self,
-        key: &K,
-        value: &V,
-    ) -> Result<(), Self::Error>
-    where
-        K: ser::Serialize,
-        V: ser::Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-        match self.schema.attribute(&key) {
-            Some(attribute) => serialize_value(
-                self.txn,
-                attribute,
-                self.schema.props(attribute),
-                self.document_id,
-                self.document_store,
-                self.document_fields_counts,
-                self.indexer,
-                self.ranked_map,
-                value,
-            ),
-            None => Ok(()),
-        }
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(())
-    }
-}
-
-pub struct StructSerializer<'a, 'b> {
-    txn: &'a mut heed::RwTxn<'b>,
-    schema: &'a Schema,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    document_fields_counts: DocumentsFieldsCounts,
-    indexer: &'a mut RawIndexer,
-    ranked_map: &'a mut RankedMap,
-}
-
-impl<'a, 'b> ser::SerializeStruct for StructSerializer<'a, 'b> {
-    type Ok = ();
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        match self.schema.attribute(key) {
-            Some(attribute) => serialize_value(
-                self.txn,
-                attribute,
-                self.schema.props(attribute),
-                self.document_id,
-                self.document_store,
-                self.document_fields_counts,
-                self.indexer,
-                self.ranked_map,
-                value,
-            ),
-            None => Ok(()),
-        }
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(())
-    }
-}
-
-pub fn serialize_value<T: ?Sized>(
-    txn: &mut heed::RwTxn,
-    attribute: SchemaAttr,
-    props: SchemaProps,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    documents_fields_counts: DocumentsFieldsCounts,
-    indexer: &mut RawIndexer,
-    ranked_map: &mut RankedMap,
-    value: &T,
-) -> Result<(), SerializerError>
-where
-    T: ser::Serialize,
-{
-    let serialized = serde_json::to_vec(value)?;
-    document_store.put_document_field(txn, document_id, attribute, &serialized)?;
-
-    if props.is_indexed() {
-        let indexer = Indexer {
-            attribute,
-            indexer,
-            document_id,
-        };
-        if let Some(number_of_words) = value.serialize(indexer)? {
-            documents_fields_counts.put_document_field_count(
-                txn,
-                document_id,
-                attribute,
-                number_of_words as u64,
-            )?;
-        }
-    }
-
-    if props.is_ranked() {
-        let number = value.serialize(ConvertToNumber)?;
-        ranked_map.insert(document_id, attribute, number);
-    }
-
-    Ok(())
-}

meilidb-core/src/store/docs_words.rs

@@ -1,49 +0,0 @@
-use super::BEU64;
-use crate::DocumentId;
-use heed::types::{ByteSlice, OwnedType};
-use heed::Result as ZResult;
-use std::sync::Arc;
-
-#[derive(Copy, Clone)]
-pub struct DocsWords {
-    pub(crate) docs_words: heed::Database<OwnedType<BEU64>, ByteSlice>,
-}
-
-impl DocsWords {
-    pub fn put_doc_words(
-        self,
-        writer: &mut heed::RwTxn,
-        document_id: DocumentId,
-        words: &fst::Set,
-    ) -> ZResult<()> {
-        let document_id = BEU64::new(document_id.0);
-        let bytes = words.as_fst().as_bytes();
-        self.docs_words.put(writer, &document_id, bytes)
-    }
-
-    pub fn del_doc_words(self, writer: &mut heed::RwTxn, document_id: DocumentId) -> ZResult<bool> {
-        let document_id = BEU64::new(document_id.0);
-        self.docs_words.delete(writer, &document_id)
-    }
-
-    pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> {
-        self.docs_words.clear(writer)
-    }
-
-    pub fn doc_words(
-        self,
-        reader: &heed::RoTxn,
-        document_id: DocumentId,
-    ) -> ZResult<Option<fst::Set>> {
-        let document_id = BEU64::new(document_id.0);
-        match self.docs_words.get(reader, &document_id)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
-        }
-    }
-}

meilidb-core/src/store/documents_fields.rs

@@ -1,78 +0,0 @@
-use heed::types::{ByteSlice, OwnedType};
-use heed::Result as ZResult;
-use meilidb_schema::SchemaAttr;
-
-use super::DocumentAttrKey;
-use crate::DocumentId;
-
-#[derive(Copy, Clone)]
-pub struct DocumentsFields {
-    pub(crate) documents_fields: heed::Database<OwnedType<DocumentAttrKey>, ByteSlice>,
-}
-
-impl DocumentsFields {
-    pub fn put_document_field(
-        self,
-        writer: &mut heed::RwTxn,
-        document_id: DocumentId,
-        attribute: SchemaAttr,
-        value: &[u8],
-    ) -> ZResult<()> {
-        let key = DocumentAttrKey::new(document_id, attribute);
-        self.documents_fields.put(writer, &key, value)
-    }
-
-    pub fn del_all_document_fields(
-        self,
-        writer: &mut heed::RwTxn,
-        document_id: DocumentId,
-    ) -> ZResult<usize> {
-        let start = DocumentAttrKey::new(document_id, SchemaAttr::min());
-        let end = DocumentAttrKey::new(document_id, SchemaAttr::max());
-        self.documents_fields.delete_range(writer, &(start..=end))
-    }
-
-    pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> {
-        self.documents_fields.clear(writer)
-    }
-
-    pub fn document_attribute<'txn>(
-        self,
-        reader: &'txn heed::RoTxn,
-        document_id: DocumentId,
-        attribute: SchemaAttr,
-    ) -> ZResult<Option<&'txn [u8]>> {
-        let key = DocumentAttrKey::new(document_id, attribute);
-        self.documents_fields.get(reader, &key)
-    }
-
-    pub fn document_fields<'txn>(
-        self,
-        reader: &'txn heed::RoTxn,
-        document_id: DocumentId,
-    ) -> ZResult<DocumentFieldsIter<'txn>> {
-        let start = DocumentAttrKey::new(document_id, SchemaAttr::min());
-        let end = DocumentAttrKey::new(document_id, SchemaAttr::max());
-        let iter = self.documents_fields.range(reader, &(start..=end))?;
-        Ok(DocumentFieldsIter { iter })
-    }
-}
-
-pub struct DocumentFieldsIter<'txn> {
-    iter: heed::RoRange<'txn, OwnedType<DocumentAttrKey>, ByteSlice>,
-}
-
-impl<'txn> Iterator for DocumentFieldsIter<'txn> {
-    type Item = ZResult<(SchemaAttr, &'txn [u8])>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        match self.iter.next() {
-            Some(Ok((key, bytes))) => {
-                let attr = SchemaAttr(key.attr.get());
-                Some(Ok((attr, bytes)))
-            }
-            Some(Err(e)) => Some(Err(e)),
-            None => None,
-        }
-    }
-}

meilidb-core/src/store/documents_fields_counts.rs

@@ -1,142 +0,0 @@
-use super::DocumentAttrKey;
-use crate::DocumentId;
-use heed::types::OwnedType;
-use heed::Result as ZResult;
-use meilidb_schema::SchemaAttr;
-
-#[derive(Copy, Clone)]
-pub struct DocumentsFieldsCounts {
-    pub(crate) documents_fields_counts: heed::Database<OwnedType<DocumentAttrKey>, OwnedType<u64>>,
-}
-
-impl DocumentsFieldsCounts {
-    pub fn put_document_field_count(
-        self,
-        writer: &mut heed::RwTxn,
-        document_id: DocumentId,
-        attribute: SchemaAttr,
-        value: u64,
-    ) -> ZResult<()> {
-        let key = DocumentAttrKey::new(document_id, attribute);
-        self.documents_fields_counts.put(writer, &key, &value)
-    }
-
-    pub fn del_all_document_fields_counts(
-        self,
-        writer: &mut heed::RwTxn,
-        document_id: DocumentId,
-    ) -> ZResult<usize> {
-        let start = DocumentAttrKey::new(document_id, SchemaAttr::min());
-        let end = DocumentAttrKey::new(document_id, SchemaAttr::max());
-        self.documents_fields_counts
-            .delete_range(writer, &(start..=end))
-    }
-
-    pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> {
-        self.documents_fields_counts.clear(writer)
-    }
-
-    pub fn document_field_count(
-        self,
-        reader: &heed::RoTxn,
-        document_id: DocumentId,
-        attribute: SchemaAttr,
-    ) -> ZResult<Option<u64>> {
-        let key = DocumentAttrKey::new(document_id, attribute);
-        match self.documents_fields_counts.get(reader, &key)? {
-            Some(count) => Ok(Some(count)),
-            None => Ok(None),
-        }
-    }
-
-    pub fn document_fields_counts<'txn>(
-        self,
-        reader: &'txn heed::RoTxn,
-        document_id: DocumentId,
-    ) -> ZResult<DocumentFieldsCountsIter<'txn>> {
-        let start = DocumentAttrKey::new(document_id, SchemaAttr::min());
-        let end = DocumentAttrKey::new(document_id, SchemaAttr::max());
-        let iter = self.documents_fields_counts.range(reader, &(start..=end))?;
-        Ok(DocumentFieldsCountsIter { iter })
-    }
-
-    pub fn documents_ids<'txn>(self, reader: &'txn heed::RoTxn) -> ZResult<DocumentsIdsIter<'txn>> {
-        let iter = self.documents_fields_counts.iter(reader)?;
-        Ok(DocumentsIdsIter {
-            last_seen_id: None,
-            iter,
-        })
-    }
-
-    pub fn all_documents_fields_counts<'txn>(
-        self,
-        reader: &'txn heed::RoTxn,
-    ) -> ZResult<AllDocumentsFieldsCountsIter<'txn>> {
-        let iter = self.documents_fields_counts.iter(reader)?;
-        Ok(AllDocumentsFieldsCountsIter { iter })
-    }
-}
-
-pub struct DocumentFieldsCountsIter<'txn> {
-    iter: heed::RoRange<'txn, OwnedType<DocumentAttrKey>, OwnedType<u64>>,
-}
-
-impl Iterator for DocumentFieldsCountsIter<'_> {
-    type Item = ZResult<(SchemaAttr, u64)>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        match self.iter.next() {
-            Some(Ok((key, count))) => {
-                let attr = SchemaAttr(key.attr.get());
-                Some(Ok((attr, count)))
-            }
-            Some(Err(e)) => Some(Err(e)),
-            None => None,
-        }
-    }
-}
-
-pub struct DocumentsIdsIter<'txn> {
-    last_seen_id: Option<DocumentId>,
-    iter: heed::RoIter<'txn, OwnedType<DocumentAttrKey>, OwnedType<u64>>,
-}
-
-impl Iterator for DocumentsIdsIter<'_> {
-    type Item = ZResult<DocumentId>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        for result in &mut self.iter {
-            match result {
-                Ok((key, _)) => {
-                    let document_id = DocumentId(key.docid.get());
-                    if Some(document_id) != self.last_seen_id {
-                        self.last_seen_id = Some(document_id);
-                        return Some(Ok(document_id));
-                    }
-                }
-                Err(e) => return Some(Err(e)),
-            }
-        }
-        None
-    }
-}
-
-pub struct AllDocumentsFieldsCountsIter<'txn> {
-    iter: heed::RoIter<'txn, OwnedType<DocumentAttrKey>, OwnedType<u64>>,
-}
-
-impl Iterator for AllDocumentsFieldsCountsIter<'_> {
-    type Item = ZResult<(DocumentId, SchemaAttr, u64)>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        match self.iter.next() {
-            Some(Ok((key, count))) => {
-                let docid = DocumentId(key.docid.get());
-                let attr = SchemaAttr(key.attr.get());
-                Some(Ok((docid, attr, count)))
-            }
-            Some(Err(e)) => Some(Err(e)),
-            None => None,
-        }
-    }
-}

meilidb-core/src/store/main.rs

@@ -1,125 +0,0 @@
-use crate::RankedMap;
-use heed::types::{ByteSlice, OwnedType, SerdeBincode, Str};
-use heed::Result as ZResult;
-use meilidb_schema::Schema;
-use std::sync::Arc;
-
-const CUSTOMS_KEY: &str = "customs-key";
-const NUMBER_OF_DOCUMENTS_KEY: &str = "number-of-documents";
-const RANKED_MAP_KEY: &str = "ranked-map";
-const SCHEMA_KEY: &str = "schema";
-const SYNONYMS_KEY: &str = "synonyms";
-const STOP_WORDS_KEY: &str = "stop-words";
-const WORDS_KEY: &str = "words";
-
-#[derive(Copy, Clone)]
-pub struct Main {
-    pub(crate) main: heed::PolyDatabase,
-}
-
-impl Main {
-    pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> {
-        self.main.clear(writer)
-    }
-
-    pub fn put_words_fst(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> ZResult<()> {
-        let bytes = fst.as_fst().as_bytes();
-        self.main.put::<Str, ByteSlice>(writer, WORDS_KEY, bytes)
-    }
-
-    pub fn words_fst(self, reader: &heed::RoTxn) -> ZResult<Option<fst::Set>> {
-        match self.main.get::<Str, ByteSlice>(reader, WORDS_KEY)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
-        }
-    }
-
-    pub fn put_schema(self, writer: &mut heed::RwTxn, schema: &Schema) -> ZResult<()> {
-        self.main
-            .put::<Str, SerdeBincode<Schema>>(writer, SCHEMA_KEY, schema)
-    }
-
-    pub fn schema(self, reader: &heed::RoTxn) -> ZResult<Option<Schema>> {
-        self.main
-            .get::<Str, SerdeBincode<Schema>>(reader, SCHEMA_KEY)
-    }
-
-    pub fn put_ranked_map(self, writer: &mut heed::RwTxn, ranked_map: &RankedMap) -> ZResult<()> {
-        self.main
-            .put::<Str, SerdeBincode<RankedMap>>(writer, RANKED_MAP_KEY, &ranked_map)
-    }
-
-    pub fn ranked_map(self, reader: &heed::RoTxn) -> ZResult<Option<RankedMap>> {
-        self.main
-            .get::<Str, SerdeBincode<RankedMap>>(reader, RANKED_MAP_KEY)
-    }
-
-    pub fn put_synonyms_fst(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> ZResult<()> {
-        let bytes = fst.as_fst().as_bytes();
-        self.main.put::<Str, ByteSlice>(writer, SYNONYMS_KEY, bytes)
-    }
-
-    pub fn synonyms_fst(self, reader: &heed::RoTxn) -> ZResult<Option<fst::Set>> {
-        match self.main.get::<Str, ByteSlice>(reader, SYNONYMS_KEY)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
-        }
-    }
-
-    pub fn put_stop_words_fst(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> ZResult<()> {
-        let bytes = fst.as_fst().as_bytes();
-        self.main
-            .put::<Str, ByteSlice>(writer, STOP_WORDS_KEY, bytes)
-    }
-
-    pub fn stop_words_fst(self, reader: &heed::RoTxn) -> ZResult<Option<fst::Set>> {
-        match self.main.get::<Str, ByteSlice>(reader, STOP_WORDS_KEY)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
-        }
-    }
-
-    pub fn put_number_of_documents<F>(self, writer: &mut heed::RwTxn, f: F) -> ZResult<u64>
-    where
-        F: Fn(u64) -> u64,
-    {
-        let new = self.number_of_documents(writer).map(f)?;
-        self.main
-            .put::<Str, OwnedType<u64>>(writer, NUMBER_OF_DOCUMENTS_KEY, &new)?;
-        Ok(new)
-    }
-
-    pub fn number_of_documents(self, reader: &heed::RoTxn) -> ZResult<u64> {
-        match self
-            .main
-            .get::<Str, OwnedType<u64>>(reader, NUMBER_OF_DOCUMENTS_KEY)?
-        {
-            Some(value) => Ok(value),
-            None => Ok(0),
-        }
-    }
-
-    pub fn put_customs(self, writer: &mut heed::RwTxn, customs: &[u8]) -> ZResult<()> {
-        self.main
-            .put::<Str, ByteSlice>(writer, CUSTOMS_KEY, customs)
-    }
-
-    pub fn customs<'txn>(self, reader: &'txn heed::RoTxn) -> ZResult<Option<&'txn [u8]>> {
-        self.main.get::<Str, ByteSlice>(reader, CUSTOMS_KEY)
-    }
-}

meilidb-core/src/store/mod.rs

@@ -1,397 +0,0 @@
-mod docs_words;
-mod documents_fields;
-mod documents_fields_counts;
-mod main;
-mod postings_lists;
-mod synonyms;
-mod updates;
-mod updates_results;
-
-pub use self::docs_words::DocsWords;
-pub use self::documents_fields::{DocumentFieldsIter, DocumentsFields};
-pub use self::documents_fields_counts::{
-    DocumentFieldsCountsIter, DocumentsFieldsCounts, DocumentsIdsIter,
-};
-pub use self::main::Main;
-pub use self::postings_lists::PostingsLists;
-pub use self::synonyms::Synonyms;
-pub use self::updates::Updates;
-pub use self::updates_results::UpdatesResults;
-
-use std::collections::HashSet;
-
-use heed::Result as ZResult;
-use meilidb_schema::{Schema, SchemaAttr};
-use serde::de::{self, Deserialize};
-use zerocopy::{AsBytes, FromBytes};
-
-use crate::criterion::Criteria;
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::serde::Deserializer;
-use crate::{query_builder::QueryBuilder, update, DocumentId, Error, MResult};
-
-type BEU64 = zerocopy::U64<byteorder::BigEndian>;
-type BEU16 = zerocopy::U16<byteorder::BigEndian>;
-
-#[derive(Debug, Copy, Clone, AsBytes, FromBytes)]
-#[repr(C)]
-pub struct DocumentAttrKey {
-    docid: BEU64,
-    attr: BEU16,
-}
-
-impl DocumentAttrKey {
-    fn new(docid: DocumentId, attr: SchemaAttr) -> DocumentAttrKey {
-        DocumentAttrKey {
-            docid: BEU64::new(docid.0),
-            attr: BEU16::new(attr.0),
-        }
-    }
-}
-
-fn main_name(name: &str) -> String {
-    format!("store-{}", name)
-}
-
-fn postings_lists_name(name: &str) -> String {
-    format!("store-{}-postings-lists", name)
-}
-
-fn documents_fields_name(name: &str) -> String {
-    format!("store-{}-documents-fields", name)
-}
-
-fn documents_fields_counts_name(name: &str) -> String {
-    format!("store-{}-documents-fields-counts", name)
-}
-
-fn synonyms_name(name: &str) -> String {
-    format!("store-{}-synonyms", name)
-}
-
-fn docs_words_name(name: &str) -> String {
-    format!("store-{}-docs-words", name)
-}
-
-fn updates_name(name: &str) -> String {
-    format!("store-{}-updates", name)
-}
-
-fn updates_results_name(name: &str) -> String {
-    format!("store-{}-updates-results", name)
-}
-
-#[derive(Clone)]
-pub struct Index {
-    pub main: Main,
-    pub postings_lists: PostingsLists,
-    pub documents_fields: DocumentsFields,
-    pub documents_fields_counts: DocumentsFieldsCounts,
-    pub synonyms: Synonyms,
-    pub docs_words: DocsWords,
-
-    pub updates: Updates,
-    pub updates_results: UpdatesResults,
-    updates_notifier: UpdateEventsEmitter,
-}
-
-impl Index {
-    pub fn document<T: de::DeserializeOwned>(
-        &self,
-        reader: &heed::RoTxn,
-        attributes: Option<&HashSet<&str>>,
-        document_id: DocumentId,
-    ) -> MResult<Option<T>> {
-        let schema = self.main.schema(reader)?;
-        let schema = schema.ok_or(Error::SchemaMissing)?;
-
-        let attributes = match attributes {
-            Some(attributes) => attributes
-                .iter()
-                .map(|name| schema.attribute(name))
-                .collect(),
-            None => None,
-        };
-
-        let mut deserializer = Deserializer {
-            document_id,
-            reader,
-            documents_fields: self.documents_fields,
-            schema: &schema,
-            attributes: attributes.as_ref(),
-        };
-
-        Ok(Option::<T>::deserialize(&mut deserializer)?)
-    }
-
-    pub fn document_attribute<T: de::DeserializeOwned>(
-        &self,
-        reader: &heed::RoTxn,
-        document_id: DocumentId,
-        attribute: SchemaAttr,
-    ) -> MResult<Option<T>> {
-        let bytes = self
-            .documents_fields
-            .document_attribute(reader, document_id, attribute)?;
-        match bytes {
-            Some(bytes) => Ok(Some(serde_json::from_slice(bytes)?)),
-            None => Ok(None),
-        }
-    }
-
-    pub fn schema_update(&self, writer: &mut heed::RwTxn, schema: Schema) -> MResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        update::push_schema_update(writer, self.updates, self.updates_results, schema)
-    }
-
-    pub fn customs_update(&self, writer: &mut heed::RwTxn, customs: Vec<u8>) -> ZResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        update::push_customs_update(writer, self.updates, self.updates_results, customs)
-    }
-
-    pub fn documents_addition<D>(&self) -> update::DocumentsAddition<D> {
-        update::DocumentsAddition::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn documents_partial_addition<D>(&self) -> update::DocumentsAddition<D> {
-        update::DocumentsAddition::new_partial(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn documents_deletion(&self) -> update::DocumentsDeletion {
-        update::DocumentsDeletion::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn clear_all(&self, writer: &mut heed::RwTxn) -> MResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        update::push_clear_all(writer, self.updates, self.updates_results)
-    }
-
-    pub fn synonyms_addition(&self) -> update::SynonymsAddition {
-        update::SynonymsAddition::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn synonyms_deletion(&self) -> update::SynonymsDeletion {
-        update::SynonymsDeletion::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn stop_words_addition(&self) -> update::StopWordsAddition {
-        update::StopWordsAddition::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn stop_words_deletion(&self) -> update::StopWordsDeletion {
-        update::StopWordsDeletion::new(
-            self.updates,
-            self.updates_results,
-            self.updates_notifier.clone(),
-        )
-    }
-
-    pub fn current_update_id(&self, reader: &heed::RoTxn) -> MResult<Option<u64>> {
-        match self.updates.last_update_id(reader)? {
-            Some((id, _)) => Ok(Some(id)),
-            None => Ok(None),
-        }
-    }
-
-    pub fn update_status(
-        &self,
-        reader: &heed::RoTxn,
-        update_id: u64,
-    ) -> MResult<Option<update::UpdateStatus>> {
-        update::update_status(reader, self.updates, self.updates_results, update_id)
-    }
-
-    pub fn all_updates_status(&self, reader: &heed::RoTxn) -> MResult<Vec<update::UpdateStatus>> {
-        let mut updates = Vec::new();
-        let mut last_update_result_id = 0;
-
-        // retrieve all updates results
-        if let Some((last_id, _)) = self.updates_results.last_update_id(reader)? {
-            updates.reserve(last_id as usize);
-
-            for id in 0..=last_id {
-                if let Some(update) = self.update_status(reader, id)? {
-                    updates.push(update);
-                    last_update_result_id = id;
-                }
-            }
-        }
-
-        // retrieve all enqueued updates
-        if let Some((last_id, _)) = self.updates.last_update_id(reader)? {
-            for id in last_update_result_id + 1..=last_id {
-                if let Some(update) = self.update_status(reader, id)? {
-                    updates.push(update);
-                }
-            }
-        }
-
-        Ok(updates)
-    }
-
-    pub fn query_builder(&self) -> QueryBuilder {
-        QueryBuilder::new(
-            self.main,
-            self.postings_lists,
-            self.documents_fields_counts,
-            self.synonyms,
-        )
-    }
-
-    pub fn query_builder_with_criteria<'c, 'f, 'd>(
-        &self,
-        criteria: Criteria<'c>,
-    ) -> QueryBuilder<'c, 'f, 'd> {
-        QueryBuilder::with_criteria(
-            self.main,
-            self.postings_lists,
-            self.documents_fields_counts,
-            self.synonyms,
-            criteria,
-        )
-    }
-}
-
-pub fn create(
-    env: &heed::Env,
-    name: &str,
-    updates_notifier: UpdateEventsEmitter,
-) -> MResult<Index> {
-    // create all the store names
-    let main_name = main_name(name);
-    let postings_lists_name = postings_lists_name(name);
-    let documents_fields_name = documents_fields_name(name);
-    let documents_fields_counts_name = documents_fields_counts_name(name);
-    let synonyms_name = synonyms_name(name);
-    let docs_words_name = docs_words_name(name);
-    let updates_name = updates_name(name);
-    let updates_results_name = updates_results_name(name);
-
-    // open all the stores
-    let main = env.create_poly_database(Some(&main_name))?;
-    let postings_lists = env.create_database(Some(&postings_lists_name))?;
-    let documents_fields = env.create_database(Some(&documents_fields_name))?;
-    let documents_fields_counts = env.create_database(Some(&documents_fields_counts_name))?;
-    let synonyms = env.create_database(Some(&synonyms_name))?;
-    let docs_words = env.create_database(Some(&docs_words_name))?;
-    let updates = env.create_database(Some(&updates_name))?;
-    let updates_results = env.create_database(Some(&updates_results_name))?;
-
-    Ok(Index {
-        main: Main { main },
-        postings_lists: PostingsLists { postings_lists },
-        documents_fields: DocumentsFields { documents_fields },
-        documents_fields_counts: DocumentsFieldsCounts {
-            documents_fields_counts,
-        },
-        synonyms: Synonyms { synonyms },
-        docs_words: DocsWords { docs_words },
-        updates: Updates { updates },
-        updates_results: UpdatesResults { updates_results },
-        updates_notifier,
-    })
-}
-
-pub fn open(
-    env: &heed::Env,
-    name: &str,
-    updates_notifier: UpdateEventsEmitter,
-) -> MResult<Option<Index>> {
-    // create all the store names
-    let main_name = main_name(name);
-    let postings_lists_name = postings_lists_name(name);
-    let documents_fields_name = documents_fields_name(name);
-    let documents_fields_counts_name = documents_fields_counts_name(name);
-    let synonyms_name = synonyms_name(name);
-    let docs_words_name = docs_words_name(name);
-    let updates_name = updates_name(name);
-    let updates_results_name = updates_results_name(name);
-
-    // open all the stores
-    let main = match env.open_poly_database(Some(&main_name))? {
-        Some(main) => main,
-        None => return Ok(None),
-    };
-    let postings_lists = match env.open_database(Some(&postings_lists_name))? {
-        Some(postings_lists) => postings_lists,
-        None => return Ok(None),
-    };
-    let documents_fields = match env.open_database(Some(&documents_fields_name))? {
-        Some(documents_fields) => documents_fields,
-        None => return Ok(None),
-    };
-    let documents_fields_counts = match env.open_database(Some(&documents_fields_counts_name))? {
-        Some(documents_fields_counts) => documents_fields_counts,
-        None => return Ok(None),
-    };
-    let synonyms = match env.open_database(Some(&synonyms_name))? {
-        Some(synonyms) => synonyms,
-        None => return Ok(None),
-    };
-    let docs_words = match env.open_database(Some(&docs_words_name))? {
-        Some(docs_words) => docs_words,
-        None => return Ok(None),
-    };
-    let updates = match env.open_database(Some(&updates_name))? {
-        Some(updates) => updates,
-        None => return Ok(None),
-    };
-    let updates_results = match env.open_database(Some(&updates_results_name))? {
-        Some(updates_results) => updates_results,
-        None => return Ok(None),
-    };
-
-    Ok(Some(Index {
-        main: Main { main },
-        postings_lists: PostingsLists { postings_lists },
-        documents_fields: DocumentsFields { documents_fields },
-        documents_fields_counts: DocumentsFieldsCounts {
-            documents_fields_counts,
-        },
-        synonyms: Synonyms { synonyms },
-        docs_words: DocsWords { docs_words },
-        updates: Updates { updates },
-        updates_results: UpdatesResults { updates_results },
-        updates_notifier,
-    }))
-}
-
-pub fn clear(writer: &mut heed::RwTxn, index: &Index) -> MResult<()> {
-    // send a stop event to the update loop of the index
-    index.updates_notifier.send(UpdateEvent::MustStop).unwrap();
-
-    // clear all the stores
-    index.main.clear(writer)?;
-    index.postings_lists.clear(writer)?;
-    index.documents_fields.clear(writer)?;
-    index.documents_fields_counts.clear(writer)?;
-    index.synonyms.clear(writer)?;
-    index.docs_words.clear(writer)?;
-    index.updates.clear(writer)?;
-    index.updates_results.clear(writer)?;
-    Ok(())
-}

meilidb-core/src/store/postings_lists.rs

@@ -1,41 +0,0 @@
-use crate::DocIndex;
-use heed::types::{ByteSlice, CowSlice};
-use heed::Result as ZResult;
-use sdset::{Set, SetBuf};
-use std::borrow::Cow;
-
-#[derive(Copy, Clone)]
-pub struct PostingsLists {
-    pub(crate) postings_lists: heed::Database<ByteSlice, CowSlice<DocIndex>>,
-}
-
-impl PostingsLists {
-    pub fn put_postings_list(
-        self,
-        writer: &mut heed::RwTxn,
-        word: &[u8],
-        words_indexes: &Set<DocIndex>,
-    ) -> ZResult<()> {
-        self.postings_lists.put(writer, word, words_indexes)
-    }
-
-    pub fn del_postings_list(self, writer: &mut heed::RwTxn, word: &[u8]) -> ZResult<bool> {
-        self.postings_lists.delete(writer, word)
-    }
-
-    pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> {
-        self.postings_lists.clear(writer)
-    }
-
-    pub fn postings_list<'txn>(
-        self,
-        reader: &'txn heed::RoTxn,
-        word: &[u8],
-    ) -> ZResult<Option<Cow<'txn, Set<DocIndex>>>> {
-        match self.postings_lists.get(reader, word)? {
-            Some(Cow::Borrowed(slice)) => Ok(Some(Cow::Borrowed(Set::new_unchecked(slice)))),
-            Some(Cow::Owned(vec)) => Ok(Some(Cow::Owned(SetBuf::new_unchecked(vec)))),
-            None => Ok(None),
-        }
-    }
-}

meilidb-core/src/store/synonyms.rs

@@ -1,40 +0,0 @@
-use heed::types::ByteSlice;
-use heed::Result as ZResult;
-use std::sync::Arc;
-
-#[derive(Copy, Clone)]
-pub struct Synonyms {
-    pub(crate) synonyms: heed::Database<ByteSlice, ByteSlice>,
-}
-
-impl Synonyms {
-    pub fn put_synonyms(
-        self,
-        writer: &mut heed::RwTxn,
-        word: &[u8],
-        synonyms: &fst::Set,
-    ) -> ZResult<()> {
-        let bytes = synonyms.as_fst().as_bytes();
-        self.synonyms.put(writer, word, bytes)
-    }
-
-    pub fn del_synonyms(self, writer: &mut heed::RwTxn, word: &[u8]) -> ZResult<bool> {
-        self.synonyms.delete(writer, word)
-    }
-
-    pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> {
-        self.synonyms.clear(writer)
-    }
-
-    pub fn synonyms(self, reader: &heed::RoTxn, word: &[u8]) -> ZResult<Option<fst::Set>> {
-        match self.synonyms.get(reader, word)? {
-            Some(bytes) => {
-                let len = bytes.len();
-                let bytes = Arc::new(bytes.to_owned());
-                let fst = fst::raw::Fst::from_shared_bytes(bytes, 0, len).unwrap();
-                Ok(Some(fst::Set::from(fst)))
-            }
-            None => Ok(None),
-        }
-    }
-}

meilidb-core/src/store/updates.rs

@@ -1,59 +0,0 @@
-use super::BEU64;
-use crate::update::Update;
-use heed::types::{OwnedType, SerdeJson};
-use heed::Result as ZResult;
-
-#[derive(Copy, Clone)]
-pub struct Updates {
-    pub(crate) updates: heed::Database<OwnedType<BEU64>, SerdeJson<Update>>,
-}
-
-impl Updates {
-    // TODO do not trigger deserialize if possible
-    pub fn last_update_id(self, reader: &heed::RoTxn) -> ZResult<Option<(u64, Update)>> {
-        match self.updates.last(reader)? {
-            Some((key, data)) => Ok(Some((key.get(), data))),
-            None => Ok(None),
-        }
-    }
-
-    // TODO do not trigger deserialize if possible
-    fn first_update_id(self, reader: &heed::RoTxn) -> ZResult<Option<(u64, Update)>> {
-        match self.updates.first(reader)? {
-            Some((key, data)) => Ok(Some((key.get(), data))),
-            None => Ok(None),
-        }
-    }
-
-    // TODO do not trigger deserialize if possible
-    pub fn get(self, reader: &heed::RoTxn, update_id: u64) -> ZResult<Option<Update>> {
-        let update_id = BEU64::new(update_id);
-        self.updates.get(reader, &update_id)
-    }
-
-    pub fn put_update(
-        self,
-        writer: &mut heed::RwTxn,
-        update_id: u64,
-        update: &Update,
-    ) -> ZResult<()> {
-        // TODO prefer using serde_json?
-        let update_id = BEU64::new(update_id);
-        self.updates.put(writer, &update_id, update)
-    }
-
-    pub fn pop_front(self, writer: &mut heed::RwTxn) -> ZResult<Option<(u64, Update)>> {
-        match self.first_update_id(writer)? {
-            Some((update_id, update)) => {
-                let key = BEU64::new(update_id);
-                self.updates.delete(writer, &key)?;
-                Ok(Some((update_id, update)))
-            }
-            None => Ok(None),
-        }
-    }
-
-    pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> {
-        self.updates.clear(writer)
-    }
-}

meilidb-core/src/store/updates_results.rs

@@ -1,44 +0,0 @@
-use super::BEU64;
-use crate::update::ProcessedUpdateResult;
-use heed::types::{OwnedType, SerdeJson};
-use heed::Result as ZResult;
-
-#[derive(Copy, Clone)]
-pub struct UpdatesResults {
-    pub(crate) updates_results: heed::Database<OwnedType<BEU64>, SerdeJson<ProcessedUpdateResult>>,
-}
-
-impl UpdatesResults {
-    pub fn last_update_id(
-        self,
-        reader: &heed::RoTxn,
-    ) -> ZResult<Option<(u64, ProcessedUpdateResult)>> {
-        match self.updates_results.last(reader)? {
-            Some((key, data)) => Ok(Some((key.get(), data))),
-            None => Ok(None),
-        }
-    }
-
-    pub fn put_update_result(
-        self,
-        writer: &mut heed::RwTxn,
-        update_id: u64,
-        update_result: &ProcessedUpdateResult,
-    ) -> ZResult<()> {
-        let update_id = BEU64::new(update_id);
-        self.updates_results.put(writer, &update_id, update_result)
-    }
-
-    pub fn update_result(
-        self,
-        reader: &heed::RoTxn,
-        update_id: u64,
-    ) -> ZResult<Option<ProcessedUpdateResult>> {
-        let update_id = BEU64::new(update_id);
-        self.updates_results.get(reader, &update_id)
-    }
-
-    pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> {
-        self.updates_results.clear(writer)
-    }
-}

meilidb-core/src/update/clear_all.rs

@@ -1,33 +0,0 @@
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult, RankedMap};
-
-pub fn apply_clear_all(
-    writer: &mut heed::RwTxn,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-) -> MResult<()> {
-    main_store.put_words_fst(writer, &fst::Set::default())?;
-    main_store.put_ranked_map(writer, &RankedMap::default())?;
-    main_store.put_number_of_documents(writer, |_| 0)?;
-    documents_fields_store.clear(writer)?;
-    documents_fields_counts_store.clear(writer)?;
-    postings_lists_store.clear(writer)?;
-    docs_words_store.clear(writer)?;
-
-    Ok(())
-}
-
-pub fn push_clear_all(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-    let update = Update::clear_all();
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}

meilidb-core/src/update/customs_update.rs

@@ -1,25 +0,0 @@
-use crate::store;
-use crate::update::{next_update_id, Update};
-use heed::Result as ZResult;
-
-pub fn apply_customs_update(
-    writer: &mut heed::RwTxn,
-    main_store: store::Main,
-    customs: &[u8],
-) -> ZResult<()> {
-    main_store.put_customs(writer, customs)
-}
-
-pub fn push_customs_update(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    customs: Vec<u8>,
-) -> ZResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::customs(customs);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}

meilidb-core/src/update/documents_addition.rs

@@ -1,410 +0,0 @@
-use std::collections::HashMap;
-
-use fst::{set::OpBuilder, SetBuilder};
-use sdset::{duo::Union, SetOperation};
-use serde::{Deserialize, Serialize};
-
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::raw_indexer::RawIndexer;
-use crate::serde::{extract_document_id, serialize_value, Deserializer, Serializer};
-use crate::store;
-use crate::update::{apply_documents_deletion, next_update_id, Update};
-use crate::{Error, MResult, RankedMap};
-
-pub struct DocumentsAddition<D> {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: UpdateEventsEmitter,
-    documents: Vec<D>,
-    is_partial: bool,
-}
-
-impl<D> DocumentsAddition<D> {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: UpdateEventsEmitter,
-    ) -> DocumentsAddition<D> {
-        DocumentsAddition {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            documents: Vec::new(),
-            is_partial: false,
-        }
-    }
-
-    pub fn new_partial(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: UpdateEventsEmitter,
-    ) -> DocumentsAddition<D> {
-        DocumentsAddition {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            documents: Vec::new(),
-            is_partial: true,
-        }
-    }
-
-    pub fn update_document(&mut self, document: D) {
-        self.documents.push(document);
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64>
-    where
-        D: serde::Serialize,
-    {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        let update_id = push_documents_addition(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.documents,
-            self.is_partial,
-        )?;
-        Ok(update_id)
-    }
-}
-
-impl<D> Extend<D> for DocumentsAddition<D> {
-    fn extend<T: IntoIterator<Item = D>>(&mut self, iter: T) {
-        self.documents.extend(iter)
-    }
-}
-
-pub fn push_documents_addition<D: serde::Serialize>(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    addition: Vec<D>,
-    is_partial: bool,
-) -> MResult<u64> {
-    let mut values = Vec::with_capacity(addition.len());
-    for add in addition {
-        let vec = serde_json::to_vec(&add)?;
-        let add = serde_json::from_slice(&vec)?;
-        values.push(add);
-    }
-
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = if is_partial {
-        Update::documents_partial(values)
-    } else {
-        Update::documents_addition(values)
-    };
-
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_documents_addition<'a, 'b>(
-    writer: &'a mut heed::RwTxn<'b>,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-    addition: Vec<HashMap<String, serde_json::Value>>,
-) -> MResult<()> {
-    let mut documents_additions = HashMap::new();
-
-    let schema = match main_store.schema(writer)? {
-        Some(schema) => schema,
-        None => return Err(Error::SchemaMissing),
-    };
-
-    let identifier = schema.identifier_name();
-
-    // 1. store documents ids for future deletion
-    for document in addition {
-        let document_id = match extract_document_id(identifier, &document)? {
-            Some(id) => id,
-            None => return Err(Error::MissingDocumentId),
-        };
-
-        documents_additions.insert(document_id, document);
-    }
-
-    // 2. remove the documents posting lists
-    let number_of_inserted_documents = documents_additions.len();
-    let documents_ids = documents_additions.iter().map(|(id, _)| *id).collect();
-    apply_documents_deletion(
-        writer,
-        main_store,
-        documents_fields_store,
-        documents_fields_counts_store,
-        postings_lists_store,
-        docs_words_store,
-        documents_ids,
-    )?;
-
-    let mut ranked_map = match main_store.ranked_map(writer)? {
-        Some(ranked_map) => ranked_map,
-        None => RankedMap::default(),
-    };
-
-    let stop_words = match main_store.stop_words_fst(writer)? {
-        Some(stop_words) => stop_words,
-        None => fst::Set::default(),
-    };
-
-    // 3. index the documents fields in the stores
-    let mut indexer = RawIndexer::new(stop_words);
-
-    for (document_id, document) in documents_additions {
-        let serializer = Serializer {
-            txn: writer,
-            schema: &schema,
-            document_store: documents_fields_store,
-            document_fields_counts: documents_fields_counts_store,
-            indexer: &mut indexer,
-            ranked_map: &mut ranked_map,
-            document_id,
-        };
-
-        document.serialize(serializer)?;
-    }
-
-    write_documents_addition_index(
-        writer,
-        main_store,
-        postings_lists_store,
-        docs_words_store,
-        &ranked_map,
-        number_of_inserted_documents,
-        indexer,
-    )
-}
-
-pub fn apply_documents_partial_addition<'a, 'b>(
-    writer: &'a mut heed::RwTxn<'b>,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-    addition: Vec<HashMap<String, serde_json::Value>>,
-) -> MResult<()> {
-    let mut documents_additions = HashMap::new();
-
-    let schema = match main_store.schema(writer)? {
-        Some(schema) => schema,
-        None => return Err(Error::SchemaMissing),
-    };
-
-    let identifier = schema.identifier_name();
-
-    // 1. store documents ids for future deletion
-    for mut document in addition {
-        let document_id = match extract_document_id(identifier, &document)? {
-            Some(id) => id,
-            None => return Err(Error::MissingDocumentId),
-        };
-
-        let mut deserializer = Deserializer {
-            document_id,
-            reader: writer,
-            documents_fields: documents_fields_store,
-            schema: &schema,
-            attributes: None,
-        };
-
-        // retrieve the old document and
-        // update the new one with missing keys found in the old one
-        let result = Option::<HashMap<String, serde_json::Value>>::deserialize(&mut deserializer)?;
-        if let Some(old_document) = result {
-            for (key, value) in old_document {
-                document.entry(key).or_insert(value);
-            }
-        }
-
-        documents_additions.insert(document_id, document);
-    }
-
-    // 2. remove the documents posting lists
-    let number_of_inserted_documents = documents_additions.len();
-    let documents_ids = documents_additions.iter().map(|(id, _)| *id).collect();
-    apply_documents_deletion(
-        writer,
-        main_store,
-        documents_fields_store,
-        documents_fields_counts_store,
-        postings_lists_store,
-        docs_words_store,
-        documents_ids,
-    )?;
-
-    let mut ranked_map = match main_store.ranked_map(writer)? {
-        Some(ranked_map) => ranked_map,
-        None => RankedMap::default(),
-    };
-
-    let stop_words = match main_store.stop_words_fst(writer)? {
-        Some(stop_words) => stop_words,
-        None => fst::Set::default(),
-    };
-
-    // 3. index the documents fields in the stores
-    let mut indexer = RawIndexer::new(stop_words);
-
-    for (document_id, document) in documents_additions {
-        let serializer = Serializer {
-            txn: writer,
-            schema: &schema,
-            document_store: documents_fields_store,
-            document_fields_counts: documents_fields_counts_store,
-            indexer: &mut indexer,
-            ranked_map: &mut ranked_map,
-            document_id,
-        };
-
-        document.serialize(serializer)?;
-    }
-
-    write_documents_addition_index(
-        writer,
-        main_store,
-        postings_lists_store,
-        docs_words_store,
-        &ranked_map,
-        number_of_inserted_documents,
-        indexer,
-    )
-}
-
-pub fn reindex_all_documents(
-    writer: &mut heed::RwTxn,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-) -> MResult<()> {
-    let schema = match main_store.schema(writer)? {
-        Some(schema) => schema,
-        None => return Err(Error::SchemaMissing),
-    };
-
-    let mut ranked_map = RankedMap::default();
-
-    // 1. retrieve all documents ids
-    let mut documents_ids_to_reindex = Vec::new();
-    for result in documents_fields_counts_store.documents_ids(writer)? {
-        let document_id = result?;
-        documents_ids_to_reindex.push(document_id);
-    }
-
-    // 2. remove the documents posting lists
-    main_store.put_words_fst(writer, &fst::Set::default())?;
-    main_store.put_ranked_map(writer, &ranked_map)?;
-    main_store.put_number_of_documents(writer, |_| 0)?;
-    postings_lists_store.clear(writer)?;
-    docs_words_store.clear(writer)?;
-
-    // 3. re-index chunks of documents (otherwise we make the borrow checker unhappy)
-    for documents_ids in documents_ids_to_reindex.chunks(100) {
-        let stop_words = match main_store.stop_words_fst(writer)? {
-            Some(stop_words) => stop_words,
-            None => fst::Set::default(),
-        };
-
-        let number_of_inserted_documents = documents_ids.len();
-        let mut indexer = RawIndexer::new(stop_words);
-        let mut ram_store = HashMap::new();
-
-        for document_id in documents_ids {
-            for result in documents_fields_store.document_fields(writer, *document_id)? {
-                let (attr, bytes) = result?;
-                let value: serde_json::Value = serde_json::from_slice(bytes)?;
-                ram_store.insert((document_id, attr), value);
-            }
-
-            for ((docid, attr), value) in ram_store.drain() {
-                serialize_value(
-                    writer,
-                    attr,
-                    schema.props(attr),
-                    *docid,
-                    documents_fields_store,
-                    documents_fields_counts_store,
-                    &mut indexer,
-                    &mut ranked_map,
-                    &value,
-                )?;
-            }
-        }
-
-        // 4. write the new index in the main store
-        write_documents_addition_index(
-            writer,
-            main_store,
-            postings_lists_store,
-            docs_words_store,
-            &ranked_map,
-            number_of_inserted_documents,
-            indexer,
-        )?;
-    }
-
-    Ok(())
-}
-
-pub fn write_documents_addition_index(
-    writer: &mut heed::RwTxn,
-    main_store: store::Main,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-    ranked_map: &RankedMap,
-    number_of_inserted_documents: usize,
-    indexer: RawIndexer,
-) -> MResult<()> {
-    let indexed = indexer.build();
-    let mut delta_words_builder = SetBuilder::memory();
-
-    for (word, delta_set) in indexed.words_doc_indexes {
-        delta_words_builder.insert(&word).unwrap();
-
-        let set = match postings_lists_store.postings_list(writer, &word)? {
-            Some(set) => Union::new(&set, &delta_set).into_set_buf(),
-            None => delta_set,
-        };
-
-        postings_lists_store.put_postings_list(writer, &word, &set)?;
-    }
-
-    for (id, words) in indexed.docs_words {
-        docs_words_store.put_doc_words(writer, id, &words)?;
-    }
-
-    let delta_words = delta_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    let words = match main_store.words_fst(writer)? {
-        Some(words) => {
-            let op = OpBuilder::new()
-                .add(words.stream())
-                .add(delta_words.stream())
-                .r#union();
-
-            let mut words_builder = SetBuilder::memory();
-            words_builder.extend_stream(op).unwrap();
-            words_builder
-                .into_inner()
-                .and_then(fst::Set::from_bytes)
-                .unwrap()
-        }
-        None => delta_words,
-    };
-
-    main_store.put_words_fst(writer, &words)?;
-    main_store.put_ranked_map(writer, ranked_map)?;
-    main_store.put_number_of_documents(writer, |old| old + number_of_inserted_documents as u64)?;
-
-    Ok(())
-}

meilidb-core/src/update/documents_deletion.rs

@@ -1,192 +0,0 @@
-use std::collections::{BTreeSet, HashMap, HashSet};
-
-use fst::{SetBuilder, Streamer};
-use meilidb_schema::Schema;
-use sdset::{duo::DifferenceByKey, SetBuf, SetOperation};
-
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::serde::extract_document_id;
-use crate::store;
-use crate::update::{next_update_id, Update};
-use crate::{DocumentId, Error, MResult, RankedMap};
-
-pub struct DocumentsDeletion {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: UpdateEventsEmitter,
-    documents: Vec<DocumentId>,
-}
-
-impl DocumentsDeletion {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: UpdateEventsEmitter,
-    ) -> DocumentsDeletion {
-        DocumentsDeletion {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            documents: Vec::new(),
-        }
-    }
-
-    pub fn delete_document_by_id(&mut self, document_id: DocumentId) {
-        self.documents.push(document_id);
-    }
-
-    pub fn delete_document<D>(&mut self, schema: &Schema, document: D) -> MResult<()>
-    where
-        D: serde::Serialize,
-    {
-        let identifier = schema.identifier_name();
-        let document_id = match extract_document_id(identifier, &document)? {
-            Some(id) => id,
-            None => return Err(Error::MissingDocumentId),
-        };
-
-        self.delete_document_by_id(document_id);
-
-        Ok(())
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        let update_id = push_documents_deletion(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.documents,
-        )?;
-        Ok(update_id)
-    }
-}
-
-impl Extend<DocumentId> for DocumentsDeletion {
-    fn extend<T: IntoIterator<Item = DocumentId>>(&mut self, iter: T) {
-        self.documents.extend(iter)
-    }
-}
-
-pub fn push_documents_deletion(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    deletion: Vec<DocumentId>,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::documents_deletion(deletion);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_documents_deletion(
-    writer: &mut heed::RwTxn,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-    deletion: Vec<DocumentId>,
-) -> MResult<()> {
-    let idset = SetBuf::from_dirty(deletion);
-
-    let schema = match main_store.schema(writer)? {
-        Some(schema) => schema,
-        None => return Err(Error::SchemaMissing),
-    };
-
-    let mut ranked_map = match main_store.ranked_map(writer)? {
-        Some(ranked_map) => ranked_map,
-        None => RankedMap::default(),
-    };
-
-    // collect the ranked attributes according to the schema
-    let ranked_attrs: Vec<_> = schema
-        .iter()
-        .filter_map(
-            |(_, attr, prop)| {
-                if prop.is_ranked() {
-                    Some(attr)
-                } else {
-                    None
-                }
-            },
-        )
-        .collect();
-
-    let mut words_document_ids = HashMap::new();
-    for id in idset {
-        // remove all the ranked attributes from the ranked_map
-        for ranked_attr in &ranked_attrs {
-            ranked_map.remove(id, *ranked_attr);
-        }
-
-        if let Some(words) = docs_words_store.doc_words(writer, id)? {
-            let mut stream = words.stream();
-            while let Some(word) = stream.next() {
-                let word = word.to_vec();
-                words_document_ids
-                    .entry(word)
-                    .or_insert_with(Vec::new)
-                    .push(id);
-            }
-        }
-    }
-
-    let mut deleted_documents = HashSet::new();
-    let mut removed_words = BTreeSet::new();
-    for (word, document_ids) in words_document_ids {
-        let document_ids = SetBuf::from_dirty(document_ids);
-
-        if let Some(doc_indexes) = postings_lists_store.postings_list(writer, &word)? {
-            let op = DifferenceByKey::new(&doc_indexes, &document_ids, |d| d.document_id, |id| *id);
-            let doc_indexes = op.into_set_buf();
-
-            if !doc_indexes.is_empty() {
-                postings_lists_store.put_postings_list(writer, &word, &doc_indexes)?;
-            } else {
-                postings_lists_store.del_postings_list(writer, &word)?;
-                removed_words.insert(word);
-            }
-        }
-
-        for id in document_ids {
-            documents_fields_counts_store.del_all_document_fields_counts(writer, id)?;
-            if documents_fields_store.del_all_document_fields(writer, id)? != 0 {
-                deleted_documents.insert(id);
-            }
-        }
-    }
-
-    let deleted_documents_len = deleted_documents.len() as u64;
-    for id in deleted_documents {
-        docs_words_store.del_doc_words(writer, id)?;
-    }
-
-    let removed_words = fst::Set::from_iter(removed_words).unwrap();
-    let words = match main_store.words_fst(writer)? {
-        Some(words_set) => {
-            let op = fst::set::OpBuilder::new()
-                .add(words_set.stream())
-                .add(removed_words.stream())
-                .difference();
-
-            let mut words_builder = SetBuilder::memory();
-            words_builder.extend_stream(op).unwrap();
-            words_builder
-                .into_inner()
-                .and_then(fst::Set::from_bytes)
-                .unwrap()
-        }
-        None => fst::Set::default(),
-    };
-
-    main_store.put_words_fst(writer, &words)?;
-    main_store.put_ranked_map(writer, &ranked_map)?;
-    main_store.put_number_of_documents(writer, |old| old - deleted_documents_len)?;
-
-    Ok(())
-}

meilidb-core/src/update/mod.rs

@@ -1,420 +0,0 @@
-mod clear_all;
-mod customs_update;
-mod documents_addition;
-mod documents_deletion;
-mod schema_update;
-mod stop_words_addition;
-mod stop_words_deletion;
-mod synonyms_addition;
-mod synonyms_deletion;
-
-pub use self::clear_all::{apply_clear_all, push_clear_all};
-pub use self::customs_update::{apply_customs_update, push_customs_update};
-pub use self::documents_addition::{
-    apply_documents_addition, apply_documents_partial_addition, DocumentsAddition,
-};
-pub use self::documents_deletion::{apply_documents_deletion, DocumentsDeletion};
-pub use self::schema_update::{apply_schema_update, push_schema_update};
-pub use self::stop_words_addition::{apply_stop_words_addition, StopWordsAddition};
-pub use self::stop_words_deletion::{apply_stop_words_deletion, StopWordsDeletion};
-pub use self::synonyms_addition::{apply_synonyms_addition, SynonymsAddition};
-pub use self::synonyms_deletion::{apply_synonyms_deletion, SynonymsDeletion};
-
-use std::cmp;
-use std::collections::{BTreeMap, BTreeSet, HashMap};
-use std::time::Instant;
-
-use chrono::{DateTime, Utc};
-use heed::Result as ZResult;
-use log::debug;
-use serde::{Deserialize, Serialize};
-
-use crate::{store, DocumentId, MResult};
-use meilidb_schema::Schema;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct Update {
-    data: UpdateData,
-    enqueued_at: DateTime<Utc>,
-}
-
-impl Update {
-    fn clear_all() -> Update {
-        Update {
-            data: UpdateData::ClearAll,
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn schema(data: Schema) -> Update {
-        Update {
-            data: UpdateData::Schema(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn customs(data: Vec<u8>) -> Update {
-        Update {
-            data: UpdateData::Customs(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn documents_addition(data: Vec<HashMap<String, serde_json::Value>>) -> Update {
-        Update {
-            data: UpdateData::DocumentsAddition(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn documents_partial(data: Vec<HashMap<String, serde_json::Value>>) -> Update {
-        Update {
-            data: UpdateData::DocumentsPartial(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn documents_deletion(data: Vec<DocumentId>) -> Update {
-        Update {
-            data: UpdateData::DocumentsDeletion(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn synonyms_addition(data: BTreeMap<String, Vec<String>>) -> Update {
-        Update {
-            data: UpdateData::SynonymsAddition(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn synonyms_deletion(data: BTreeMap<String, Option<Vec<String>>>) -> Update {
-        Update {
-            data: UpdateData::SynonymsDeletion(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn stop_words_addition(data: BTreeSet<String>) -> Update {
-        Update {
-            data: UpdateData::StopWordsAddition(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-
-    fn stop_words_deletion(data: BTreeSet<String>) -> Update {
-        Update {
-            data: UpdateData::StopWordsDeletion(data),
-            enqueued_at: Utc::now(),
-        }
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum UpdateData {
-    ClearAll,
-    Schema(Schema),
-    Customs(Vec<u8>),
-    DocumentsAddition(Vec<HashMap<String, serde_json::Value>>),
-    DocumentsPartial(Vec<HashMap<String, serde_json::Value>>),
-    DocumentsDeletion(Vec<DocumentId>),
-    SynonymsAddition(BTreeMap<String, Vec<String>>),
-    SynonymsDeletion(BTreeMap<String, Option<Vec<String>>>),
-    StopWordsAddition(BTreeSet<String>),
-    StopWordsDeletion(BTreeSet<String>),
-}
-
-impl UpdateData {
-    pub fn update_type(&self) -> UpdateType {
-        match self {
-            UpdateData::ClearAll => UpdateType::ClearAll,
-            UpdateData::Schema(_) => UpdateType::Schema,
-            UpdateData::Customs(_) => UpdateType::Customs,
-            UpdateData::DocumentsAddition(addition) => UpdateType::DocumentsAddition {
-                number: addition.len(),
-            },
-            UpdateData::DocumentsPartial(addition) => UpdateType::DocumentsPartial {
-                number: addition.len(),
-            },
-            UpdateData::DocumentsDeletion(deletion) => UpdateType::DocumentsDeletion {
-                number: deletion.len(),
-            },
-            UpdateData::SynonymsAddition(addition) => UpdateType::SynonymsAddition {
-                number: addition.len(),
-            },
-            UpdateData::SynonymsDeletion(deletion) => UpdateType::SynonymsDeletion {
-                number: deletion.len(),
-            },
-            UpdateData::StopWordsAddition(addition) => UpdateType::StopWordsAddition {
-                number: addition.len(),
-            },
-            UpdateData::StopWordsDeletion(deletion) => UpdateType::StopWordsDeletion {
-                number: deletion.len(),
-            },
-        }
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(tag = "name")]
-pub enum UpdateType {
-    ClearAll,
-    Schema,
-    Customs,
-    DocumentsAddition { number: usize },
-    DocumentsPartial { number: usize },
-    DocumentsDeletion { number: usize },
-    SynonymsAddition { number: usize },
-    SynonymsDeletion { number: usize },
-    StopWordsAddition { number: usize },
-    StopWordsDeletion { number: usize },
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ProcessedUpdateResult {
-    pub update_id: u64,
-    #[serde(rename = "type")]
-    pub update_type: UpdateType,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub error: Option<String>,
-    pub duration: f64, // in seconds
-    pub enqueued_at: DateTime<Utc>,
-    pub processed_at: DateTime<Utc>,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct EnqueuedUpdateResult {
-    pub update_id: u64,
-    pub update_type: UpdateType,
-    pub enqueued_at: DateTime<Utc>,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase", tag = "status")]
-pub enum UpdateStatus {
-    Enqueued {
-        #[serde(flatten)]
-        content: EnqueuedUpdateResult,
-    },
-    Processed {
-        #[serde(flatten)]
-        content: ProcessedUpdateResult,
-    },
-}
-
-pub fn update_status(
-    reader: &heed::RoTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    update_id: u64,
-) -> MResult<Option<UpdateStatus>> {
-    match updates_results_store.update_result(reader, update_id)? {
-        Some(result) => Ok(Some(UpdateStatus::Processed { content: result })),
-        None => match updates_store.get(reader, update_id)? {
-            Some(update) => Ok(Some(UpdateStatus::Enqueued {
-                content: EnqueuedUpdateResult {
-                    update_id,
-                    update_type: update.data.update_type(),
-                    enqueued_at: update.enqueued_at,
-                },
-            })),
-            None => Ok(None),
-        },
-    }
-}
-
-pub fn next_update_id(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-) -> ZResult<u64> {
-    let last_update_id = updates_store.last_update_id(writer)?;
-    let last_update_id = last_update_id.map(|(n, _)| n);
-
-    let last_update_results_id = updates_results_store.last_update_id(writer)?;
-    let last_update_results_id = last_update_results_id.map(|(n, _)| n);
-
-    let max_update_id = cmp::max(last_update_id, last_update_results_id);
-    let new_update_id = max_update_id.map_or(0, |n| n + 1);
-
-    Ok(new_update_id)
-}
-
-pub fn update_task<'a, 'b>(
-    writer: &'a mut heed::RwTxn<'b>,
-    index: store::Index,
-    update_id: u64,
-    update: Update,
-) -> MResult<ProcessedUpdateResult> {
-    debug!("Processing update number {}", update_id);
-
-    let Update { enqueued_at, data } = update;
-
-    let (update_type, result, duration) = match data {
-        UpdateData::ClearAll => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::ClearAll;
-            let result = apply_clear_all(
-                writer,
-                index.main,
-                index.documents_fields,
-                index.documents_fields_counts,
-                index.postings_lists,
-                index.docs_words,
-            );
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::Schema(schema) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::Schema;
-            let result = apply_schema_update(
-                writer,
-                &schema,
-                index.main,
-                index.documents_fields,
-                index.documents_fields_counts,
-                index.postings_lists,
-                index.docs_words,
-            );
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::Customs(customs) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::Customs;
-            let result = apply_customs_update(writer, index.main, &customs).map_err(Into::into);
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::DocumentsAddition(documents) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::DocumentsAddition {
-                number: documents.len(),
-            };
-
-            let result = apply_documents_addition(
-                writer,
-                index.main,
-                index.documents_fields,
-                index.documents_fields_counts,
-                index.postings_lists,
-                index.docs_words,
-                documents,
-            );
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::DocumentsPartial(documents) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::DocumentsPartial {
-                number: documents.len(),
-            };
-
-            let result = apply_documents_partial_addition(
-                writer,
-                index.main,
-                index.documents_fields,
-                index.documents_fields_counts,
-                index.postings_lists,
-                index.docs_words,
-                documents,
-            );
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::DocumentsDeletion(documents) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::DocumentsDeletion {
-                number: documents.len(),
-            };
-
-            let result = apply_documents_deletion(
-                writer,
-                index.main,
-                index.documents_fields,
-                index.documents_fields_counts,
-                index.postings_lists,
-                index.docs_words,
-                documents,
-            );
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::SynonymsAddition(synonyms) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::SynonymsAddition {
-                number: synonyms.len(),
-            };
-
-            let result = apply_synonyms_addition(writer, index.main, index.synonyms, synonyms);
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::SynonymsDeletion(synonyms) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::SynonymsDeletion {
-                number: synonyms.len(),
-            };
-
-            let result = apply_synonyms_deletion(writer, index.main, index.synonyms, synonyms);
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::StopWordsAddition(stop_words) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::StopWordsAddition {
-                number: stop_words.len(),
-            };
-
-            let result =
-                apply_stop_words_addition(writer, index.main, index.postings_lists, stop_words);
-
-            (update_type, result, start.elapsed())
-        }
-        UpdateData::StopWordsDeletion(stop_words) => {
-            let start = Instant::now();
-
-            let update_type = UpdateType::StopWordsDeletion {
-                number: stop_words.len(),
-            };
-
-            let result = apply_stop_words_deletion(
-                writer,
-                index.main,
-                index.documents_fields,
-                index.documents_fields_counts,
-                index.postings_lists,
-                index.docs_words,
-                stop_words,
-            );
-
-            (update_type, result, start.elapsed())
-        }
-    };
-
-    debug!(
-        "Processed update number {} {:?} {:?}",
-        update_id, update_type, result
-    );
-
-    let status = ProcessedUpdateResult {
-        update_id,
-        update_type,
-        error: result.map_err(|e| e.to_string()).err(),
-        duration: duration.as_secs_f64(),
-        enqueued_at,
-        processed_at: Utc::now(),
-    };
-
-    Ok(status)
-}

meilidb-core/src/update/schema_update.rs

@@ -1,75 +0,0 @@
-use meilidb_schema::{Diff, Schema};
-
-use crate::update::documents_addition::reindex_all_documents;
-use crate::update::{next_update_id, Update};
-use crate::{error::UnsupportedOperation, store, MResult};
-
-pub fn apply_schema_update(
-    writer: &mut heed::RwTxn,
-    new_schema: &Schema,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-) -> MResult<()> {
-    use UnsupportedOperation::{
-        CanOnlyIntroduceNewSchemaAttributesAtEnd, CannotRemoveSchemaAttribute,
-        CannotReorderSchemaAttribute, CannotUpdateSchemaIdentifier,
-    };
-
-    let mut need_full_reindexing = false;
-
-    if let Some(old_schema) = main_store.schema(writer)? {
-        for diff in meilidb_schema::diff(&old_schema, new_schema) {
-            match diff {
-                Diff::IdentChange { .. } => return Err(CannotUpdateSchemaIdentifier.into()),
-                Diff::AttrMove { .. } => return Err(CannotReorderSchemaAttribute.into()),
-                Diff::AttrPropsChange { old, new, .. } => {
-                    if new.indexed != old.indexed {
-                        need_full_reindexing = true;
-                    }
-                    if new.ranked != old.ranked {
-                        need_full_reindexing = true;
-                    }
-                }
-                Diff::NewAttr { pos, .. } => {
-                    // new attribute not at the end of the schema
-                    if pos < old_schema.number_of_attributes() {
-                        return Err(CanOnlyIntroduceNewSchemaAttributesAtEnd.into());
-                    }
-                }
-                Diff::RemovedAttr { .. } => return Err(CannotRemoveSchemaAttribute.into()),
-            }
-        }
-    }
-
-    main_store.put_schema(writer, new_schema)?;
-
-    if need_full_reindexing {
-        reindex_all_documents(
-            writer,
-            main_store,
-            documents_fields_store,
-            documents_fields_counts_store,
-            postings_lists_store,
-            docs_words_store,
-        )?
-    }
-
-    Ok(())
-}
-
-pub fn push_schema_update(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    schema: Schema,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::schema(schema);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}

meilidb-core/src/update/stop_words_addition.rs

@@ -1,117 +0,0 @@
-use std::collections::BTreeSet;
-
-use fst::{set::OpBuilder, SetBuilder};
-
-use crate::automaton::normalize_str;
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult};
-
-pub struct StopWordsAddition {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: UpdateEventsEmitter,
-    stop_words: BTreeSet<String>,
-}
-
-impl StopWordsAddition {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: UpdateEventsEmitter,
-    ) -> StopWordsAddition {
-        StopWordsAddition {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            stop_words: BTreeSet::new(),
-        }
-    }
-
-    pub fn add_stop_word<S: AsRef<str>>(&mut self, stop_word: S) {
-        let stop_word = normalize_str(stop_word.as_ref());
-        self.stop_words.insert(stop_word);
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        let update_id = push_stop_words_addition(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.stop_words,
-        )?;
-        Ok(update_id)
-    }
-}
-
-pub fn push_stop_words_addition(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    addition: BTreeSet<String>,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::stop_words_addition(addition);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_stop_words_addition(
-    writer: &mut heed::RwTxn,
-    main_store: store::Main,
-    postings_lists_store: store::PostingsLists,
-    addition: BTreeSet<String>,
-) -> MResult<()> {
-    let mut stop_words_builder = SetBuilder::memory();
-
-    for word in addition {
-        stop_words_builder.insert(&word).unwrap();
-        // we remove every posting list associated to a new stop word
-        postings_lists_store.del_postings_list(writer, word.as_bytes())?;
-    }
-
-    // create the new delta stop words fst
-    let delta_stop_words = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    // we also need to remove all the stop words from the main fst
-    if let Some(word_fst) = main_store.words_fst(writer)? {
-        let op = OpBuilder::new()
-            .add(&word_fst)
-            .add(&delta_stop_words)
-            .difference();
-
-        let mut word_fst_builder = SetBuilder::memory();
-        word_fst_builder.extend_stream(op).unwrap();
-        let word_fst = word_fst_builder
-            .into_inner()
-            .and_then(fst::Set::from_bytes)
-            .unwrap();
-
-        main_store.put_words_fst(writer, &word_fst)?;
-    }
-
-    // now we add all of these stop words from the main store
-    let stop_words_fst = main_store.stop_words_fst(writer)?.unwrap_or_default();
-
-    let op = OpBuilder::new()
-        .add(&stop_words_fst)
-        .add(&delta_stop_words)
-        .r#union();
-
-    let mut stop_words_builder = SetBuilder::memory();
-    stop_words_builder.extend_stream(op).unwrap();
-    let stop_words_fst = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    main_store.put_stop_words_fst(writer, &stop_words_fst)?;
-
-    Ok(())
-}

meilidb-core/src/update/stop_words_deletion.rs

@@ -1,113 +0,0 @@
-use std::collections::BTreeSet;
-
-use fst::{set::OpBuilder, SetBuilder};
-
-use crate::automaton::normalize_str;
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::update::documents_addition::reindex_all_documents;
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult};
-
-pub struct StopWordsDeletion {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: UpdateEventsEmitter,
-    stop_words: BTreeSet<String>,
-}
-
-impl StopWordsDeletion {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: UpdateEventsEmitter,
-    ) -> StopWordsDeletion {
-        StopWordsDeletion {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            stop_words: BTreeSet::new(),
-        }
-    }
-
-    pub fn delete_stop_word<S: AsRef<str>>(&mut self, stop_word: S) {
-        let stop_word = normalize_str(stop_word.as_ref());
-        self.stop_words.insert(stop_word);
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        let update_id = push_stop_words_deletion(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.stop_words,
-        )?;
-        Ok(update_id)
-    }
-}
-
-pub fn push_stop_words_deletion(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    deletion: BTreeSet<String>,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::stop_words_deletion(deletion);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_stop_words_deletion(
-    writer: &mut heed::RwTxn,
-    main_store: store::Main,
-    documents_fields_store: store::DocumentsFields,
-    documents_fields_counts_store: store::DocumentsFieldsCounts,
-    postings_lists_store: store::PostingsLists,
-    docs_words_store: store::DocsWords,
-    deletion: BTreeSet<String>,
-) -> MResult<()> {
-    let mut stop_words_builder = SetBuilder::memory();
-
-    for word in deletion {
-        stop_words_builder.insert(&word).unwrap();
-    }
-
-    // create the new delta stop words fst
-    let delta_stop_words = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    // now we delete all of these stop words from the main store
-    let stop_words_fst = main_store.stop_words_fst(writer)?.unwrap_or_default();
-
-    let op = OpBuilder::new()
-        .add(&stop_words_fst)
-        .add(&delta_stop_words)
-        .difference();
-
-    let mut stop_words_builder = SetBuilder::memory();
-    stop_words_builder.extend_stream(op).unwrap();
-    let stop_words_fst = stop_words_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    main_store.put_stop_words_fst(writer, &stop_words_fst)?;
-
-    // now that we have setup the stop words
-    // lets reindex everything...
-    reindex_all_documents(
-        writer,
-        main_store,
-        documents_fields_store,
-        documents_fields_counts_store,
-        postings_lists_store,
-        docs_words_store,
-    )?;
-
-    Ok(())
-}

meilidb-core/src/update/synonyms_addition.rs

@@ -1,119 +0,0 @@
-use std::collections::BTreeMap;
-
-use fst::{set::OpBuilder, SetBuilder};
-use sdset::SetBuf;
-
-use crate::automaton::normalize_str;
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult};
-
-pub struct SynonymsAddition {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: UpdateEventsEmitter,
-    synonyms: BTreeMap<String, Vec<String>>,
-}
-
-impl SynonymsAddition {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: UpdateEventsEmitter,
-    ) -> SynonymsAddition {
-        SynonymsAddition {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            synonyms: BTreeMap::new(),
-        }
-    }
-
-    pub fn add_synonym<S, T, I>(&mut self, synonym: S, alternatives: I)
-    where
-        S: AsRef<str>,
-        T: AsRef<str>,
-        I: IntoIterator<Item = T>,
-    {
-        let synonym = normalize_str(synonym.as_ref());
-        let alternatives = alternatives.into_iter().map(|s| s.as_ref().to_lowercase());
-        self.synonyms
-            .entry(synonym)
-            .or_insert_with(Vec::new)
-            .extend(alternatives);
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        let update_id = push_synonyms_addition(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.synonyms,
-        )?;
-        Ok(update_id)
-    }
-}
-
-pub fn push_synonyms_addition(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    addition: BTreeMap<String, Vec<String>>,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::synonyms_addition(addition);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_synonyms_addition(
-    writer: &mut heed::RwTxn,
-    main_store: store::Main,
-    synonyms_store: store::Synonyms,
-    addition: BTreeMap<String, Vec<String>>,
-) -> MResult<()> {
-    let mut synonyms_builder = SetBuilder::memory();
-
-    for (word, alternatives) in addition {
-        synonyms_builder.insert(&word).unwrap();
-
-        let alternatives = {
-            let alternatives = SetBuf::from_dirty(alternatives);
-            let mut alternatives_builder = SetBuilder::memory();
-            alternatives_builder.extend_iter(alternatives).unwrap();
-            let bytes = alternatives_builder.into_inner().unwrap();
-            fst::Set::from_bytes(bytes).unwrap()
-        };
-
-        synonyms_store.put_synonyms(writer, word.as_bytes(), &alternatives)?;
-    }
-
-    let delta_synonyms = synonyms_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    let synonyms = match main_store.synonyms_fst(writer)? {
-        Some(synonyms) => {
-            let op = OpBuilder::new()
-                .add(synonyms.stream())
-                .add(delta_synonyms.stream())
-                .r#union();
-
-            let mut synonyms_builder = SetBuilder::memory();
-            synonyms_builder.extend_stream(op).unwrap();
-            synonyms_builder
-                .into_inner()
-                .and_then(fst::Set::from_bytes)
-                .unwrap()
-        }
-        None => delta_synonyms,
-    };
-
-    main_store.put_synonyms_fst(writer, &synonyms)?;
-
-    Ok(())
-}

meilidb-core/src/update/synonyms_deletion.rs

@@ -1,157 +0,0 @@
-use std::collections::BTreeMap;
-use std::iter::FromIterator;
-
-use fst::{set::OpBuilder, SetBuilder};
-use sdset::SetBuf;
-
-use crate::automaton::normalize_str;
-use crate::database::{UpdateEvent, UpdateEventsEmitter};
-use crate::update::{next_update_id, Update};
-use crate::{store, MResult};
-
-pub struct SynonymsDeletion {
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    updates_notifier: UpdateEventsEmitter,
-    synonyms: BTreeMap<String, Option<Vec<String>>>,
-}
-
-impl SynonymsDeletion {
-    pub fn new(
-        updates_store: store::Updates,
-        updates_results_store: store::UpdatesResults,
-        updates_notifier: UpdateEventsEmitter,
-    ) -> SynonymsDeletion {
-        SynonymsDeletion {
-            updates_store,
-            updates_results_store,
-            updates_notifier,
-            synonyms: BTreeMap::new(),
-        }
-    }
-
-    pub fn delete_all_alternatives_of<S: AsRef<str>>(&mut self, synonym: S) {
-        let synonym = normalize_str(synonym.as_ref());
-        self.synonyms.insert(synonym, None);
-    }
-
-    pub fn delete_specific_alternatives_of<S, T, I>(&mut self, synonym: S, alternatives: I)
-    where
-        S: AsRef<str>,
-        T: AsRef<str>,
-        I: Iterator<Item = T>,
-    {
-        let synonym = normalize_str(synonym.as_ref());
-        let value = self.synonyms.entry(synonym).or_insert(None);
-        let alternatives = alternatives.map(|s| s.as_ref().to_lowercase());
-        match value {
-            Some(v) => v.extend(alternatives),
-            None => *value = Some(Vec::from_iter(alternatives)),
-        }
-    }
-
-    pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64> {
-        let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
-        let update_id = push_synonyms_deletion(
-            writer,
-            self.updates_store,
-            self.updates_results_store,
-            self.synonyms,
-        )?;
-        Ok(update_id)
-    }
-}
-
-pub fn push_synonyms_deletion(
-    writer: &mut heed::RwTxn,
-    updates_store: store::Updates,
-    updates_results_store: store::UpdatesResults,
-    deletion: BTreeMap<String, Option<Vec<String>>>,
-) -> MResult<u64> {
-    let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
-
-    let update = Update::synonyms_deletion(deletion);
-    updates_store.put_update(writer, last_update_id, &update)?;
-
-    Ok(last_update_id)
-}
-
-pub fn apply_synonyms_deletion(
-    writer: &mut heed::RwTxn,
-    main_store: store::Main,
-    synonyms_store: store::Synonyms,
-    deletion: BTreeMap<String, Option<Vec<String>>>,
-) -> MResult<()> {
-    let mut delete_whole_synonym_builder = SetBuilder::memory();
-
-    for (synonym, alternatives) in deletion {
-        match alternatives {
-            Some(alternatives) => {
-                let prev_alternatives = synonyms_store.synonyms(writer, synonym.as_bytes())?;
-                let prev_alternatives = match prev_alternatives {
-                    Some(alternatives) => alternatives,
-                    None => continue,
-                };
-
-                let delta_alternatives = {
-                    let alternatives = SetBuf::from_dirty(alternatives);
-                    let mut builder = SetBuilder::memory();
-                    builder.extend_iter(alternatives).unwrap();
-                    builder.into_inner().and_then(fst::Set::from_bytes).unwrap()
-                };
-
-                let op = OpBuilder::new()
-                    .add(prev_alternatives.stream())
-                    .add(delta_alternatives.stream())
-                    .difference();
-
-                let (alternatives, empty_alternatives) = {
-                    let mut builder = SetBuilder::memory();
-                    let len = builder.get_ref().len();
-                    builder.extend_stream(op).unwrap();
-                    let is_empty = len == builder.get_ref().len();
-                    let bytes = builder.into_inner().unwrap();
-                    let alternatives = fst::Set::from_bytes(bytes).unwrap();
-
-                    (alternatives, is_empty)
-                };
-
-                if empty_alternatives {
-                    delete_whole_synonym_builder.insert(synonym.as_bytes())?;
-                } else {
-                    synonyms_store.put_synonyms(writer, synonym.as_bytes(), &alternatives)?;
-                }
-            }
-            None => {
-                delete_whole_synonym_builder.insert(&synonym).unwrap();
-                synonyms_store.del_synonyms(writer, synonym.as_bytes())?;
-            }
-        }
-    }
-
-    let delta_synonyms = delete_whole_synonym_builder
-        .into_inner()
-        .and_then(fst::Set::from_bytes)
-        .unwrap();
-
-    let synonyms = match main_store.synonyms_fst(writer)? {
-        Some(synonyms) => {
-            let op = OpBuilder::new()
-                .add(synonyms.stream())
-                .add(delta_synonyms.stream())
-                .difference();
-
-            let mut synonyms_builder = SetBuilder::memory();
-            synonyms_builder.extend_stream(op).unwrap();
-            synonyms_builder
-                .into_inner()
-                .and_then(fst::Set::from_bytes)
-                .unwrap()
-        }
-        None => fst::Set::default(),
-    };
-
-    main_store.put_synonyms_fst(writer, &synonyms)?;
-
-    Ok(())
-}

meilidb-http/Cargo.toml

@@ -1,56 +0,0 @@
-[package]
-name = "meilidb-http"
-version = "0.2.0"
-authors = [
-    "Quentin de Quelen <quentin@dequelen.me>",
-    "Clément Renault <clement@meilisearch.com>",
-]
-edition = "2018"
-
-[dependencies]
-bincode = "1.2.0"
-chrono = { version = "0.4.9", features = ["serde"] }
-crossbeam-channel = "0.4.0"
-env_logger = "0.7.1"
-envconfig = "0.5.1"
-envconfig_derive = "0.5.1"
-heed = "0.5.0"
-http = "0.1.19"
-indexmap = { version = "1.3.0", features = ["serde-1"] }
-jemallocator = "0.3.2"
-log = "0.4.8"
-main_error = "0.1.0"
-meilidb-core = { path = "../meilidb-core", version = "0.7.0" }
-meilidb-schema = { path = "../meilidb-schema", version = "0.6.0" }
-pretty-bytes = "0.2.2"
-rand = "0.7.2"
-rayon = "1.2.0"
-serde = { version = "1.0.101", features = ["derive"] }
-serde_json = { version = "1.0.41", features = ["preserve_order"] }
-structopt = "0.3.3"
-sysinfo = "0.9.5"
-walkdir = "2.2.9"
-
-[dependencies.async-compression]
-default-features = false
-features = ["stream", "gzip", "zlib", "brotli", "zstd"]
-version = "=0.1.0-alpha.7"
-
-[dependencies.tide]
-git = "https://github.com/rustasync/tide"
-rev = "e77709370bb24cf776fe6da902467c35131535b1"
-
-[dependencies.tide-log]
-git = "https://github.com/rustasync/tide"
-rev = "e77709370bb24cf776fe6da902467c35131535b1"
-
-[dependencies.tide-slog]
-git = "https://github.com/rustasync/tide"
-rev = "e77709370bb24cf776fe6da902467c35131535b1"
-
-[dependencies.tide-compression]
-git = "https://github.com/rustasync/tide"
-rev = "e77709370bb24cf776fe6da902467c35131535b1"
-
-[build-dependencies]
-vergen = "3.0.4"

meilidb-http/build.rs

@@ -1,10 +0,0 @@
-use vergen::{generate_cargo_keys, ConstantsFlags};
-
-fn main() {
-    // Setup the flags, toggling off the 'SEMVER_FROM_CARGO_PKG' flag
-    let mut flags = ConstantsFlags::all();
-    flags.toggle(ConstantsFlags::SEMVER_FROM_CARGO_PKG);
-
-    // Generate the 'cargo:' key output
-    generate_cargo_keys(ConstantsFlags::all()).expect("Unable to generate the cargo keys!");
-}

meilidb-http/src/data.rs

@@ -1,153 +0,0 @@
-use std::collections::HashMap;
-use std::ops::Deref;
-use std::sync::Arc;
-
-use chrono::{DateTime, Utc};
-use heed::types::{SerdeBincode, Str};
-use log::*;
-use meilidb_core::{Database, MResult};
-use sysinfo::Pid;
-
-use crate::option::Opt;
-use crate::routes::index::index_update_callback;
-
-pub type FreqsMap = HashMap<String, usize>;
-type SerdeFreqsMap = SerdeBincode<FreqsMap>;
-type SerdeDatetime = SerdeBincode<DateTime<Utc>>;
-
-#[derive(Clone)]
-pub struct Data {
-    inner: Arc<DataInner>,
-}
-
-impl Deref for Data {
-    type Target = DataInner;
-
-    fn deref(&self) -> &Self::Target {
-        &self.inner
-    }
-}
-
-#[derive(Clone)]
-pub struct DataInner {
-    pub db: Arc<Database>,
-    pub db_path: String,
-    pub admin_token: Option<String>,
-    pub server_pid: Pid,
-}
-
-impl DataInner {
-    pub fn is_indexing(&self, reader: &heed::RoTxn, index: &str) -> MResult<Option<bool>> {
-        match self.db.open_index(&index) {
-            Some(index) => index.current_update_id(&reader).map(|u| Some(u.is_some())),
-            None => Ok(None),
-        }
-    }
-
-    pub fn last_update(
-        &self,
-        reader: &heed::RoTxn,
-        index_name: &str,
-    ) -> MResult<Option<DateTime<Utc>>> {
-        let key = format!("last-update-{}", index_name);
-        match self
-            .db
-            .common_store()
-            .get::<Str, SerdeDatetime>(&reader, &key)?
-        {
-            Some(datetime) => Ok(Some(datetime)),
-            None => Ok(None),
-        }
-    }
-
-    pub fn set_last_update(&self, writer: &mut heed::RwTxn, index_name: &str) -> MResult<()> {
-        let key = format!("last-update-{}", index_name);
-        self.db
-            .common_store()
-            .put::<Str, SerdeDatetime>(writer, &key, &Utc::now())
-            .map_err(Into::into)
-    }
-
-    pub fn fields_frequency(
-        &self,
-        reader: &heed::RoTxn,
-        index_name: &str,
-    ) -> MResult<Option<FreqsMap>> {
-        let key = format!("fields-frequency-{}", index_name);
-        match self
-            .db
-            .common_store()
-            .get::<Str, SerdeFreqsMap>(&reader, &key)?
-        {
-            Some(freqs) => Ok(Some(freqs)),
-            None => Ok(None),
-        }
-    }
-
-    pub fn compute_stats(&self, writer: &mut heed::RwTxn, index_name: &str) -> MResult<()> {
-        let index = match self.db.open_index(&index_name) {
-            Some(index) => index,
-            None => {
-                error!("Impossible to retrieve index {}", index_name);
-                return Ok(());
-            }
-        };
-
-        let schema = match index.main.schema(&writer)? {
-            Some(schema) => schema,
-            None => return Ok(()),
-        };
-
-        let all_documents_fields = index
-            .documents_fields_counts
-            .all_documents_fields_counts(&writer)?;
-
-        // count fields frequencies
-        let mut fields_frequency = HashMap::<_, usize>::new();
-        for result in all_documents_fields {
-            let (_, attr, _) = result?;
-            *fields_frequency.entry(attr).or_default() += 1;
-        }
-
-        // convert attributes to their names
-        let frequency: HashMap<_, _> = fields_frequency
-            .into_iter()
-            .map(|(a, c)| (schema.attribute_name(a).to_owned(), c))
-            .collect();
-
-        let key = format!("fields-frequency-{}", index_name);
-        self.db
-            .common_store()
-            .put::<Str, SerdeFreqsMap>(writer, &key, &frequency)?;
-
-        Ok(())
-    }
-}
-
-impl Data {
-    pub fn new(opt: Opt) -> Data {
-        let db_path = opt.database_path.clone();
-        let admin_token = opt.admin_token.clone();
-        let server_pid = sysinfo::get_current_pid().unwrap();
-
-        let db = Arc::new(Database::open_or_create(opt.database_path.clone()).unwrap());
-
-        let inner_data = DataInner {
-            db: db.clone(),
-            db_path,
-            admin_token,
-            server_pid,
-        };
-
-        let data = Data {
-            inner: Arc::new(inner_data),
-        };
-
-        let callback_context = data.clone();
-        db.set_update_callback(Box::new(move |index_name, status| {
-            index_update_callback(&index_name, &callback_context, status);
-        }));
-
-        data
-    }
-}

meilidb-http/src/error.rs

@@ -1,126 +0,0 @@
-use std::fmt::Display;
-
-use http::status::StatusCode;
-use log::{error, warn};
-use serde::{Deserialize, Serialize};
-use tide::response::IntoResponse;
-use tide::Response;
-
-pub type SResult<T> = Result<T, ResponseError>;
-
-pub enum ResponseError {
-    Internal(String),
-    BadRequest(String),
-    InvalidToken(String),
-    NotFound(String),
-    IndexNotFound(String),
-    DocumentNotFound(String),
-    MissingHeader(String),
-    BadParameter(String, String),
-    OpenIndex(String),
-    CreateIndex(String),
-    Maintenance,
-}
-
-impl ResponseError {
-    pub fn internal(message: impl Display) -> ResponseError {
-        ResponseError::Internal(message.to_string())
-    }
-
-    pub fn bad_request(message: impl Display) -> ResponseError {
-        ResponseError::BadRequest(message.to_string())
-    }
-
-    pub fn invalid_token(message: impl Display) -> ResponseError {
-        ResponseError::InvalidToken(message.to_string())
-    }
-
-    pub fn not_found(message: impl Display) -> ResponseError {
-        ResponseError::NotFound(message.to_string())
-    }
-
-    pub fn index_not_found(message: impl Display) -> ResponseError {
-        ResponseError::IndexNotFound(message.to_string())
-    }
-
-    pub fn document_not_found(message: impl Display) -> ResponseError {
-        ResponseError::DocumentNotFound(message.to_string())
-    }
-
-    pub fn missing_header(message: impl Display) -> ResponseError {
-        ResponseError::MissingHeader(message.to_string())
-    }
-
-    pub fn bad_parameter(name: impl Display, message: impl Display) -> ResponseError {
-        ResponseError::BadParameter(name.to_string(), message.to_string())
-    }
-
-    pub fn open_index(message: impl Display) -> ResponseError {
-        ResponseError::OpenIndex(message.to_string())
-    }
-
-    pub fn create_index(message: impl Display) -> ResponseError {
-        ResponseError::CreateIndex(message.to_string())
-    }
-}
-
-impl IntoResponse for ResponseError {
-    fn into_response(self) -> Response {
-        match self {
-            ResponseError::Internal(err) => {
-                error!("internal server error: {}", err);
-                error(
-                    String::from("Internal server error"),
-                    StatusCode::INTERNAL_SERVER_ERROR,
-                )
-            }
-            ResponseError::BadRequest(err) => {
-                warn!("bad request: {}", err);
-                error(err, StatusCode::BAD_REQUEST)
-            }
-            ResponseError::InvalidToken(err) => {
-                error(format!("Invalid Token: {}", err), StatusCode::FORBIDDEN)
-            }
-            ResponseError::NotFound(err) => error(err, StatusCode::NOT_FOUND),
-            ResponseError::IndexNotFound(index) => {
-                error(format!("Index {} not found", index), StatusCode::NOT_FOUND)
-            }
-            ResponseError::DocumentNotFound(id) => error(
-                format!("Document with id {} not found", id),
-                StatusCode::NOT_FOUND,
-            ),
-            ResponseError::MissingHeader(header) => error(
-                format!("Header {} is missing", header),
-                StatusCode::UNAUTHORIZED,
-            ),
-            ResponseError::BadParameter(param, e) => error(
-                format!("Url parameter {} error: {}", param, e),
-                StatusCode::BAD_REQUEST,
-            ),
-            ResponseError::CreateIndex(err) => error(
-                format!("Impossible to create index; {}", err),
-                StatusCode::BAD_REQUEST,
-            ),
-            ResponseError::OpenIndex(err) => error(
-                format!("Impossible to open index; {}", err),
-                StatusCode::BAD_REQUEST,
-            ),
-            ResponseError::Maintenance => error(
-                String::from("Server is in maintenance, please try again later"),
-                StatusCode::SERVICE_UNAVAILABLE,
-            ),
-        }
-    }
-}
-
-#[derive(Serialize, Deserialize)]
-struct ErrorMessage {
-    message: String,
-}
-
-fn error(message: String, status: StatusCode) -> Response {
-    let message = ErrorMessage { message };
-    tide::response::json(message)
-        .with_status(status)
-        .into_response()
-}

meilidb-http/src/helpers/meilidb.rs

@@ -1,570 +0,0 @@
-use crate::routes::setting::{RankingOrdering, SettingBody};
-use indexmap::IndexMap;
-use log::*;
-use meilidb_core::criterion::*;
-use meilidb_core::Highlight;
-use meilidb_core::{Index, RankedMap};
-use meilidb_schema::{Schema, SchemaAttr};
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
-use std::cmp::Ordering;
-use std::collections::{HashMap, HashSet};
-use std::convert::From;
-use std::error;
-use std::fmt;
-use std::time::{Duration, Instant};
-
-#[derive(Debug)]
-pub enum Error {
-    SearchDocuments(String),
-    RetrieveDocument(u64, String),
-    DocumentNotFound(u64),
-    CropFieldWrongType(String),
-    AttributeNotFoundOnDocument(String),
-    AttributeNotFoundOnSchema(String),
-    MissingFilterValue,
-    UnknownFilteredAttribute,
-    Internal(String),
-}
-
-impl error::Error for Error {}
-
-impl fmt::Display for Error {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        use Error::*;
-
-        match self {
-            SearchDocuments(err) => write!(f, "impossible to search documents; {}", err),
-            RetrieveDocument(id, err) => write!(
-                f,
-                "impossible to retrieve the document with id: {}; {}",
-                id, err
-            ),
-            DocumentNotFound(id) => write!(f, "document {} not found", id),
-            CropFieldWrongType(field) => {
-                write!(f, "the field {} cannot be cropped it's not a string", field)
-            }
-            AttributeNotFoundOnDocument(field) => {
-                write!(f, "field {} is not found on document", field)
-            }
-            AttributeNotFoundOnSchema(field) => write!(f, "field {} is not found on schema", field),
-            MissingFilterValue => f.write_str("a filter doesn't have a value to compare it with"),
-            UnknownFilteredAttribute => {
-                f.write_str("a filter is specifying an unknown schema attribute")
-            }
-            Internal(err) => write!(f, "internal error; {}", err),
-        }
-    }
-}
-
-impl From<meilidb_core::Error> for Error {
-    fn from(error: meilidb_core::Error) -> Self {
-        Error::Internal(error.to_string())
-    }
-}
-
-pub trait IndexSearchExt {
-    fn new_search(&self, query: String) -> SearchBuilder;
-}
-
-impl IndexSearchExt for Index {
-    fn new_search(&self, query: String) -> SearchBuilder {
-        SearchBuilder {
-            index: self,
-            query,
-            offset: 0,
-            limit: 20,
-            attributes_to_crop: None,
-            attributes_to_retrieve: None,
-            attributes_to_search_in: None,
-            attributes_to_highlight: None,
-            filters: None,
-            timeout: Duration::from_millis(30),
-            matches: false,
-        }
-    }
-}
-
-pub struct SearchBuilder<'a> {
-    index: &'a Index,
-    query: String,
-    offset: usize,
-    limit: usize,
-    attributes_to_crop: Option<HashMap<String, usize>>,
-    attributes_to_retrieve: Option<HashSet<String>>,
-    attributes_to_search_in: Option<HashSet<String>>,
-    attributes_to_highlight: Option<HashSet<String>>,
-    filters: Option<String>,
-    timeout: Duration,
-    matches: bool,
-}
-
-impl<'a> SearchBuilder<'a> {
-    pub fn offset(&mut self, value: usize) -> &SearchBuilder {
-        self.offset = value;
-        self
-    }
-
-    pub fn limit(&mut self, value: usize) -> &SearchBuilder {
-        self.limit = value;
-        self
-    }
-
-    pub fn attributes_to_crop(&mut self, value: HashMap<String, usize>) -> &SearchBuilder {
-        self.attributes_to_crop = Some(value);
-        self
-    }
-
-    pub fn attributes_to_retrieve(&mut self, value: HashSet<String>) -> &SearchBuilder {
-        self.attributes_to_retrieve = Some(value);
-        self
-    }
-
-    pub fn add_retrievable_field(&mut self, value: String) -> &SearchBuilder {
-        let attributes_to_retrieve = self.attributes_to_retrieve.get_or_insert(HashSet::new());
-        attributes_to_retrieve.insert(value);
-        self
-    }
-
-    pub fn attributes_to_search_in(&mut self, value: HashSet<String>) -> &SearchBuilder {
-        self.attributes_to_search_in = Some(value);
-        self
-    }
-
-    pub fn add_attribute_to_search_in(&mut self, value: String) -> &SearchBuilder {
-        let attributes_to_search_in = self.attributes_to_search_in.get_or_insert(HashSet::new());
-        attributes_to_search_in.insert(value);
-        self
-    }
-
-    pub fn attributes_to_highlight(&mut self, value: HashSet<String>) -> &SearchBuilder {
-        self.attributes_to_highlight = Some(value);
-        self
-    }
-
-    pub fn filters(&mut self, value: String) -> &SearchBuilder {
-        self.filters = Some(value);
-        self
-    }
-
-    pub fn timeout(&mut self, value: Duration) -> &SearchBuilder {
-        self.timeout = value;
-        self
-    }
-
-    pub fn get_matches(&mut self) -> &SearchBuilder {
-        self.matches = true;
-        self
-    }
-
-    pub fn search(&self, reader: &heed::RoTxn) -> Result<SearchResult, Error> {
-        let schema = self.index.main.schema(reader);
-        let schema = schema.map_err(|e| Error::Internal(e.to_string()))?;
-        let schema = match schema {
-            Some(schema) => schema,
-            None => return Err(Error::Internal(String::from("missing schema"))),
-        };
-
-        let ranked_map = self.index.main.ranked_map(reader);
-        let ranked_map = ranked_map.map_err(|e| Error::Internal(e.to_string()))?;
-        let ranked_map = ranked_map.unwrap_or_default();
-
-        let start = Instant::now();
-
-        // Change criteria
-        let mut query_builder = match self.get_criteria(reader, &ranked_map, &schema)? {
-            Some(criteria) => self.index.query_builder_with_criteria(criteria),
-            None => self.index.query_builder(),
-        };
-
-        // Filter searchable fields
-        if let Some(fields) = &self.attributes_to_search_in {
-            for attribute in fields.iter().filter_map(|f| schema.attribute(f)) {
-                query_builder.add_searchable_attribute(attribute.0);
-            }
-        }
-
-        if let Some(filters) = &self.filters {
-            let mut split = filters.split(':');
-            match (split.next(), split.next()) {
-                (Some(_), None) | (Some(_), Some("")) => return Err(Error::MissingFilterValue),
-                (Some(attr), Some(value)) => {
-                    let ref_reader = reader;
-                    let ref_index = &self.index;
-                    let value = value.trim().to_lowercase();
-
-                    let attr = match schema.attribute(attr) {
-                        Some(attr) => attr,
-                        None => return Err(Error::UnknownFilteredAttribute),
-                    };
-
-                    query_builder.with_filter(move |id| {
-                        let attr = attr;
-                        let index = ref_index;
-                        let reader = ref_reader;
-
-                        match index.document_attribute::<Value>(reader, id, attr) {
-                            Ok(Some(Value::String(s))) => s.to_lowercase() == value,
-                            Ok(Some(Value::Bool(b))) => {
-                                (value == "true" && b) || (value == "false" && !b)
-                            }
-                            Ok(Some(Value::Array(a))) => {
-                                a.into_iter().any(|s| s.as_str() == Some(&value))
-                            }
-                            _ => false,
-                        }
-                    });
-                }
-                (_, _) => (),
-            }
-        }
-
-        query_builder.with_fetch_timeout(self.timeout);
-
-        let docs =
-            query_builder.query(reader, &self.query, self.offset..(self.offset + self.limit));
-
-        let mut hits = Vec::with_capacity(self.limit);
-        for doc in docs.map_err(|e| Error::SearchDocuments(e.to_string()))? {
-            // retrieve the content of document in kv store
-            let mut fields: Option<HashSet<&str>> = None;
-            if let Some(attributes_to_retrieve) = &self.attributes_to_retrieve {
-                let mut set = HashSet::new();
-                for field in attributes_to_retrieve {
-                    set.insert(field.as_str());
-                }
-                fields = Some(set);
-            }
-
-            let document: IndexMap<String, Value> = self
-                .index
-                .document(reader, fields.as_ref(), doc.id)
-                .map_err(|e| Error::RetrieveDocument(doc.id.0, e.to_string()))?
-                .ok_or(Error::DocumentNotFound(doc.id.0))?;
-
-            let mut formatted = document.clone();
-            let mut matches = doc.highlights.clone();
-
-            // Crops fields if needed
-            if let Some(fields) = &self.attributes_to_crop {
-                crop_document(&mut formatted, &mut matches, &schema, fields);
-            }
-
-            // Transform to readable matches
-            let matches = calculate_matches(matches, self.attributes_to_retrieve.clone(), &schema);
-
-            if !self.matches {
-                if let Some(attributes_to_highlight) = &self.attributes_to_highlight {
-                    formatted = calculate_highlights(&formatted, &matches, attributes_to_highlight);
-                }
-            }
-
-            let matches_info = if self.matches { Some(matches) } else { None };
-
-            let hit = SearchHit {
-                document,
-                formatted,
-                matches_info,
-            };
-
-            hits.push(hit);
-        }
-
-        let time_ms = start.elapsed().as_millis() as usize;
-
-        let results = SearchResult {
-            hits,
-            offset: self.offset,
-            limit: self.limit,
-            processing_time_ms: time_ms,
-            query: self.query.to_string(),
-        };
-
-        Ok(results)
-    }
-
-    pub fn get_criteria(
-        &self,
-        reader: &heed::RoTxn,
-        ranked_map: &'a RankedMap,
-        schema: &Schema,
-    ) -> Result<Option<Criteria<'a>>, Error> {
-        let current_settings = match self.index.main.customs(reader).unwrap() {
-            Some(bytes) => bincode::deserialize(bytes).unwrap(),
-            None => SettingBody::default(),
-        };
-
-        let ranking_rules = &current_settings.ranking_rules;
-        let ranking_order = &current_settings.ranking_order;
-
-        if let Some(ranking_rules) = ranking_rules {
-            let mut builder = CriteriaBuilder::with_capacity(7 + ranking_rules.len());
-            if let Some(ranking_rules_order) = ranking_order {
-                for rule in ranking_rules_order {
-                    match rule.as_str() {
-                        "_sum_of_typos" => builder.push(SumOfTypos),
-                        "_number_of_words" => builder.push(NumberOfWords),
-                        "_word_proximity" => builder.push(WordsProximity),
-                        "_sum_of_words_attribute" => builder.push(SumOfWordsAttribute),
-                        "_sum_of_words_position" => builder.push(SumOfWordsPosition),
-                        "_exact" => builder.push(Exact),
-                        _ => {
-                            let order = match ranking_rules.get(rule.as_str()) {
-                                Some(o) => o,
-                                None => continue,
-                            };
-
-                            let custom_ranking = match order {
-                                RankingOrdering::Asc => {
-                                    SortByAttr::lower_is_better(&ranked_map, &schema, &rule)
-                                        .unwrap()
-                                }
-                                RankingOrdering::Dsc => {
-                                    SortByAttr::higher_is_better(&ranked_map, &schema, &rule)
-                                        .unwrap()
-                                }
-                            };
-
-                            builder.push(custom_ranking);
-                        }
-                    }
-                }
-                builder.push(DocumentId);
-                return Ok(Some(builder.build()));
-            } else {
-                builder.push(SumOfTypos);
-                builder.push(NumberOfWords);
-                builder.push(WordsProximity);
-                builder.push(SumOfWordsAttribute);
-                builder.push(SumOfWordsPosition);
-                builder.push(Exact);
-                for (rule, order) in ranking_rules.iter() {
-                    let custom_ranking = match order {
-                        RankingOrdering::Asc => {
-                            SortByAttr::lower_is_better(&ranked_map, &schema, &rule).unwrap()
-                        }
-                        RankingOrdering::Dsc => {
-                            SortByAttr::higher_is_better(&ranked_map, &schema, &rule).unwrap()
-                        }
-                    };
-                    builder.push(custom_ranking);
-                }
-                builder.push(DocumentId);
-                return Ok(Some(builder.build()));
-            }
-        }
-
-        Ok(None)
-    }
-}
-
-#[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Serialize, Deserialize)]
-pub struct MatchPosition {
-    pub start: usize,
-    pub length: usize,
-}
-
-impl Ord for MatchPosition {
-    fn cmp(&self, other: &Self) -> Ordering {
-        match self.start.cmp(&other.start) {
-            Ordering::Equal => self.length.cmp(&other.length),
-            _ => self.start.cmp(&other.start),
-        }
-    }
-}
-
-pub type HighlightInfos = HashMap<String, Value>;
-pub type MatchesInfos = HashMap<String, Vec<MatchPosition>>;
-// pub type RankingInfos = HashMap<String, u64>;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct SearchHit {
-    #[serde(flatten)]
-    pub document: IndexMap<String, Value>,
-    #[serde(rename = "_formatted", skip_serializing_if = "IndexMap::is_empty")]
-    pub formatted: IndexMap<String, Value>,
-    #[serde(rename = "_matchesInfo", skip_serializing_if = "Option::is_none")]
-    pub matches_info: Option<MatchesInfos>,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct SearchResult {
-    pub hits: Vec<SearchHit>,
-    pub offset: usize,
-    pub limit: usize,
-    pub processing_time_ms: usize,
-    pub query: String,
-    // pub parsed_query: String,
-    // pub params: Option<String>,
-}
-
-fn crop_text(
-    text: &str,
-    matches: impl IntoIterator<Item = Highlight>,
-    context: usize,
-) -> (String, Vec<Highlight>) {
-    let mut matches = matches.into_iter().peekable();
-
-    let char_index = matches.peek().map(|m| m.char_index as usize).unwrap_or(0);
-    let start = char_index.saturating_sub(context);
-    let text = text.chars().skip(start).take(context * 2).collect();
-
-    let matches = matches
-        .take_while(|m| (m.char_index as usize) + (m.char_length as usize) <= start + (context * 2))
-        .map(|match_| Highlight {
-            char_index: match_.char_index - start as u16,
-            ..match_
-        })
-        .collect();
-
-    (text, matches)
-}
-
-fn crop_document(
-    document: &mut IndexMap<String, Value>,
-    matches: &mut Vec<Highlight>,
-    schema: &Schema,
-    fields: &HashMap<String, usize>,
-) {
-    matches.sort_unstable_by_key(|m| (m.char_index, m.char_length));
-
-    for (field, length) in fields {
-        let attribute = match schema.attribute(field) {
-            Some(attribute) => attribute,
-            None => continue,
-        };
-
-        let selected_matches = matches
-            .iter()
-            .filter(|m| SchemaAttr::new(m.attribute) == attribute)
-            .cloned();
-
-        if let Some(Value::String(ref mut original_text)) = document.get_mut(field) {
-            let (cropped_text, cropped_matches) =
-                crop_text(original_text, selected_matches, *length);
-
-            *original_text = cropped_text;
-
-            matches.retain(|m| SchemaAttr::new(m.attribute) != attribute);
-            matches.extend_from_slice(&cropped_matches);
-        }
-    }
-}
-
-fn calculate_matches(
-    matches: Vec<Highlight>,
-    attributes_to_retrieve: Option<HashSet<String>>,
-    schema: &Schema,
-) -> MatchesInfos {
-    let mut matches_result: HashMap<String, Vec<MatchPosition>> = HashMap::new();
-    for m in matches.iter() {
-        let attribute = schema
-            .attribute_name(SchemaAttr::new(m.attribute))
-            .to_string();
-        if let Some(attributes_to_retrieve) = attributes_to_retrieve.clone() {
-            if !attributes_to_retrieve.contains(attribute.as_str()) {
-                continue;
-            }
-        };
-        if let Some(pos) = matches_result.get_mut(&attribute) {
-            pos.push(MatchPosition {
-                start: m.char_index as usize,
-                length: m.char_length as usize,
-            });
-        } else {
-            let mut positions = Vec::new();
-            positions.push(MatchPosition {
-                start: m.char_index as usize,
-                length: m.char_length as usize,
-            });
-            matches_result.insert(attribute, positions);
-        }
-    }
-    for (_, val) in matches_result.iter_mut() {
-        val.sort_unstable();
-        val.dedup();
-    }
-    matches_result
-}
-
-fn calculate_highlights(
-    document: &IndexMap<String, Value>,
-    matches: &MatchesInfos,
-    attributes_to_highlight: &HashSet<String>,
-) -> IndexMap<String, Value> {
-    let mut highlight_result = IndexMap::new();
-
-    for (attribute, matches) in matches.iter() {
-        if attributes_to_highlight.contains(attribute) {
-            if let Some(Value::String(value)) = document.get(attribute) {
-                let value: Vec<_> = value.chars().collect();
-                let mut highlighted_value = String::new();
-                let mut index = 0;
-                for m in matches {
-                    if m.start >= index {
-                        let before = value.get(index..m.start);
-                        let highlighted = value.get(m.start..(m.start + m.length));
-                        if let (Some(before), Some(highlighted)) = (before, highlighted) {
-                            highlighted_value.extend(before);
-                            highlighted_value.push_str("<em>");
-                            highlighted_value.extend(highlighted);
-                            highlighted_value.push_str("</em>");
-                            index = m.start + m.length;
-                        } else {
-                            error!("value: {:?}; index: {:?}, match: {:?}", value, index, m);
-                        }
-                    }
-                }
-                highlighted_value.extend(value[index..].iter());
-                highlight_result.insert(attribute.to_string(), Value::String(highlighted_value));
-            };
-        }
-    }
-
-    highlight_result
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn calculate_highlights() {
-        let data = r#"{
-            "title": "Fondation (Isaac ASIMOV)",
-            "description": "En ce début de trentième millénaire, l'Empire n'a jamais été aussi puissant, aussi étendu à travers toute la galaxie. C'est dans sa capitale, Trantor, que l'éminent savant Hari Seldon invente la psychohistoire, une science toute nouvelle, à base de psychologie et de mathématiques, qui lui permet de prédire l'avenir... C'est-à-dire l'effondrement de l'Empire d'ici cinq siècles et au-delà, trente mille années de chaos et de ténèbres. Pour empêcher cette catastrophe et sauver la civilisation, Seldon crée la Fondation."
-        }"#;
-
-        let document: IndexMap<String, Value> = serde_json::from_str(data).unwrap();
-        let mut attributes_to_highlight = HashSet::new();
-        attributes_to_highlight.insert("title".to_string());
-        attributes_to_highlight.insert("description".to_string());
-
-        let mut matches = HashMap::new();
-
-        let mut m = Vec::new();
-        m.push(MatchPosition {
-            start: 0,
-            length: 9,
-        });
-        matches.insert("title".to_string(), m);
-
-        let mut m = Vec::new();
-        m.push(MatchPosition {
-            start: 510,
-            length: 9,
-        });
-        matches.insert("description".to_string(), m);
-        let result = super::calculate_highlights(&document, &matches, &attributes_to_highlight);
-
-        let mut result_expected = IndexMap::new();
-        result_expected.insert(
-            "title".to_string(),
-            Value::String("<em>Fondation</em> (Isaac ASIMOV)".to_string()),
-        );
-        result_expected.insert("description".to_string(), Value::String("En ce début de trentième millénaire, l'Empire n'a jamais été aussi puissant, aussi étendu à travers toute la galaxie. C'est dans sa capitale, Trantor, que l'éminent savant Hari Seldon invente la psychohistoire, une science toute nouvelle, à base de psychologie et de mathématiques, qui lui permet de prédire l'avenir... C'est-à-dire l'effondrement de l'Empire d'ici cinq siècles et au-delà, trente mille années de chaos et de ténèbres. Pour empêcher cette catastrophe et sauver la civilisation, Seldon crée la <em>Fondation</em>.".to_string()));
-
-        assert_eq!(result, result_expected);
-    }
-}

meilidb-http/src/helpers/mod.rs

@@ -1,2 +0,0 @@
-pub mod meilidb;
-pub mod tide;

meilidb-http/src/helpers/tide.rs

@@ -1,118 +0,0 @@
-use crate::error::{ResponseError, SResult};
-use crate::models::token::*;
-use crate::Data;
-use chrono::Utc;
-use heed::types::{SerdeBincode, Str};
-use meilidb_core::Index;
-use serde_json::Value;
-use tide::Context;
-
-pub trait ContextExt {
-    fn is_allowed(&self, acl: ACL) -> SResult<()>;
-    fn header(&self, name: &str) -> Result<String, ResponseError>;
-    fn url_param(&self, name: &str) -> Result<String, ResponseError>;
-    fn index(&self) -> Result<Index, ResponseError>;
-    fn identifier(&self) -> Result<String, ResponseError>;
-}
-
-impl ContextExt for Context<Data> {
-    fn is_allowed(&self, acl: ACL) -> SResult<()> {
-        let admin_token = match &self.state().admin_token {
-            Some(admin_token) => admin_token,
-            None => return Ok(()),
-        };
-
-        let user_api_key = self.header("X-Meili-API-Key")?;
-        if user_api_key == *admin_token {
-            return Ok(());
-        }
-        let request_index: Option<String> = None; //self.param::<String>("index").ok();
-
-        let db = &self.state().db;
-        let env = &db.env;
-        let reader = env.read_txn().map_err(ResponseError::internal)?;
-
-        let token_key = format!("{}{}", TOKEN_PREFIX_KEY, user_api_key);
-
-        let token_config = db
-            .common_store()
-            .get::<Str, SerdeBincode<Token>>(&reader, &token_key)
-            .map_err(ResponseError::internal)?
-            .ok_or(ResponseError::not_found(format!(
-                "token key: {}",
-                token_key
-            )))?;
-
-        if token_config.revoked {
-            return Err(ResponseError::invalid_token("token revoked"));
-        }
-
-        if let Some(index) = request_index {
-            if !token_config
-                .indexes
-                .iter()
-                .any(|r| match_wildcard(&r, &index))
-            {
-                return Err(ResponseError::invalid_token(
-                    "token is not allowed to access to this index",
-                ));
-            }
-        }
-
-        if token_config.expires_at < Utc::now() {
-            return Err(ResponseError::invalid_token("token expired"));
-        }
-
-        if token_config.acl.contains(&ACL::All) {
-            return Ok(());
-        }
-
-        if !token_config.acl.contains(&acl) {
-            return Err(ResponseError::invalid_token("token do not have this ACL"));
-        }
-
-        Ok(())
-    }
-
-    fn header(&self, name: &str) -> Result<String, ResponseError> {
-        let header = self
-            .headers()
-            .get(name)
-            .ok_or(ResponseError::missing_header(name))?
-            .to_str()
-            .map_err(|_| ResponseError::missing_header("X-Meili-API-Key"))?
-            .to_string();
-        Ok(header)
-    }
-
-    fn url_param(&self, name: &str) -> Result<String, ResponseError> {
-        let param = self
-            .param::<String>(name)
-            .map_err(|e| ResponseError::bad_parameter(name, e))?;
-        Ok(param)
-    }
-
-    fn index(&self) -> Result<Index, ResponseError> {
-        let index_name = self.url_param("index")?;
-        let index = self
-            .state()
-            .db
-            .open_index(&index_name)
-            .ok_or(ResponseError::index_not_found(index_name))?;
-        Ok(index)
-    }
-
-    fn identifier(&self) -> Result<String, ResponseError> {
-        let name = self
-            .param::<Value>("identifier")
-            .as_ref()
-            .map(meilidb_core::serde::value_to_string)
-            .map_err(|e| ResponseError::bad_parameter("identifier", e))?
-            .ok_or(ResponseError::bad_parameter(
-                "identifier",
-                "missing parameter",
-            ))?;
-
-        Ok(name)
-    }
-}