Fix test again

.github/scripts/check-release.sh

@@ -1,41 +1,24 @@
-#!/usr/bin/env bash
-set -eu -o pipefail
+#!/bin/bash
 
-check_tag() {
-    local expected=$1
-    local actual=$2
-    local filename=$3
-
-    if [[ $actual != $expected ]]; then
-        echo >&2 "Error: the current tag does not match the version in $filename: found $actual, expected $expected"
-        return 1
-    fi
+# check_tag $current_tag $file_tag $file_name
+function check_tag {
+  if [[ "$1" != "$2" ]]; then
+      echo "Error: the current tag does not match the version in Cargo.toml: found $2 - expected $1"
+      ret=1
+  fi
 }
 
-read_version() {
-    grep '^version = ' | cut -d \" -f 2
-}
-
-if [[ -z "${GITHUB_REF:-}" ]]; then
-    echo >&2 "Error: GITHUB_REF is not set"
-    exit 1
-fi
-
-if [[ ! "$GITHUB_REF" =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+(-[a-z0-9]+)?$ ]]; then
-    echo >&2 "Error: GITHUB_REF is not a valid tag: $GITHUB_REF"
-    exit 1
-fi
-
-current_tag=${GITHUB_REF#refs/tags/v}
 ret=0
+current_tag=${GITHUB_REF#'refs/tags/v'}
 
-toml_tag="$(cat Cargo.toml | read_version)"
-check_tag "$current_tag" "$toml_tag" Cargo.toml || ret=1
+file_tag="$(grep '^version = ' Cargo.toml | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')"
+check_tag $current_tag $file_tag
 
-lock_tag=$(grep -A 1 '^name = "meilisearch-auth"' Cargo.lock | read_version)
-check_tag "$current_tag" "$lock_tag" Cargo.lock || ret=1
+lock_file='Cargo.lock'
+lock_tag=$(grep -A 1 'name = "meilisearch-auth"' $lock_file | grep version | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')
+check_tag $current_tag $lock_tag $lock_file
 
-if (( ret == 0 )); then
-    echo 'OK'
+if [[ "$ret" -eq 0 ]] ; then
+  echo 'OK'
 fi
 exit $ret

.github/workflows/fuzzer-indexing.yml

@@ -1,24 +0,0 @@
-name: Run the indexing fuzzer
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  fuzz:
-    name: Setup the action
-    runs-on: ubuntu-latest
-    timeout-minutes: 4320 # 72h
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      # Run benchmarks
-      - name: Run the fuzzer
-        run: |
-          cargo run --release --bin fuzz-indexing

.github/workflows/sdks-tests.yml

@@ -16,28 +16,13 @@ env:
   MEILI_NO_ANALYTICS: 'true'
 
 jobs:
-  define-docker-image:
-    runs-on: ubuntu-latest
-    outputs:
-      docker-image: ${{ steps.define-image.outputs.docker-image }}
-    steps:
-      - uses: actions/checkout@v3
-      - name: Define the Docker image we need to use
-        id: define-image
-        run: |
-          event=${{ github.event_name }}
-          echo "docker-image=nightly" >> $GITHUB_OUTPUT
-          if [[ $event == 'workflow_dispatch' ]]; then
-            echo "docker-image=${{ github.event.inputs.docker_image }}" >> $GITHUB_OUTPUT
-          fi
 
   meilisearch-js-tests:
-    needs: define-docker-image
     name: JS SDK tests
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ github.event.inputs.docker_image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
@@ -67,12 +52,11 @@ jobs:
         run: yarn test:env:browser
 
   instant-meilisearch-tests:
-    needs: define-docker-image
     name: instant-meilisearch tests
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ github.event.inputs.docker_image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
@@ -94,12 +78,11 @@ jobs:
         run: yarn build
 
   meilisearch-php-tests:
-    needs: define-docker-image
     name: PHP SDK tests
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ github.event.inputs.docker_image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
@@ -125,12 +108,11 @@ jobs:
           composer remove --dev guzzlehttp/guzzle http-interop/http-factory-guzzle
 
   meilisearch-python-tests:
-    needs: define-docker-image
     name: Python SDK tests
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ github.event.inputs.docker_image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
@@ -150,12 +132,11 @@ jobs:
         run: pipenv run pytest
 
   meilisearch-go-tests:
-    needs: define-docker-image
     name: Go SDK tests
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ github.event.inputs.docker_image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
@@ -180,12 +161,11 @@ jobs:
         run: go test -v ./...
 
   meilisearch-ruby-tests:
-    needs: define-docker-image
     name: Ruby SDK tests
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ github.event.inputs.docker_image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
@@ -205,12 +185,11 @@ jobs:
         run: bundle exec rspec
 
   meilisearch-rust-tests:
-    needs: define-docker-image
     name: Rust SDK tests
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ github.event.inputs.docker_image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}

.github/workflows/test-suite.yml

@@ -30,20 +30,20 @@ jobs:
         run: |
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
-      - name: Setup test with Rust stable
+      - name: Run test with Rust stable
         if: github.event_name != 'schedule'
         uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
           override: true
-      - name: Setup test with Rust nightly
+      - name: Run test with Rust nightly
         if: github.event_name == 'schedule'
         uses: actions-rs/toolchain@v1
         with:
           toolchain: nightly
           override: true
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.0
+        uses: Swatinem/rust-cache@v2.4.0
       - name: Run cargo check without any default features
         uses: actions-rs/cargo@v1
         with:
@@ -65,7 +65,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.0
+        uses: Swatinem/rust-cache@v2.4.0
       - name: Run cargo check without any default features
         uses: actions-rs/cargo@v1
         with:
@@ -117,17 +117,17 @@ jobs:
         run: |
           apt-get update
           apt-get install --assume-yes build-essential curl
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+      - uses: actions-rs/toolchain@v1 
+        with: 
+          toolchain: stable 
+          override: true 
       - name: Run cargo tree without default features and check lindera is not present
         run: |
           cargo tree -f '{p} {f}' -e normal --no-default-features | grep lindera -vqz
       - name: Run cargo tree with default features and check lindera is pressent
         run: |
           cargo tree -f '{p} {f}' -e normal | grep lindera -qz
-
+                
   # We run tests in debug also, to make sure that the debug_assertions are hit
   test-debug:
     name: Run tests in debug
@@ -146,7 +146,7 @@ jobs:
           toolchain: stable
           override: true
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.0
+        uses: Swatinem/rust-cache@v2.4.0
       - name: Run tests in debug
         uses: actions-rs/cargo@v1
         with:
@@ -165,7 +165,7 @@ jobs:
           override: true
           components: clippy
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.0
+        uses: Swatinem/rust-cache@v2.4.0
       - name: Run cargo clippy
         uses: actions-rs/cargo@v1
         with:
@@ -184,7 +184,7 @@ jobs:
           override: true
           components: rustfmt
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.0
+        uses: Swatinem/rust-cache@v2.4.0
       - name: Run cargo fmt
         # Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file.
         # Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate

Cargo.toml

@@ -13,12 +13,11 @@ members = [
     "filter-parser",
     "flatten-serde-json",
     "json-depth-checker",
-    "benchmarks",
-    "fuzzers",
+    "benchmarks"
 ]
 
 [workspace.package]
-version = "1.3.0"
+version = "1.2.0"
 authors = ["Quentin de Quelen <quentin@dequelen.me>", "Clément Renault <clement@meilisearch.com>"]
 description = "Meilisearch HTTP server"
 homepage = "https://meilisearch.com"

PROFILING.md

@@ -1,19 +0,0 @@
-# Profiling Meilisearch
-
-Search engine technologies are complex pieces of software that require thorough profiling tools. We chose to use [Puffin](https://github.com/EmbarkStudios/puffin), which the Rust gaming industry uses extensively. You can export and import the profiling reports using the top bar's _File_ menu options.
-
-![An example profiling with Puffin viewer](assets/profiling-example.png)
-
-## Profiling the Indexing Process
-
-When you enable the `profile-with-puffin` feature of Meilisearch, a Puffin HTTP server will run on Meilisearch and listen on the default _0.0.0.0:8585_ address. This server will record a "frame" whenever it executes the `IndexScheduler::tick` method.
-
-Once your Meilisearch is running and awaits new indexation operations, you must [install and run the `puffin_viewer` tool](https://github.com/EmbarkStudios/puffin/tree/main/puffin_viewer) to see the profiling results. I advise you to run the viewer with the `RUST_LOG=puffin_http::client=debug` environment variable to see the client trying to connect to your server.
-
-Another piece of advice on the Puffin viewer UI interface is to consider the _Merge children with same ID_ option. It can hide the exact actual timings at which events were sent. Please turn it off when you see strange gaps on the Flamegraph. It can help.
-
-## Profiling the Search Process
-
-We still need to take the time to profile the search side of the engine with Puffin. It would require time to profile the filtering phase, query parsing, creation, and execution. We could even profile the Actix HTTP server.
-
-The only issue we see is the framing system. Puffin requires a global frame-based profiling phase, which collides with Meilisearch's ability to accept and answer multiple requests on different threads simultaneously.

README.md

@@ -1,20 +1,15 @@
 <p align="center">
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=logo#gh-light-mode-only" target="_blank">
-    <img src="assets/meilisearch-logo-light.svg?sanitize=true#gh-light-mode-only">
-  </a>
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=logo#gh-dark-mode-only" target="_blank">
-    <img src="assets/meilisearch-logo-dark.svg?sanitize=true#gh-dark-mode-only">
-  </a>
+  <img src="assets/meilisearch-logo-light.svg?sanitize=true#gh-light-mode-only">
+  <img src="assets/meilisearch-logo-dark.svg?sanitize=true#gh-dark-mode-only">
 </p>
 
 <h4 align="center">
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Website</a> |
+  <a href="https://www.meilisearch.com">Website</a> |
   <a href="https://roadmap.meilisearch.com/tabs/1-under-consideration">Roadmap</a> |
-  <a href="https://www.meilisearch.com/pricing?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Meilisearch Cloud</a> |
-  <a href="https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Blog</a> |
-  <a href="https://www.meilisearch.com/docs?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Documentation</a> |
-  <a href="https://www.meilisearch.com/docs/faq?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">FAQ</a> |
-  <a href="https://discord.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Discord</a>
+  <a href="https://blog.meilisearch.com">Blog</a> |
+  <a href="https://www.meilisearch.com/docs">Documentation</a> |
+  <a href="https://www.meilisearch.com/docs/faq">FAQ</a> |
+  <a href="https://discord.meilisearch.com">Discord</a>
 </h4>
 
 <p align="center">
@@ -28,72 +23,72 @@
 Meilisearch helps you shape a delightful search experience in a snap, offering features that work out-of-the-box to speed up your workflow.
 
 <p align="center" name="demo">
-  <a href="https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-gif#gh-light-mode-only" target="_blank">
+  <a href="https://where2watch.meilisearch.com/#gh-light-mode-only" target="_blank">
     <img src="assets/demo-light.gif#gh-light-mode-only" alt="A bright colored application for finding movies screening near the user">
   </a>
-  <a href="https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-gif#gh-dark-mode-only" target="_blank">
+  <a href="https://where2watch.meilisearch.com/#gh-dark-mode-only" target="_blank">
     <img src="assets/demo-dark.gif#gh-dark-mode-only" alt="A dark colored application for finding movies screening near the user">
   </a>
 </p>
 
-🔥 [**Try it!**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-link) 🔥
+🔥 [**Try it!**](https://where2watch.meilisearch.com/) 🔥
 
 ## ✨ Features
 
 - **Search-as-you-type:** find search results in less than 50 milliseconds
-- **[Typo tolerance](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features#typo-tolerance):** get relevant matches even when queries contain typos and misspellings
-- **[Filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features) and [faceted search](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** enhance your user's search experience with custom filters and build a faceted search interface in a few lines of code
-- **[Sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** sort results based on price, date, or pretty much anything else your users need
-- **[Synonym support](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features#synonyms):** configure synonyms to include more relevant content in your search results
-- **[Geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** filter and sort documents based on geographic data
-- **[Extensive language support](https://www.meilisearch.com/docs/learn/what_is_meilisearch/language?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** search datasets in any language, with optimized support for Chinese, Japanese, Hebrew, and languages using the Latin alphabet
-- **[Security management](https://www.meilisearch.com/docs/learn/security/master_api_keys?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** control which users can access what data with API keys that allow fine-grained permissions handling
-- **[Multi-Tenancy](https://www.meilisearch.com/docs/learn/security/tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** personalize search results for any number of application tenants
+- **[Typo tolerance](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy#typo-tolerance):** get relevant matches even when queries contain typos and misspellings
+- **[Filtering](https://www.meilisearch.com/docs/learn/advanced/filtering) and [faceted search](https://www.meilisearch.com/docs/learn/advanced/faceted_search):** enhance your user's search experience with custom filters and build a faceted search interface in a few lines of code
+- **[Sorting](https://www.meilisearch.com/docs/learn/advanced/sorting):** sort results based on price, date, or pretty much anything else your users need
+- **[Synonym support](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy#synonyms):** configure synonyms to include more relevant content in your search results
+- **[Geosearch](https://www.meilisearch.com/docs/learn/advanced/geosearch):** filter and sort documents based on geographic data
+- **[Extensive language support](https://www.meilisearch.com/docs/learn/what_is_meilisearch/language):** search datasets in any language, with optimized support for Chinese, Japanese, Hebrew, and languages using the Latin alphabet
+- **[Security management](https://www.meilisearch.com/docs/learn/security/master_api_keys):** control which users can access what data with API keys that allow fine-grained permissions handling
+- **[Multi-Tenancy](https://www.meilisearch.com/docs/learn/security/tenant_tokens):** personalize search results for any number of application tenants
 - **Highly Customizable:** customize Meilisearch to your specific needs or use our out-of-the-box and hassle-free presets
-- **[RESTful API](https://www.meilisearch.com/docs/reference/api/overview?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** integrate Meilisearch in your technical stack with our plugins and SDKs
+- **[RESTful API](https://www.meilisearch.com/docs/reference/api/overview):** integrate Meilisearch in your technical stack with our plugins and SDKs
 - **Easy to install, deploy, and maintain**
 
 ## 📖 Documentation
 
-You can consult Meilisearch's documentation at [https://www.meilisearch.com/docs](https://www.meilisearch.com/docs/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=docs).
+You can consult Meilisearch's documentation at [https://www.meilisearch.com/docs](https://www.meilisearch.com/docs/).
 
 ## 🚀 Getting started
 
-For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [Quick Start](https://www.meilisearch.com/docs/learn/getting_started/quick_start?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) guide.
+For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [Quick Start](https://www.meilisearch.com/docs/learn/getting_started/quick_start) guide.
 
-You may also want to check out [Meilisearch 101](https://www.meilisearch.com/docs/learn/getting_started/filtering_and_sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) for an introduction to some of Meilisearch's most popular features.
+You may also want to check out [Meilisearch 101](https://www.meilisearch.com/docs/learn/getting_started/filtering_and_sorting) for an introduction to some of Meilisearch's most popular features.
 
-## ⚡ Supercharge your Meilisearch experience
+## ☁️ Meilisearch cloud
 
-Say goodbye to server deployment and manual updates with [Meilisearch Cloud](https://www.meilisearch.com/pricing?utm_campaign=oss&utm_source=engine&utm_medium=meilisearch). Get started with a 14-day free trial! No credit card required.
+Let us manage your infrastructure so you can focus on integrating a great search experience. Try [Meilisearch Cloud](https://meilisearch.com/pricing) today.
 
 ## 🧰 SDKs & integration tools
 
 Install one of our SDKs in your project for seamless integration between Meilisearch and your favorite language or framework!
 
-Take a look at the complete [Meilisearch integration list](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=sdks-link).
+Take a look at the complete [Meilisearch integration list](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks).
 
-[![Logos belonging to different languages and frameworks supported by Meilisearch, including React, Ruby on Rails, Go, Rust, and PHP](assets/integrations.png)](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=sdks-logos)
+[![Logos belonging to different languages and frameworks supported by Meilisearch, including React, Ruby on Rails, Go, Rust, and PHP](assets/integrations.png)](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks)
 
 ## ⚙️ Advanced usage
 
-Experienced users will want to keep our [API Reference](https://www.meilisearch.com/docs/reference/api/overview?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced) close at hand.
+Experienced users will want to keep our [API Reference](https://www.meilisearch.com/docs/reference/api/overview) close at hand.
 
-We also offer a wide range of dedicated guides to all Meilisearch features, such as [filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [API keys](https://www.meilisearch.com/docs/learn/security/master_api_keys?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), and [tenant tokens](https://www.meilisearch.com/docs/learn/security/tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced).
+We also offer a wide range of dedicated guides to all Meilisearch features, such as [filtering](https://www.meilisearch.com/docs/learn/advanced/filtering), [sorting](https://www.meilisearch.com/docs/learn/advanced/sorting), [geosearch](https://www.meilisearch.com/docs/learn/advanced/geosearch), [API keys](https://www.meilisearch.com/docs/learn/security/master_api_keys), and [tenant tokens](https://www.meilisearch.com/docs/learn/security/tenant_tokens).
 
-Finally, for more in-depth information, refer to our articles explaining fundamental Meilisearch concepts such as [documents](https://www.meilisearch.com/docs/learn/core_concepts/documents?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced) and [indexes](https://www.meilisearch.com/docs/learn/core_concepts/indexes?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced).
+Finally, for more in-depth information, refer to our articles explaining fundamental Meilisearch concepts such as [documents](https://www.meilisearch.com/docs/learn/core_concepts/documents) and [indexes](https://www.meilisearch.com/docs/learn/core_concepts/indexes).
 
 ## 📊 Telemetry
 
-Meilisearch collects **anonymized** data from users to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) whenever you want.
+Meilisearch collects **anonymized** data from users to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry#how-to-disable-data-collection) whenever you want.
 
 To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Don't forget to include your `Instance UID` in the message, as this helps us quickly find and delete your data.
 
-If you want to know more about the kind of data we collect and what we use it for, check the [telemetry section](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) of our documentation.
+If you want to know more about the kind of data we collect and what we use it for, check the [telemetry section](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry) of our documentation.
 
 ## 📫 Get in touch!
 
-Meilisearch is a search engine created by [Meili](https://www.welcometothejungle.com/en/companies/meilisearch), a software development company based in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact)
+Meilisearch is a search engine created by [Meili](https://www.welcometothejungle.com/en/companies/meilisearch), a software development company based in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/)
 
 🗞 [Subscribe to our newsletter](https://meilisearch.us2.list-manage.com/subscribe?u=27870f7b71c908a8b359599fb&id=79582d828e) if you don't want to miss any updates! We promise we won't clutter your mailbox: we only send one edition every two months.
 

assets/grafana-dashboard.json

@@ -98,7 +98,7 @@
         "showThresholdMarkers": true,
         "text": {}
       },
-      "pluginVersion": "10.0.1",
+      "pluginVersion": "9.5.2",
       "targets": [
         {
           "datasource": {
@@ -158,7 +158,7 @@
         "showThresholdMarkers": true,
         "text": {}
       },
-      "pluginVersion": "10.0.1",
+      "pluginVersion": "9.5.2",
       "targets": [
         {
           "datasource": {
@@ -176,7 +176,8 @@
     },
     {
       "datasource": {
-        "type": "prometheus"
+        "type": "prometheus",
+        "uid": "c4085c47-f6d3-45dd-b761-6809055bb749"
       },
       "fieldConfig": {
         "defaults": {
@@ -220,7 +221,7 @@
         "showThresholdMarkers": true,
         "text": {}
       },
-      "pluginVersion": "10.0.1",
+      "pluginVersion": "9.5.2",
       "targets": [
         {
           "datasource": {
@@ -240,7 +241,8 @@
     },
     {
       "datasource": {
-        "type": "prometheus"
+        "type": "prometheus",
+        "uid": "c4085c47-f6d3-45dd-b761-6809055bb749"
       },
       "fieldConfig": {
         "defaults": {
@@ -280,7 +282,7 @@
         "showThresholdMarkers": true,
         "text": {}
       },
-      "pluginVersion": "10.0.1",
+      "pluginVersion": "9.5.2",
       "targets": [
         {
           "datasource": {
@@ -300,7 +302,8 @@
     },
     {
       "datasource": {
-        "type": "prometheus"
+        "type": "prometheus",
+        "uid": "c4085c47-f6d3-45dd-b761-6809055bb749"
       },
       "fieldConfig": {
         "defaults": {
@@ -340,7 +343,7 @@
         "showThresholdMarkers": true,
         "text": {}
       },
-      "pluginVersion": "10.0.1",
+      "pluginVersion": "9.5.2",
       "targets": [
         {
           "datasource": {
@@ -360,7 +363,8 @@
     },
     {
       "datasource": {
-        "type": "prometheus"
+        "type": "prometheus",
+        "uid": "c4085c47-f6d3-45dd-b761-6809055bb749"
       },
       "description": "",
       "fieldConfig": {
@@ -407,7 +411,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -455,7 +460,8 @@
         },
         {
           "datasource": {
-            "type": "prometheus"
+            "type": "prometheus",
+            "uid": "c4085c47-f6d3-45dd-b761-6809055bb749"
           },
           "editorMode": "builder",
           "expr": "meilisearch_used_db_size_bytes{job=\"meilisearch\", instance=\"$instance\"}",
@@ -553,7 +559,7 @@
           },
           "editorMode": "builder",
           "exemplar": true,
-          "expr": "rate(meilisearch_http_response_time_seconds_sum{instance=\"$instance\", job=\"meilisearch\"}[5m]) / rate(meilisearch_http_response_time_seconds_count[5m])",
+          "expr": "rate(http_response_time_seconds_sum{instance=\"$instance\", job=\"meilisearch\"}[5m]) / rate(http_response_time_seconds_count[5m])",
           "interval": "",
           "legendFormat": "{{method}} {{path}}",
           "range": true,
@@ -565,7 +571,8 @@
     },
     {
       "datasource": {
-        "type": "prometheus"
+        "type": "prometheus",
+        "uid": "c4085c47-f6d3-45dd-b761-6809055bb749"
       },
       "fieldConfig": {
         "defaults": {
@@ -608,7 +615,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -735,7 +743,7 @@
           "unit": "s"
         }
       },
-      "pluginVersion": "10.0.1",
+      "pluginVersion": "9.5.2",
       "reverseYBuckets": false,
       "targets": [
         {
@@ -744,7 +752,7 @@
           },
           "editorMode": "builder",
           "exemplar": true,
-          "expr": "sum by(le) (increase(meilisearch_http_response_time_seconds_bucket{path=\"/indexes/$Index/search\", instance=\"$instance\", job=\"meilisearch\"}[30s]))",
+          "expr": "sum by(le) (increase(http_response_time_seconds_bucket{path=\"/indexes/$Index/search\", instance=\"$instance\", job=\"meilisearch\"}[30s]))",
           "format": "heatmap",
           "interval": "",
           "legendFormat": "{{le}}",
@@ -1298,7 +1306,8 @@
           "value": "localhost:7700"
         },
         "datasource": {
-          "type": "prometheus"
+          "type": "prometheus",
+          "uid": "bb3298a4-9acf-4da1-b86a-813f29f50888"
         },
         "definition": "label_values(instance)",
         "hide": 0,
@@ -1320,11 +1329,12 @@
       {
         "current": {
           "selected": false,
-          "text": "index-word-count-10-count",
-          "value": "index-word-count-10-count"
+          "text": "mieli",
+          "value": "mieli"
         },
         "datasource": {
-          "type": "prometheus"
+          "type": "prometheus",
+          "uid": "bb3298a4-9acf-4da1-b86a-813f29f50888"
         },
         "definition": "label_values(index)",
         "hide": 0,
@@ -1361,6 +1371,6 @@
   "timezone": "",
   "title": "Meilisearch",
   "uid": "7wcZ94dnz",
-  "version": 5,
+  "version": 6,
   "weekStart": ""
-}
+}

dump/src/lib.rs

@@ -208,13 +208,12 @@ pub(crate) mod test {
     use std::str::FromStr;
 
     use big_s::S;
-    use maplit::{btreemap, btreeset};
-    use meilisearch_types::facet_values_sort::FacetValuesSort;
+    use maplit::btreeset;
     use meilisearch_types::index_uid_pattern::IndexUidPattern;
     use meilisearch_types::keys::{Action, Key};
-    use meilisearch_types::milli;
     use meilisearch_types::milli::update::Setting;
-    use meilisearch_types::settings::{Checked, FacetingSettings, Settings};
+    use meilisearch_types::milli::{self};
+    use meilisearch_types::settings::{Checked, Settings};
     use meilisearch_types::tasks::{Details, Status};
     use serde_json::{json, Map, Value};
     use time::macros::datetime;
@@ -261,18 +260,10 @@ pub(crate) mod test {
             sortable_attributes: Setting::Set(btreeset! { S("age") }),
             ranking_rules: Setting::NotSet,
             stop_words: Setting::NotSet,
-            non_separator_tokens: Setting::NotSet,
-            separator_tokens: Setting::NotSet,
-            dictionary: Setting::NotSet,
             synonyms: Setting::NotSet,
             distinct_attribute: Setting::NotSet,
             typo_tolerance: Setting::NotSet,
-            faceting: Setting::Set(FacetingSettings {
-                max_values_per_facet: Setting::Set(111),
-                sort_facet_values_by: Setting::Set(
-                    btreemap! { S("age") => FacetValuesSort::Count },
-                ),
-            }),
+            faceting: Setting::NotSet,
             pagination: Setting::NotSet,
             _kind: std::marker::PhantomData,
         };
@@ -421,8 +412,6 @@ pub(crate) mod test {
         }
         keys.flush().unwrap();
 
-        // ========== TODO: create features here
-
         // create the dump
         let mut file = tempfile::tempfile().unwrap();
         dump.persist_to(&mut file).unwrap();

dump/src/reader/compat/v5_to_v6.rs

@@ -191,10 +191,6 @@ impl CompatV5ToV6 {
             })
         })))
     }
-
-    pub fn features(&self) -> Result<Option<v6::RuntimeTogglableFeatures>> {
-        Ok(None)
-    }
 }
 
 pub enum CompatIndexV5ToV6 {
@@ -340,9 +336,6 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
                 }
             },
             stop_words: settings.stop_words.into(),
-            non_separator_tokens: v6::Setting::NotSet,
-            separator_tokens: v6::Setting::NotSet,
-            dictionary: v6::Setting::NotSet,
             synonyms: settings.synonyms.into(),
             distinct_attribute: settings.distinct_attribute.into(),
             typo_tolerance: match settings.typo_tolerance {
@@ -365,7 +358,6 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
             faceting: match settings.faceting {
                 v5::Setting::Set(faceting) => v6::Setting::Set(v6::FacetingSettings {
                     max_values_per_facet: faceting.max_values_per_facet.into(),
-                    sort_facet_values_by: v6::Setting::NotSet,
                 }),
                 v5::Setting::Reset => v6::Setting::Reset,
                 v5::Setting::NotSet => v6::Setting::NotSet,

dump/src/reader/mod.rs

@@ -107,13 +107,6 @@ impl DumpReader {
             DumpReader::Compat(compat) => compat.keys(),
         }
     }
-
-    pub fn features(&self) -> Result<Option<v6::RuntimeTogglableFeatures>> {
-        match self {
-            DumpReader::Current(current) => Ok(current.features()),
-            DumpReader::Compat(compat) => compat.features(),
-        }
-    }
 }
 
 impl From<V6Reader> for DumpReader {
@@ -196,8 +189,6 @@ pub(crate) mod test {
 
     use super::*;
 
-    // TODO: add `features` to tests
-
     #[test]
     fn import_dump_v5() {
         let dump = File::open("tests/assets/v5.dump").unwrap();

dump/src/reader/v6/mod.rs

@@ -2,7 +2,6 @@ use std::fs::{self, File};
 use std::io::{BufRead, BufReader, ErrorKind};
 use std::path::Path;
 
-use log::debug;
 pub use meilisearch_types::milli;
 use tempfile::TempDir;
 use time::OffsetDateTime;
@@ -19,7 +18,6 @@ pub type Unchecked = meilisearch_types::settings::Unchecked;
 
 pub type Task = crate::TaskDump;
 pub type Key = meilisearch_types::keys::Key;
-pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures;
 
 // ===== Other types to clarify the code of the compat module
 // everything related to the tasks
@@ -49,7 +47,6 @@ pub struct V6Reader {
     metadata: Metadata,
     tasks: BufReader<File>,
     keys: BufReader<File>,
-    features: Option<RuntimeTogglableFeatures>,
 }
 
 impl V6Reader {
@@ -61,29 +58,11 @@ impl V6Reader {
             Err(e) => return Err(e.into()),
         };
 
-        let feature_file = match fs::read(dump.path().join("experimental-features.json")) {
-            Ok(feature_file) => Some(feature_file),
-            Err(error) => match error.kind() {
-                // Allows the file to be missing, this will only result in all experimental features disabled.
-                ErrorKind::NotFound => {
-                    debug!("`experimental-features.json` not found in dump");
-                    None
-                }
-                _ => return Err(error.into()),
-            },
-        };
-        let features = if let Some(feature_file) = feature_file {
-            Some(serde_json::from_reader(&*feature_file)?)
-        } else {
-            None
-        };
-
         Ok(V6Reader {
             metadata: serde_json::from_reader(&*meta_file)?,
             instance_uid,
             tasks: BufReader::new(File::open(dump.path().join("tasks").join("queue.jsonl"))?),
             keys: BufReader::new(File::open(dump.path().join("keys.jsonl"))?),
-            features,
             dump,
         })
     }
@@ -150,10 +129,6 @@ impl V6Reader {
             (&mut self.keys).lines().map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }),
         )
     }
-
-    pub fn features(&self) -> Option<RuntimeTogglableFeatures> {
-        self.features
-    }
 }
 
 pub struct UpdateFile {

dump/src/writer.rs

@@ -4,7 +4,6 @@ use std::path::PathBuf;
 
 use flate2::write::GzEncoder;
 use flate2::Compression;
-use meilisearch_types::features::RuntimeTogglableFeatures;
 use meilisearch_types::keys::Key;
 use meilisearch_types::settings::{Checked, Settings};
 use serde_json::{Map, Value};
@@ -54,13 +53,6 @@ impl DumpWriter {
         TaskWriter::new(self.dir.path().join("tasks"))
     }
 
-    pub fn create_experimental_features(&self, features: RuntimeTogglableFeatures) -> Result<()> {
-        Ok(std::fs::write(
-            self.dir.path().join("experimental-features.json"),
-            serde_json::to_string(&features)?,
-        )?)
-    }
-
     pub fn persist_to(self, mut writer: impl Write) -> Result<()> {
         let gz_encoder = GzEncoder::new(&mut writer, Compression::default());
         let mut tar_encoder = tar::Builder::new(gz_encoder);

fuzzers/Cargo.toml

@@ -1,20 +0,0 @@
-[package]
-name = "fuzzers"
-publish = false
-
-version.workspace = true
-authors.workspace = true
-description.workspace = true
-homepage.workspace = true
-readme.workspace = true
-edition.workspace = true
-license.workspace = true
-
-[dependencies]
-arbitrary = { version = "1.3.0", features = ["derive"] }
-clap = { version = "4.3.0", features = ["derive"] }
-fastrand = "1.9.0"
-milli = { path = "../milli" }
-serde = { version = "1.0.160", features = ["derive"] }
-serde_json = { version = "1.0.95", features = ["preserve_order"] }
-tempfile = "3.5.0"

fuzzers/README.md

@@ -1,3 +0,0 @@
-# Fuzzers
-
-The purpose of this crate is to contains all the handmade "fuzzer" we may need.

fuzzers/src/bin/fuzz-indexing.rs

@@ -1,152 +0,0 @@
-use std::num::NonZeroUsize;
-use std::path::PathBuf;
-use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
-use std::time::Duration;
-
-use arbitrary::{Arbitrary, Unstructured};
-use clap::Parser;
-use fuzzers::Operation;
-use milli::heed::EnvOpenOptions;
-use milli::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig};
-use milli::Index;
-use tempfile::TempDir;
-
-#[derive(Debug, Arbitrary)]
-struct Batch([Operation; 5]);
-
-#[derive(Debug, Clone, Parser)]
-struct Opt {
-    /// The number of fuzzer to run in parallel.
-    #[clap(long)]
-    par: Option<NonZeroUsize>,
-    // We need to put a lot of newlines in the following documentation or else everything gets collapsed on one line
-    /// The path in which the databases will be created.
-    /// Using a ramdisk is recommended.
-    ///
-    /// Linux:
-    ///
-    /// sudo mount -t tmpfs -o size=2g tmpfs ramdisk # to create it
-    ///
-    /// sudo umount ramdisk # to remove it
-    ///
-    /// MacOS:
-    ///
-    /// diskutil erasevolume HFS+ 'RAM Disk' `hdiutil attach -nobrowse -nomount ram://4194304 # create it
-    ///
-    /// hdiutil detach /dev/:the_disk
-    #[clap(long)]
-    path: Option<PathBuf>,
-}
-
-fn main() {
-    let opt = Opt::parse();
-    let progression: &'static AtomicUsize = Box::leak(Box::new(AtomicUsize::new(0)));
-    let stop: &'static AtomicBool = Box::leak(Box::new(AtomicBool::new(false)));
-
-    let par = opt.par.unwrap_or_else(|| std::thread::available_parallelism().unwrap()).get();
-    let mut handles = Vec::with_capacity(par);
-
-    for _ in 0..par {
-        let opt = opt.clone();
-
-        let handle = std::thread::spawn(move || {
-            let mut options = EnvOpenOptions::new();
-            options.map_size(1024 * 1024 * 1024 * 1024);
-            let tempdir = match opt.path {
-                Some(path) => TempDir::new_in(path).unwrap(),
-                None => TempDir::new().unwrap(),
-            };
-            let index = Index::new(options, tempdir.path()).unwrap();
-            let indexer_config = IndexerConfig::default();
-            let index_documents_config = IndexDocumentsConfig::default();
-
-            std::thread::scope(|s| {
-                loop {
-                    if stop.load(Ordering::Relaxed) {
-                        return;
-                    }
-                    let v: Vec<u8> =
-                        std::iter::repeat_with(|| fastrand::u8(..)).take(1000).collect();
-
-                    let mut data = Unstructured::new(&v);
-                    let batches = <[Batch; 5]>::arbitrary(&mut data).unwrap();
-                    // will be used to display the error once a thread crashes
-                    let dbg_input = format!("{:#?}", batches);
-
-                    let handle = s.spawn(|| {
-                        let mut wtxn = index.write_txn().unwrap();
-
-                        for batch in batches {
-                            let mut builder = IndexDocuments::new(
-                                &mut wtxn,
-                                &index,
-                                &indexer_config,
-                                index_documents_config.clone(),
-                                |_| (),
-                                || false,
-                            )
-                            .unwrap();
-
-                            for op in batch.0 {
-                                match op {
-                                    Operation::AddDoc(doc) => {
-                                        let documents =
-                                            milli::documents::objects_from_json_value(doc.to_d());
-                                        let documents =
-                                            milli::documents::documents_batch_reader_from_objects(
-                                                documents,
-                                            );
-                                        let (b, _added) = builder.add_documents(documents).unwrap();
-                                        builder = b;
-                                    }
-                                    Operation::DeleteDoc(id) => {
-                                        let (b, _removed) =
-                                            builder.remove_documents(vec![id.to_s()]).unwrap();
-                                        builder = b;
-                                    }
-                                }
-                            }
-                            builder.execute().unwrap();
-
-                            // after executing a batch we check if the database is corrupted
-                            let res = index.search(&wtxn).execute().unwrap();
-                            index.documents(&wtxn, res.documents_ids).unwrap();
-                            progression.fetch_add(1, Ordering::Relaxed);
-                        }
-                        wtxn.abort().unwrap();
-                    });
-                    if let err @ Err(_) = handle.join() {
-                        stop.store(true, Ordering::Relaxed);
-                        err.expect(&dbg_input);
-                    }
-                }
-            });
-        });
-        handles.push(handle);
-    }
-
-    std::thread::spawn(|| {
-        let mut last_value = 0;
-        let start = std::time::Instant::now();
-        loop {
-            let total = progression.load(Ordering::Relaxed);
-            let elapsed = start.elapsed().as_secs();
-            if elapsed > 3600 {
-                // after 1 hour, stop the fuzzer, success
-                std::process::exit(0);
-            }
-            println!(
-                "Has been running for {:?} seconds. Tested {} new values for a total of {}.",
-                elapsed,
-                total - last_value,
-                total
-            );
-            last_value = total;
-            std::thread::sleep(Duration::from_secs(1));
-        }
-    });
-
-    for handle in handles {
-        handle.join().unwrap();
-    }
-}

fuzzers/src/lib.rs

@@ -1,46 +0,0 @@
-use arbitrary::Arbitrary;
-use serde_json::{json, Value};
-
-#[derive(Debug, Arbitrary)]
-pub enum Document {
-    One,
-    Two,
-    Three,
-    Four,
-    Five,
-    Six,
-}
-
-impl Document {
-    pub fn to_d(&self) -> Value {
-        match self {
-            Document::One => json!({ "id": 0, "doggo": "bernese" }),
-            Document::Two => json!({ "id": 0, "doggo": "golden" }),
-            Document::Three => json!({ "id": 0, "catto": "jorts" }),
-            Document::Four => json!({ "id": 1, "doggo": "bernese" }),
-            Document::Five => json!({ "id": 1, "doggo": "golden" }),
-            Document::Six => json!({ "id": 1, "catto": "jorts" }),
-        }
-    }
-}
-
-#[derive(Debug, Arbitrary)]
-pub enum DocId {
-    Zero,
-    One,
-}
-
-impl DocId {
-    pub fn to_s(&self) -> String {
-        match self {
-            DocId::Zero => "0".to_string(),
-            DocId::One => "1".to_string(),
-        }
-    }
-}
-
-#[derive(Debug, Arbitrary)]
-pub enum Operation {
-    AddDoc(Document),
-    DeleteDoc(DocId),
-}

index-scheduler/Cargo.toml

@@ -22,7 +22,6 @@ log = "0.4.17"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
 page_size = "0.5.0"
-puffin = "0.16.0"
 roaring = { version = "0.10.1", features = ["serde"] }
 serde = { version = "1.0.160", features = ["derive"] }
 serde_json = { version = "1.0.95", features = ["preserve_order"] }

index-scheduler/src/autobatcher.rs

@@ -160,7 +160,7 @@ impl BatchKind {
 impl BatchKind {
     /// Returns a `ControlFlow::Break` if you must stop right now.
     /// The boolean tell you if an index has been created by the batched task.
-    /// To ease the writing of the code. `true` can be returned when you don't need to create an index
+    /// To ease the writting of the code. `true` can be returned when you don't need to create an index
     /// but false can't be returned if you needs to create an index.
     // TODO use an AutoBatchKind as input
     pub fn new(
@@ -214,7 +214,7 @@ impl BatchKind {
 
     /// Returns a `ControlFlow::Break` if you must stop right now.
     /// The boolean tell you if an index has been created by the batched task.
-    /// To ease the writing of the code. `true` can be returned when you don't need to create an index
+    /// To ease the writting of the code. `true` can be returned when you don't need to create an index
     /// but false can't be returned if you needs to create an index.
     #[rustfmt::skip]
     fn accumulate(self, id: TaskId, kind: AutobatchKind, index_already_exists: bool, primary_key: Option<&str>) -> ControlFlow<BatchKind, BatchKind> {
@@ -321,18 +321,9 @@ impl BatchKind {
                 })
             }
             (
-                BatchKind::DocumentOperation { method, allow_index_creation, primary_key, mut operation_ids },
+                this @ BatchKind::DocumentOperation { .. },
                 K::DocumentDeletion,
-            ) => {
-                operation_ids.push(id);
-
-                Continue(BatchKind::DocumentOperation {
-                    method,
-                    allow_index_creation,
-                    primary_key,
-                    operation_ids,
-                })
-            }
+            ) => Break(this),
             // but we can't autobatch documents if it's not the same kind
             // this match branch MUST be AFTER the previous one
             (
@@ -355,35 +346,7 @@ impl BatchKind {
                 deletion_ids.push(id);
                 Continue(BatchKind::DocumentClear { ids: deletion_ids })
             }
-            // we can autobatch the deletion and import if the index already exists
-            (
-                BatchKind::DocumentDeletion { mut deletion_ids },
-                K::DocumentImport { method, allow_index_creation, primary_key }
-            ) if index_already_exists => {
-                deletion_ids.push(id);
-
-                Continue(BatchKind::DocumentOperation {
-                    method,
-                    allow_index_creation,
-                    primary_key,
-                    operation_ids: deletion_ids,
-                })
-            }
-            // we can autobatch the deletion and import if both can't create an index
-            (
-                BatchKind::DocumentDeletion { mut deletion_ids },
-                K::DocumentImport { method, allow_index_creation, primary_key }
-            ) if !allow_index_creation => {
-                deletion_ids.push(id);
-
-                Continue(BatchKind::DocumentOperation {
-                    method,
-                    allow_index_creation,
-                    primary_key,
-                    operation_ids: deletion_ids,
-                })
-            }
-            // we can't autobatch a deletion and an import if the index does not exists but would be created by an addition
+            // we can't autobatch a deletion and an import
             (
                 this @ BatchKind::DocumentDeletion { .. },
                 K::DocumentImport { .. }
@@ -685,36 +648,36 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [settings(false)]), @"Some((Settings { allow_index_creation: false, settings_ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [settings(false), settings(false), settings(false)]), @"Some((Settings { allow_index_creation: false, settings_ids: [0, 1, 2] }, false))");
 
-        // We can autobatch document addition with document deletion
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, true))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, true))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, true))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, true))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-        // And the other way around
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, true, Some("catto"))]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
+        // We can't autobatch document addition with document deletion
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
+        // we also can't do the only way around
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
     }
 
     #[test]

index-scheduler/src/batch.rs

@@ -471,8 +471,6 @@ impl IndexScheduler {
         #[cfg(test)]
         self.maybe_fail(crate::tests::FailureLocation::InsideCreateBatch)?;
 
-        puffin::profile_function!();
-
         let enqueued = &self.get_status(rtxn, Status::Enqueued)?;
         let to_cancel = self.get_kind(rtxn, Kind::TaskCancelation)? & enqueued;
 
@@ -577,9 +575,6 @@ impl IndexScheduler {
             self.maybe_fail(crate::tests::FailureLocation::PanicInsideProcessBatch)?;
             self.breakpoint(crate::Breakpoint::InsideProcessBatch);
         }
-
-        puffin::profile_function!(format!("{:?}", batch));
-
         match batch {
             Batch::TaskCancelation { mut task, previous_started_at, previous_processing_tasks } => {
                 // 1. Retrieve the tasks that matched the query at enqueue-time.
@@ -844,10 +839,6 @@ impl IndexScheduler {
                     Ok(())
                 })?;
 
-                // 4. Dump experimental feature settings
-                let features = self.features()?.runtime_features();
-                dump.create_experimental_features(features)?;
-
                 let dump_uid = started_at.format(format_description!(
                     "[year repr:full][month repr:numerical][day padding:zero]-[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]"
                 )).unwrap();
@@ -1007,7 +998,7 @@ impl IndexScheduler {
                 }()
                 .unwrap_or_default();
 
-                // The write transaction is directly owned and committed inside.
+                // The write transaction is directly owned and commited inside.
                 match self.index_mapper.delete_index(wtxn, &index_uid) {
                     Ok(()) => (),
                     Err(Error::IndexNotFound(_)) if index_has_been_created => (),
@@ -1116,8 +1107,6 @@ impl IndexScheduler {
         index: &'i Index,
         operation: IndexOperation,
     ) -> Result<Vec<Task>> {
-        puffin::profile_function!();
-
         match operation {
             IndexOperation::DocumentClear { mut tasks, .. } => {
                 let count = milli::update::ClearDocuments::new(index_wtxn, index).execute()?;

index-scheduler/src/error.rs

@@ -123,8 +123,6 @@ pub enum Error {
     IoError(#[from] std::io::Error),
     #[error(transparent)]
     Persist(#[from] tempfile::PersistError),
-    #[error(transparent)]
-    FeatureNotEnabled(#[from] FeatureNotEnabledError),
 
     #[error(transparent)]
     Anyhow(#[from] anyhow::Error),
@@ -144,16 +142,6 @@ pub enum Error {
     PlannedFailure,
 }
 
-#[derive(Debug, thiserror::Error)]
-#[error(
-    "{disabled_action} requires enabling the `{feature}` experimental feature. See {issue_link}"
-)]
-pub struct FeatureNotEnabledError {
-    pub disabled_action: &'static str,
-    pub feature: &'static str,
-    pub issue_link: &'static str,
-}
-
 impl Error {
     pub fn is_recoverable(&self) -> bool {
         match self {
@@ -182,7 +170,6 @@ impl Error {
             | Error::FileStore(_)
             | Error::IoError(_)
             | Error::Persist(_)
-            | Error::FeatureNotEnabled(_)
             | Error::Anyhow(_) => true,
             Error::CreateBatch(_)
             | Error::CorruptedTaskQueue
@@ -227,7 +214,6 @@ impl ErrorCode for Error {
             Error::FileStore(e) => e.error_code(),
             Error::IoError(e) => e.error_code(),
             Error::Persist(e) => e.error_code(),
-            Error::FeatureNotEnabled(_) => Code::FeatureNotEnabled,
 
             // Irrecoverable errors
             Error::Anyhow(_) => Code::Internal,

index-scheduler/src/features.rs

@@ -1,98 +0,0 @@
-use meilisearch_types::features::{InstanceTogglableFeatures, RuntimeTogglableFeatures};
-use meilisearch_types::heed::types::{SerdeJson, Str};
-use meilisearch_types::heed::{Database, Env, RoTxn, RwTxn};
-
-use crate::error::FeatureNotEnabledError;
-use crate::Result;
-
-const EXPERIMENTAL_FEATURES: &str = "experimental-features";
-
-#[derive(Clone)]
-pub(crate) struct FeatureData {
-    runtime: Database<Str, SerdeJson<RuntimeTogglableFeatures>>,
-    instance: InstanceTogglableFeatures,
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct RoFeatures {
-    runtime: RuntimeTogglableFeatures,
-    instance: InstanceTogglableFeatures,
-}
-
-impl RoFeatures {
-    fn new(txn: RoTxn<'_>, data: &FeatureData) -> Result<Self> {
-        let runtime = data.runtime_features(txn)?;
-        Ok(Self { runtime, instance: data.instance })
-    }
-
-    pub fn runtime_features(&self) -> RuntimeTogglableFeatures {
-        self.runtime
-    }
-
-    pub fn check_score_details(&self) -> Result<()> {
-        if self.runtime.score_details {
-            Ok(())
-        } else {
-            Err(FeatureNotEnabledError {
-                disabled_action: "Computing score details",
-                feature: "score details",
-                issue_link: "https://github.com/meilisearch/product/discussions/674",
-            }
-            .into())
-        }
-    }
-
-    pub fn check_metrics(&self) -> Result<()> {
-        if self.instance.metrics {
-            Ok(())
-        } else {
-            Err(FeatureNotEnabledError {
-                disabled_action: "Getting metrics",
-                feature: "metrics",
-                issue_link: "https://github.com/meilisearch/meilisearch/discussions/3518",
-            }
-            .into())
-        }
-    }
-
-    pub fn check_vector(&self) -> Result<()> {
-        if self.runtime.vector_store {
-            Ok(())
-        } else {
-            Err(FeatureNotEnabledError {
-                disabled_action: "Passing `vector` as a query parameter",
-                feature: "vector store",
-                issue_link: "https://github.com/meilisearch/product/discussions/677",
-            }
-            .into())
-        }
-    }
-}
-
-impl FeatureData {
-    pub fn new(env: &Env, instance_features: InstanceTogglableFeatures) -> Result<Self> {
-        let mut wtxn = env.write_txn()?;
-        let runtime_features = env.create_database(&mut wtxn, Some(EXPERIMENTAL_FEATURES))?;
-        wtxn.commit()?;
-
-        Ok(Self { runtime: runtime_features, instance: instance_features })
-    }
-
-    pub fn put_runtime_features(
-        &self,
-        mut wtxn: RwTxn,
-        features: RuntimeTogglableFeatures,
-    ) -> Result<()> {
-        self.runtime.put(&mut wtxn, EXPERIMENTAL_FEATURES, &features)?;
-        wtxn.commit()?;
-        Ok(())
-    }
-
-    fn runtime_features(&self, txn: RoTxn) -> Result<RuntimeTogglableFeatures> {
-        Ok(self.runtime.get(&txn, EXPERIMENTAL_FEATURES)?.unwrap_or_default())
-    }
-
-    pub fn features(&self, txn: RoTxn) -> Result<RoFeatures> {
-        RoFeatures::new(txn, self)
-    }
-}

index-scheduler/src/index_mapper/index_map.rs

@@ -223,9 +223,7 @@ impl IndexMap {
         enable_mdb_writemap: bool,
         map_size_growth: usize,
     ) {
-        let Some(index) = self.available.remove(uuid) else {
-            return;
-        };
+        let Some(index) = self.available.remove(uuid) else { return; };
         self.close(*uuid, index, enable_mdb_writemap, map_size_growth);
     }
 

index-scheduler/src/insta_snapshot.rs

@@ -28,7 +28,6 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
         started_at,
         finished_at,
         index_mapper,
-        features: _,
         max_number_of_tasks: _,
         wake_up: _,
         dumps_path: _,

index-scheduler/src/lib.rs

@@ -21,7 +21,6 @@ content of the scheduler or enqueue new tasks.
 mod autobatcher;
 mod batch;
 pub mod error;
-mod features;
 mod index_mapper;
 #[cfg(test)]
 mod insta_snapshot;
@@ -42,10 +41,8 @@ use std::time::Duration;
 
 use dump::{KindDump, TaskDump, UpdateFile};
 pub use error::Error;
-pub use features::RoFeatures;
 use file_store::FileStore;
 use meilisearch_types::error::ResponseError;
-use meilisearch_types::features::{InstanceTogglableFeatures, RuntimeTogglableFeatures};
 use meilisearch_types::heed::types::{OwnedType, SerdeBincode, SerdeJson, Str};
 use meilisearch_types::heed::{self, Database, Env, RoTxn, RwTxn};
 use meilisearch_types::milli::documents::DocumentsBatchBuilder;
@@ -250,8 +247,6 @@ pub struct IndexSchedulerOptions {
     /// The maximum number of tasks stored in the task queue before starting
     /// to auto schedule task deletions.
     pub max_number_of_tasks: usize,
-    /// The experimental features enabled for this instance.
-    pub instance_features: InstanceTogglableFeatures,
 }
 
 /// Structure which holds meilisearch's indexes and schedules the tasks
@@ -295,9 +290,6 @@ pub struct IndexScheduler {
     /// In charge of creating, opening, storing and returning indexes.
     pub(crate) index_mapper: IndexMapper,
 
-    /// In charge of fetching and setting the status of experimental features.
-    features: features::FeatureData,
-
     /// Get a signal when a batch needs to be processed.
     pub(crate) wake_up: Arc<SignalEvent>,
 
@@ -368,7 +360,6 @@ impl IndexScheduler {
             planned_failures: self.planned_failures.clone(),
             #[cfg(test)]
             run_loop_iteration: self.run_loop_iteration.clone(),
-            features: self.features.clone(),
         }
     }
 }
@@ -407,12 +398,9 @@ impl IndexScheduler {
         };
 
         let env = heed::EnvOpenOptions::new()
-            .max_dbs(11)
+            .max_dbs(10)
             .map_size(budget.task_db_size)
             .open(options.tasks_path)?;
-
-        let features = features::FeatureData::new(&env, options.instance_features)?;
-
         let file_store = FileStore::new(&options.update_file_path)?;
 
         let mut wtxn = env.write_txn()?;
@@ -464,7 +452,6 @@ impl IndexScheduler {
             planned_failures,
             #[cfg(test)]
             run_loop_iteration: Arc::new(RwLock::new(0)),
-            features,
         };
 
         this.run();
@@ -1032,8 +1019,6 @@ impl IndexScheduler {
             self.breakpoint(Breakpoint::Start);
         }
 
-        puffin::GlobalProfiler::lock().new_frame();
-
         self.cleanup_task_queue()?;
 
         let rtxn = self.env.read_txn().map_err(Error::HeedTransaction)?;
@@ -1229,17 +1214,6 @@ impl IndexScheduler {
         Ok(IndexStats { is_indexing, inner_stats: index_stats })
     }
 
-    pub fn features(&self) -> Result<RoFeatures> {
-        let rtxn = self.read_txn()?;
-        self.features.features(rtxn)
-    }
-
-    pub fn put_runtime_features(&self, features: RuntimeTogglableFeatures) -> Result<()> {
-        let wtxn = self.env.write_txn().map_err(Error::HeedTransaction)?;
-        self.features.put_runtime_features(wtxn, features)?;
-        Ok(())
-    }
-
     pub(crate) fn delete_persisted_task_data(&self, task: &Task) -> Result<()> {
         match task.content_uuid() {
             Some(content_file) => self.delete_update_file(content_file),
@@ -1560,7 +1534,6 @@ mod tests {
                 indexer_config,
                 autobatching_enabled: true,
                 max_number_of_tasks: 1_000_000,
-                instance_features: Default::default(),
             };
             configuration(&mut options);
 
@@ -1812,7 +1785,7 @@ mod tests {
             assert_eq!(task.kind.as_kind(), k);
         }
 
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "everything_is_successfully_registered");
+        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "everything_is_succesfully_registered");
     }
 
     #[test]
@@ -2102,105 +2075,6 @@ mod tests {
         snapshot!(snapshot_index_scheduler(&index_scheduler), name: "both_task_succeeded");
     }
 
-    #[test]
-    fn document_addition_and_document_deletion() {
-        let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);
-
-        let content = r#"[
-            { "id": 1, "doggo": "jean bob" },
-            { "id": 2, "catto": "jorts" },
-            { "id": 3, "doggo": "bork" }
-        ]"#;
-
-        let (uuid, mut file) = index_scheduler.create_update_file_with_uuid(0).unwrap();
-        let documents_count = read_json(content.as_bytes(), file.as_file_mut()).unwrap();
-        file.persist().unwrap();
-        index_scheduler
-            .register(KindWithContent::DocumentAdditionOrUpdate {
-                index_uid: S("doggos"),
-                primary_key: Some(S("id")),
-                method: ReplaceDocuments,
-                content_file: uuid,
-                documents_count,
-                allow_index_creation: true,
-            })
-            .unwrap();
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_first_task");
-        index_scheduler
-            .register(KindWithContent::DocumentDeletion {
-                index_uid: S("doggos"),
-                documents_ids: vec![S("1"), S("2")],
-            })
-            .unwrap();
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_second_task");
-
-        handle.advance_one_successful_batch(); // The addition AND deletion should've been batched together
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_processing_the_batch");
-
-        let index = index_scheduler.index("doggos").unwrap();
-        let rtxn = index.read_txn().unwrap();
-        let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
-        let field_ids = field_ids_map.ids().collect::<Vec<_>>();
-        let documents = index
-            .all_documents(&rtxn)
-            .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
-            .collect::<Vec<_>>();
-        snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
-    }
-
-    #[test]
-    fn document_deletion_and_document_addition() {
-        let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);
-        index_scheduler
-            .register(KindWithContent::DocumentDeletion {
-                index_uid: S("doggos"),
-                documents_ids: vec![S("1"), S("2")],
-            })
-            .unwrap();
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_first_task");
-
-        let content = r#"[
-            { "id": 1, "doggo": "jean bob" },
-            { "id": 2, "catto": "jorts" },
-            { "id": 3, "doggo": "bork" }
-        ]"#;
-
-        let (uuid, mut file) = index_scheduler.create_update_file_with_uuid(0).unwrap();
-        let documents_count = read_json(content.as_bytes(), file.as_file_mut()).unwrap();
-        file.persist().unwrap();
-        index_scheduler
-            .register(KindWithContent::DocumentAdditionOrUpdate {
-                index_uid: S("doggos"),
-                primary_key: Some(S("id")),
-                method: ReplaceDocuments,
-                content_file: uuid,
-                documents_count,
-                allow_index_creation: true,
-            })
-            .unwrap();
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_second_task");
-
-        // The deletion should have failed because it can't create an index
-        handle.advance_one_failed_batch();
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_failing_the_deletion");
-
-        // The addition should works
-        handle.advance_one_successful_batch();
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_last_successful_addition");
-
-        let index = index_scheduler.index("doggos").unwrap();
-        let rtxn = index.read_txn().unwrap();
-        let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
-        let field_ids = field_ids_map.ids().collect::<Vec<_>>();
-        let documents = index
-            .all_documents(&rtxn)
-            .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
-            .collect::<Vec<_>>();
-        snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
-    }
-
     #[test]
     fn do_not_batch_task_of_different_indexes() {
         let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);

index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/after_processing_the_batch.snap

@@ -1,43 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
-1 {uid: 1, status: succeeded, details: { received_document_ids: 2, deleted_documents: Some(2) }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1", "2"] }}
-----------------------------------------------------------------------
-### Status:
-enqueued []
-succeeded [0,1,]
-----------------------------------------------------------------------
-### Kind:
-"documentAdditionOrUpdate" [0,]
-"documentDeletion" [1,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,1,]
-----------------------------------------------------------------------
-### Index Mapper:
-doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} }
-
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### Started At:
-[timestamp] [0,1,]
-----------------------------------------------------------------------
-### Finished At:
-[timestamp] [0,1,]
-----------------------------------------------------------------------
-### File Store:
-
-----------------------------------------------------------------------
-

index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/documents.snap

@@ -1,9 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-[
-  {
-    "id": 3,
-    "doggo": "bork"
-  }
-]

index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_first_task.snap

@@ -1,37 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
-----------------------------------------------------------------------
-### Status:
-enqueued [0,]
-----------------------------------------------------------------------
-### Kind:
-"documentAdditionOrUpdate" [0,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,]
-----------------------------------------------------------------------
-### Index Mapper:
-
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-----------------------------------------------------------------------
-### Started At:
-----------------------------------------------------------------------
-### Finished At:
-----------------------------------------------------------------------
-### File Store:
-00000000-0000-0000-0000-000000000000
-
-----------------------------------------------------------------------
-

index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_second_task.snap

@@ -1,40 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
-1 {uid: 1, status: enqueued, details: { received_document_ids: 2, deleted_documents: None }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1", "2"] }}
-----------------------------------------------------------------------
-### Status:
-enqueued [0,1,]
-----------------------------------------------------------------------
-### Kind:
-"documentAdditionOrUpdate" [0,]
-"documentDeletion" [1,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,1,]
-----------------------------------------------------------------------
-### Index Mapper:
-
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### Started At:
-----------------------------------------------------------------------
-### Finished At:
-----------------------------------------------------------------------
-### File Store:
-00000000-0000-0000-0000-000000000000
-
-----------------------------------------------------------------------
-

index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_failing_the_deletion.snap

@@ -1,43 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: failed, error: ResponseError { code: 200, message: "Index `doggos` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { received_document_ids: 2, deleted_documents: Some(0) }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1", "2"] }}
-1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
-----------------------------------------------------------------------
-### Status:
-enqueued [1,]
-failed [0,]
-----------------------------------------------------------------------
-### Kind:
-"documentAdditionOrUpdate" [1,]
-"documentDeletion" [0,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,1,]
-----------------------------------------------------------------------
-### Index Mapper:
-
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### Started At:
-[timestamp] [0,]
-----------------------------------------------------------------------
-### Finished At:
-[timestamp] [0,]
-----------------------------------------------------------------------
-### File Store:
-00000000-0000-0000-0000-000000000000
-
-----------------------------------------------------------------------
-

index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_last_successful_addition.snap

@@ -1,46 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: failed, error: ResponseError { code: 200, message: "Index `doggos` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { received_document_ids: 2, deleted_documents: Some(0) }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1", "2"] }}
-1 {uid: 1, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
-----------------------------------------------------------------------
-### Status:
-enqueued []
-succeeded [1,]
-failed [0,]
-----------------------------------------------------------------------
-### Kind:
-"documentAdditionOrUpdate" [1,]
-"documentDeletion" [0,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,1,]
-----------------------------------------------------------------------
-### Index Mapper:
-doggos: { number_of_documents: 3, field_distribution: {"catto": 1, "doggo": 2, "id": 3} }
-
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### Started At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### Finished At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### File Store:
-
-----------------------------------------------------------------------
-

index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/documents.snap

@@ -1,17 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-[
-  {
-    "id": 1,
-    "doggo": "jean bob"
-  },
-  {
-    "id": 2,
-    "catto": "jorts"
-  },
-  {
-    "id": 3,
-    "doggo": "bork"
-  }
-]

index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_first_task.snap

@@ -1,36 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: enqueued, details: { received_document_ids: 2, deleted_documents: None }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1", "2"] }}
-----------------------------------------------------------------------
-### Status:
-enqueued [0,]
-----------------------------------------------------------------------
-### Kind:
-"documentDeletion" [0,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,]
-----------------------------------------------------------------------
-### Index Mapper:
-
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-----------------------------------------------------------------------
-### Started At:
-----------------------------------------------------------------------
-### Finished At:
-----------------------------------------------------------------------
-### File Store:
-
-----------------------------------------------------------------------
-

index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_second_task.snap

@@ -1,40 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: enqueued, details: { received_document_ids: 2, deleted_documents: None }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1", "2"] }}
-1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
-----------------------------------------------------------------------
-### Status:
-enqueued [0,1,]
-----------------------------------------------------------------------
-### Kind:
-"documentAdditionOrUpdate" [1,]
-"documentDeletion" [0,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,1,]
-----------------------------------------------------------------------
-### Index Mapper:
-
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### Started At:
-----------------------------------------------------------------------
-### Finished At:
-----------------------------------------------------------------------
-### File Store:
-00000000-0000-0000-0000-000000000000
-
-----------------------------------------------------------------------
-

meilisearch-types/src/deserr/mod.rs

@@ -151,10 +151,6 @@ make_missing_field_convenience_builder!(MissingApiKeyExpiresAt, missing_api_key_
 make_missing_field_convenience_builder!(MissingApiKeyIndexes, missing_api_key_indexes);
 make_missing_field_convenience_builder!(MissingSwapIndexes, missing_swap_indexes);
 make_missing_field_convenience_builder!(MissingDocumentFilter, missing_document_filter);
-make_missing_field_convenience_builder!(
-    MissingFacetSearchFacetName,
-    missing_facet_search_facet_name
-);
 
 // Integrate a sub-error into a [`DeserrError`] by taking its error message but using
 // the default error code (C) from `Self`

meilisearch-types/src/error.rs

@@ -217,8 +217,6 @@ InvalidDocumentFields                 , InvalidRequest       , BAD_REQUEST ;
 MissingDocumentFilter                 , InvalidRequest       , BAD_REQUEST ;
 InvalidDocumentFilter                 , InvalidRequest       , BAD_REQUEST ;
 InvalidDocumentGeoField               , InvalidRequest       , BAD_REQUEST ;
-InvalidVectorDimensions               , InvalidRequest       , BAD_REQUEST ;
-InvalidVectorsType                    , InvalidRequest       , BAD_REQUEST ;
 InvalidDocumentId                     , InvalidRequest       , BAD_REQUEST ;
 InvalidDocumentLimit                  , InvalidRequest       , BAD_REQUEST ;
 InvalidDocumentOffset                 , InvalidRequest       , BAD_REQUEST ;
@@ -226,14 +224,12 @@ InvalidIndexLimit                     , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexOffset                    , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexPrimaryKey                , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexUid                       , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchAttributesToSearchOn     , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToCrop         , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToHighlight    , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToRetrieve     , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchCropLength               , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchCropMarker               , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchFacets                   , InvalidRequest       , BAD_REQUEST ;
-InvalidFacetSearchFacetName           , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchFilter                   , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchHighlightPostTag         , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchHighlightPreTag          , InvalidRequest       , BAD_REQUEST ;
@@ -243,9 +239,6 @@ InvalidSearchMatchingStrategy         , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchOffset                   , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchPage                     , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchQ                        , InvalidRequest       , BAD_REQUEST ;
-InvalidFacetSearchQuery               , InvalidRequest       , BAD_REQUEST ;
-InvalidFacetSearchName                , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchVector                   , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchShowMatchesPosition      , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchShowRankingScore         , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchShowRankingScoreDetails  , InvalidRequest       , BAD_REQUEST ;
@@ -259,9 +252,6 @@ InvalidSettingsRankingRules           , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsSearchableAttributes   , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsSortableAttributes     , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsStopWords              , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsNonSeparatorTokens     , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsSeparatorTokens        , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsDictionary             , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsSynonyms               , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsTypoTolerance          , InvalidRequest       , BAD_REQUEST ;
 InvalidState                          , Internal             , INTERNAL_SERVER_ERROR ;
@@ -281,7 +271,6 @@ InvalidTaskStatuses                   , InvalidRequest       , BAD_REQUEST ;
 InvalidTaskTypes                      , InvalidRequest       , BAD_REQUEST ;
 InvalidTaskUids                       , InvalidRequest       , BAD_REQUEST  ;
 IoError                               , System               , UNPROCESSABLE_ENTITY;
-FeatureNotEnabled                     , InvalidRequest       , BAD_REQUEST ;
 MalformedPayload                      , InvalidRequest       , BAD_REQUEST ;
 MaxFieldsLimitExceeded                , InvalidRequest       , BAD_REQUEST ;
 MissingApiKeyActions                  , InvalidRequest       , BAD_REQUEST ;
@@ -290,7 +279,6 @@ MissingApiKeyIndexes                  , InvalidRequest       , BAD_REQUEST ;
 MissingAuthorizationHeader            , Auth                 , UNAUTHORIZED ;
 MissingContentType                    , InvalidRequest       , UNSUPPORTED_MEDIA_TYPE ;
 MissingDocumentId                     , InvalidRequest       , BAD_REQUEST ;
-MissingFacetSearchFacetName           , InvalidRequest       , BAD_REQUEST ;
 MissingIndexUid                       , InvalidRequest       , BAD_REQUEST ;
 MissingMasterKey                      , Auth                 , UNAUTHORIZED ;
 MissingPayload                        , InvalidRequest       , BAD_REQUEST ;
@@ -344,16 +332,8 @@ impl ErrorCode for milli::Error {
                     UserError::SortRankingRuleMissing => Code::InvalidSearchSort,
                     UserError::InvalidFacetsDistribution { .. } => Code::InvalidSearchFacets,
                     UserError::InvalidSortableAttribute { .. } => Code::InvalidSearchSort,
-                    UserError::InvalidSearchableAttribute { .. } => {
-                        Code::InvalidSearchAttributesToSearchOn
-                    }
-                    UserError::InvalidFacetSearchFacetName { .. } => {
-                        Code::InvalidFacetSearchFacetName
-                    }
                     UserError::CriterionError(_) => Code::InvalidSettingsRankingRules,
                     UserError::InvalidGeoField { .. } => Code::InvalidDocumentGeoField,
-                    UserError::InvalidVectorDimensions { .. } => Code::InvalidVectorDimensions,
-                    UserError::InvalidVectorsType { .. } => Code::InvalidVectorsType,
                     UserError::SortError(_) => Code::InvalidSearchSort,
                     UserError::InvalidMinTypoWordLenSetting(_, _) => {
                         Code::InvalidSettingsTypoTolerance

meilisearch-types/src/facet_values_sort.rs

@@ -1,33 +0,0 @@
-use deserr::Deserr;
-use milli::OrderBy;
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Deserr)]
-#[serde(rename_all = "camelCase")]
-#[deserr(rename_all = camelCase)]
-pub enum FacetValuesSort {
-    /// Facet values are sorted in alphabetical order, ascending from A to Z.
-    #[default]
-    Alpha,
-    /// Facet values are sorted by decreasing count.
-    /// The count is the number of records containing this facet value in the results of the query.
-    Count,
-}
-
-impl From<FacetValuesSort> for OrderBy {
-    fn from(val: FacetValuesSort) -> Self {
-        match val {
-            FacetValuesSort::Alpha => OrderBy::Lexicographic,
-            FacetValuesSort::Count => OrderBy::Count,
-        }
-    }
-}
-
-impl From<OrderBy> for FacetValuesSort {
-    fn from(val: OrderBy) -> Self {
-        match val {
-            OrderBy::Lexicographic => FacetValuesSort::Alpha,
-            OrderBy::Count => FacetValuesSort::Count,
-        }
-    }
-}

meilisearch-types/src/features.rs

@@ -1,13 +0,0 @@
-use serde::{Deserialize, Serialize};
-
-#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default)]
-#[serde(rename_all = "camelCase", default)]
-pub struct RuntimeTogglableFeatures {
-    pub score_details: bool,
-    pub vector_store: bool,
-}
-
-#[derive(Default, Debug, Clone, Copy)]
-pub struct InstanceTogglableFeatures {
-    pub metrics: bool,
-}

meilisearch-types/src/keys.rs

@@ -147,7 +147,9 @@ impl Key {
 fn parse_expiration_date(
     string: Option<String>,
 ) -> std::result::Result<Option<OffsetDateTime>, ParseOffsetDateTimeError> {
-    let Some(string) = string else { return Ok(None) };
+    let Some(string) = string else {
+        return Ok(None)
+    };
     let datetime = if let Ok(datetime) = OffsetDateTime::parse(&string, &Rfc3339) {
         datetime
     } else if let Ok(primitive_datetime) = PrimitiveDateTime::parse(
@@ -272,12 +274,6 @@ pub enum Action {
     #[serde(rename = "keys.delete")]
     #[deserr(rename = "keys.delete")]
     KeysDelete,
-    #[serde(rename = "experimental.get")]
-    #[deserr(rename = "experimental.get")]
-    ExperimentalFeaturesGet,
-    #[serde(rename = "experimental.update")]
-    #[deserr(rename = "experimental.update")]
-    ExperimentalFeaturesUpdate,
 }
 
 impl Action {
@@ -314,8 +310,6 @@ impl Action {
             KEYS_GET => Some(Self::KeysGet),
             KEYS_UPDATE => Some(Self::KeysUpdate),
             KEYS_DELETE => Some(Self::KeysDelete),
-            EXPERIMENTAL_FEATURES_GET => Some(Self::ExperimentalFeaturesGet),
-            EXPERIMENTAL_FEATURES_UPDATE => Some(Self::ExperimentalFeaturesUpdate),
             _otherwise => None,
         }
     }
@@ -358,6 +352,4 @@ pub mod actions {
     pub const KEYS_GET: u8 = KeysGet.repr();
     pub const KEYS_UPDATE: u8 = KeysUpdate.repr();
     pub const KEYS_DELETE: u8 = KeysDelete.repr();
-    pub const EXPERIMENTAL_FEATURES_GET: u8 = ExperimentalFeaturesGet.repr();
-    pub const EXPERIMENTAL_FEATURES_UPDATE: u8 = ExperimentalFeaturesUpdate.repr();
 }

meilisearch-types/src/lib.rs

@@ -2,8 +2,6 @@ pub mod compression;
 pub mod deserr;
 pub mod document_formats;
 pub mod error;
-pub mod facet_values_sort;
-pub mod features;
 pub mod index_uid;
 pub mod index_uid_pattern;
 pub mod keys;

meilisearch-types/src/settings.rs

@@ -14,9 +14,8 @@ use serde::{Deserialize, Serialize, Serializer};
 
 use crate::deserr::DeserrJsonError;
 use crate::error::deserr_codes::*;
-use crate::facet_values_sort::FacetValuesSort;
 
-/// The maximum number of results that the engine
+/// The maximimum number of results that the engine
 /// will be able to return in one search call.
 pub const DEFAULT_PAGINATION_MAX_TOTAL_HITS: usize = 1000;
 
@@ -103,9 +102,6 @@ pub struct FacetingSettings {
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default)]
     pub max_values_per_facet: Setting<usize>,
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
-    #[deserr(default)]
-    pub sort_facet_values_by: Setting<BTreeMap<String, FacetValuesSort>>,
 }
 
 #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr)]
@@ -171,15 +167,6 @@ pub struct Settings<T> {
     #[deserr(default, error = DeserrJsonError<InvalidSettingsStopWords>)]
     pub stop_words: Setting<BTreeSet<String>>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
-    #[deserr(default, error = DeserrJsonError<InvalidSettingsNonSeparatorTokens>)]
-    pub non_separator_tokens: Setting<BTreeSet<String>>,
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
-    #[deserr(default, error = DeserrJsonError<InvalidSettingsSeparatorTokens>)]
-    pub separator_tokens: Setting<BTreeSet<String>>,
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
-    #[deserr(default, error = DeserrJsonError<InvalidSettingsDictionary>)]
-    pub dictionary: Setting<BTreeSet<String>>,
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsSynonyms>)]
     pub synonyms: Setting<BTreeMap<String, Vec<String>>>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
@@ -210,9 +197,6 @@ impl Settings<Checked> {
             ranking_rules: Setting::Reset,
             stop_words: Setting::Reset,
             synonyms: Setting::Reset,
-            non_separator_tokens: Setting::Reset,
-            separator_tokens: Setting::Reset,
-            dictionary: Setting::Reset,
             distinct_attribute: Setting::Reset,
             typo_tolerance: Setting::Reset,
             faceting: Setting::Reset,
@@ -229,9 +213,6 @@ impl Settings<Checked> {
             sortable_attributes,
             ranking_rules,
             stop_words,
-            non_separator_tokens,
-            separator_tokens,
-            dictionary,
             synonyms,
             distinct_attribute,
             typo_tolerance,
@@ -247,9 +228,6 @@ impl Settings<Checked> {
             sortable_attributes,
             ranking_rules,
             stop_words,
-            non_separator_tokens,
-            separator_tokens,
-            dictionary,
             synonyms,
             distinct_attribute,
             typo_tolerance,
@@ -292,9 +270,6 @@ impl Settings<Unchecked> {
             ranking_rules: self.ranking_rules,
             stop_words: self.stop_words,
             synonyms: self.synonyms,
-            non_separator_tokens: self.non_separator_tokens,
-            separator_tokens: self.separator_tokens,
-            dictionary: self.dictionary,
             distinct_attribute: self.distinct_attribute,
             typo_tolerance: self.typo_tolerance,
             faceting: self.faceting,
@@ -356,28 +331,6 @@ pub fn apply_settings_to_builder(
         Setting::NotSet => (),
     }
 
-    match settings.non_separator_tokens {
-        Setting::Set(ref non_separator_tokens) => {
-            builder.set_non_separator_tokens(non_separator_tokens.clone())
-        }
-        Setting::Reset => builder.reset_non_separator_tokens(),
-        Setting::NotSet => (),
-    }
-
-    match settings.separator_tokens {
-        Setting::Set(ref separator_tokens) => {
-            builder.set_separator_tokens(separator_tokens.clone())
-        }
-        Setting::Reset => builder.reset_separator_tokens(),
-        Setting::NotSet => (),
-    }
-
-    match settings.dictionary {
-        Setting::Set(ref dictionary) => builder.set_dictionary(dictionary.clone()),
-        Setting::Reset => builder.reset_dictionary(),
-        Setting::NotSet => (),
-    }
-
     match settings.synonyms {
         Setting::Set(ref synonyms) => builder.set_synonyms(synonyms.clone().into_iter().collect()),
         Setting::Reset => builder.reset_synonyms(),
@@ -445,25 +398,13 @@ pub fn apply_settings_to_builder(
         Setting::NotSet => (),
     }
 
-    match &settings.faceting {
-        Setting::Set(FacetingSettings { max_values_per_facet, sort_facet_values_by }) => {
-            match max_values_per_facet {
-                Setting::Set(val) => builder.set_max_values_per_facet(*val),
-                Setting::Reset => builder.reset_max_values_per_facet(),
-                Setting::NotSet => (),
-            }
-            match sort_facet_values_by {
-                Setting::Set(val) => builder.set_sort_facet_values_by(
-                    val.iter().map(|(name, order)| (name.clone(), (*order).into())).collect(),
-                ),
-                Setting::Reset => builder.reset_sort_facet_values_by(),
-                Setting::NotSet => (),
-            }
-        }
-        Setting::Reset => {
-            builder.reset_max_values_per_facet();
-            builder.reset_sort_facet_values_by();
-        }
+    match settings.faceting {
+        Setting::Set(ref value) => match value.max_values_per_facet {
+            Setting::Set(val) => builder.set_max_values_per_facet(val),
+            Setting::Reset => builder.reset_max_values_per_facet(),
+            Setting::NotSet => (),
+        },
+        Setting::Reset => builder.reset_max_values_per_facet(),
         Setting::NotSet => (),
     }
 
@@ -502,11 +443,6 @@ pub fn settings(
         })
         .transpose()?
         .unwrap_or_default();
-
-    let non_separator_tokens = index.non_separator_tokens(rtxn)?.unwrap_or_default();
-    let separator_tokens = index.separator_tokens(rtxn)?.unwrap_or_default();
-    let dictionary = index.dictionary(rtxn)?.unwrap_or_default();
-
     let distinct_field = index.distinct_field(rtxn)?.map(String::from);
 
     // in milli each word in the synonyms map were split on their separator. Since we lost
@@ -540,13 +476,6 @@ pub fn settings(
         max_values_per_facet: Setting::Set(
             index.max_values_per_facet(rtxn)?.unwrap_or(DEFAULT_VALUES_PER_FACET),
         ),
-        sort_facet_values_by: Setting::Set(
-            index
-                .sort_facet_values_by(rtxn)?
-                .into_iter()
-                .map(|(name, sort)| (name, sort.into()))
-                .collect(),
-        ),
     };
 
     let pagination = PaginationSettings {
@@ -568,9 +497,6 @@ pub fn settings(
         sortable_attributes: Setting::Set(sortable_attributes),
         ranking_rules: Setting::Set(criteria.iter().map(|c| c.clone().into()).collect()),
         stop_words: Setting::Set(stop_words),
-        non_separator_tokens: Setting::Set(non_separator_tokens),
-        separator_tokens: Setting::Set(separator_tokens),
-        dictionary: Setting::Set(dictionary),
         distinct_attribute: match distinct_field {
             Some(field) => Setting::Set(field),
             None => Setting::Reset,
@@ -693,9 +619,6 @@ pub(crate) mod test {
             sortable_attributes: Setting::NotSet,
             ranking_rules: Setting::NotSet,
             stop_words: Setting::NotSet,
-            non_separator_tokens: Setting::NotSet,
-            separator_tokens: Setting::NotSet,
-            dictionary: Setting::NotSet,
             synonyms: Setting::NotSet,
             distinct_attribute: Setting::NotSet,
             typo_tolerance: Setting::NotSet,
@@ -717,9 +640,6 @@ pub(crate) mod test {
             sortable_attributes: Setting::NotSet,
             ranking_rules: Setting::NotSet,
             stop_words: Setting::NotSet,
-            non_separator_tokens: Setting::NotSet,
-            separator_tokens: Setting::NotSet,
-            dictionary: Setting::NotSet,
             synonyms: Setting::NotSet,
             distinct_attribute: Setting::NotSet,
             typo_tolerance: Setting::NotSet,

meilisearch/Cargo.toml

@@ -14,27 +14,14 @@ default-run = "meilisearch"
 
 [dependencies]
 actix-cors = "0.6.4"
-actix-http = { version = "3.3.1", default-features = false, features = [
-    "compress-brotli",
-    "compress-gzip",
-    "rustls",
-] }
-actix-web = { version = "4.3.1", default-features = false, features = [
-    "macros",
-    "compress-brotli",
-    "compress-gzip",
-    "cookies",
-    "rustls",
-] }
+actix-http = { version = "3.3.1", default-features = false, features = ["compress-brotli", "compress-gzip", "rustls"] }
+actix-web = { version = "4.3.1", default-features = false, features = ["macros", "compress-brotli", "compress-gzip", "cookies", "rustls"] }
 actix-web-static-files = { git = "https://github.com/kilork/actix-web-static-files.git", rev = "2d3b6160", optional = true }
 anyhow = { version = "1.0.70", features = ["backtrace"] }
 async-stream = "0.3.5"
 async-trait = "0.1.68"
 bstr = "1.4.0"
-byte-unit = { version = "4.0.19", default-features = false, features = [
-    "std",
-    "serde",
-] }
+byte-unit = { version = "4.0.19", default-features = false, features = ["std", "serde"] }
 bytes = "1.4.0"
 clap = { version = "4.2.1", features = ["derive", "env"] }
 crossbeam-channel = "0.5.8"
@@ -61,21 +48,15 @@ mime = "0.3.17"
 num_cpus = "1.15.0"
 obkv = "0.2.0"
 once_cell = "1.17.1"
-ordered-float = "3.7.0"
 parking_lot = "0.12.1"
 permissive-json-pointer = { path = "../permissive-json-pointer" }
 pin-project-lite = "0.2.9"
 platform-dirs = "0.3.0"
 prometheus = { version = "0.13.3", features = ["process"] }
-puffin = "0.16.0"
-puffin_http = { version = "0.13.0", optional = true }
 rand = "0.8.5"
 rayon = "1.7.0"
 regex = "1.7.3"
-reqwest = { version = "0.11.16", features = [
-    "rustls-tls",
-    "json",
-], default-features = false }
+reqwest = { version = "0.11.16", features = ["rustls-tls", "json"], default-features = false }
 rustls = "0.20.8"
 rustls-pemfile = "1.0.2"
 segment = { version = "0.2.2", optional = true }
@@ -89,12 +70,7 @@ sysinfo = "0.28.4"
 tar = "0.4.38"
 tempfile = "3.5.0"
 thiserror = "1.0.40"
-time = { version = "0.3.20", features = [
-    "serde-well-known",
-    "formatting",
-    "parsing",
-    "macros",
-] }
+time = { version = "0.3.20", features = ["serde-well-known", "formatting", "parsing", "macros"] }
 tokio = { version = "1.27.0", features = ["full"] }
 tokio-stream = "0.1.12"
 toml = "0.7.3"
@@ -113,7 +89,7 @@ brotli = "3.3.4"
 insta = "1.29.0"
 manifest-dir-macros = "0.1.16"
 maplit = "1.0.2"
-meili-snap = { path = "../meili-snap" }
+meili-snap = {path = "../meili-snap"}
 temp-env = "0.3.3"
 urlencoding = "2.1.2"
 yaup = "0.2.1"
@@ -122,10 +98,7 @@ yaup = "0.2.1"
 anyhow = { version = "1.0.70", optional = true }
 cargo_toml = { version = "0.15.2", optional = true }
 hex = { version = "0.4.3", optional = true }
-reqwest = { version = "0.11.16", features = [
-    "blocking",
-    "rustls-tls",
-], default-features = false, optional = true }
+reqwest = { version = "0.11.16", features = ["blocking", "rustls-tls"], default-features = false, optional = true }
 sha-1 = { version = "0.10.1", optional = true }
 static-files = { version = "0.2.3", optional = true }
 tempfile = { version = "3.5.0", optional = true }
@@ -135,18 +108,7 @@ zip = { version = "0.6.4", optional = true }
 [features]
 default = ["analytics", "meilisearch-types/all-tokenizations", "mini-dashboard"]
 analytics = ["segment"]
-profile-with-puffin = ["dep:puffin_http"]
-mini-dashboard = [
-    "actix-web-static-files",
-    "static-files",
-    "anyhow",
-    "cargo_toml",
-    "hex",
-    "reqwest",
-    "sha-1",
-    "tempfile",
-    "zip",
-]
+mini-dashboard = ["actix-web-static-files", "static-files", "anyhow", "cargo_toml", "hex", "reqwest", "sha-1", "tempfile", "zip"]
 chinese = ["meilisearch-types/chinese"]
 hebrew = ["meilisearch-types/hebrew"]
 japanese = ["meilisearch-types/japanese"]

meilisearch/src/analytics/mock_analytics.rs

@@ -38,18 +38,6 @@ impl MultiSearchAggregator {
     pub fn succeed(&mut self) {}
 }
 
-#[derive(Default)]
-pub struct FacetSearchAggregator;
-
-#[allow(dead_code)]
-impl FacetSearchAggregator {
-    pub fn from_query(_: &dyn Any, _: &dyn Any) -> Self {
-        Self::default()
-    }
-
-    pub fn succeed(&mut self, _: &dyn Any) {}
-}
-
 impl MockAnalytics {
     #[allow(clippy::new_ret_no_self)]
     pub fn new(opt: &Opt) -> Arc<dyn Analytics> {
@@ -68,7 +56,6 @@ impl Analytics for MockAnalytics {
     fn get_search(&self, _aggregate: super::SearchAggregator) {}
     fn post_search(&self, _aggregate: super::SearchAggregator) {}
     fn post_multi_search(&self, _aggregate: super::MultiSearchAggregator) {}
-    fn post_facet_search(&self, _aggregate: super::FacetSearchAggregator) {}
     fn add_documents(
         &self,
         _documents_query: &UpdateDocumentsQuery,

meilisearch/src/analytics/mod.rs

@@ -25,8 +25,6 @@ pub type SegmentAnalytics = mock_analytics::MockAnalytics;
 pub type SearchAggregator = mock_analytics::SearchAggregator;
 #[cfg(any(debug_assertions, not(feature = "analytics")))]
 pub type MultiSearchAggregator = mock_analytics::MultiSearchAggregator;
-#[cfg(any(debug_assertions, not(feature = "analytics")))]
-pub type FacetSearchAggregator = mock_analytics::FacetSearchAggregator;
 
 // if we are in release mode and the feature analytics was enabled
 // we use the real analytics
@@ -36,8 +34,6 @@ pub type SegmentAnalytics = segment_analytics::SegmentAnalytics;
 pub type SearchAggregator = segment_analytics::SearchAggregator;
 #[cfg(all(not(debug_assertions), feature = "analytics"))]
 pub type MultiSearchAggregator = segment_analytics::MultiSearchAggregator;
-#[cfg(all(not(debug_assertions), feature = "analytics"))]
-pub type FacetSearchAggregator = segment_analytics::FacetSearchAggregator;
 
 /// The Meilisearch config dir:
 /// `~/.config/Meilisearch` on *NIX or *BSD.
@@ -92,9 +88,6 @@ pub trait Analytics: Sync + Send {
     /// This method should be called to aggregate a post array of searches
     fn post_multi_search(&self, aggregate: MultiSearchAggregator);
 
-    /// This method should be called to aggregate post facet values searches
-    fn post_facet_search(&self, aggregate: FacetSearchAggregator);
-
     // this method should be called to aggregate a add documents request
     fn add_documents(
         &self,

meilisearch/src/analytics/segment_analytics.rs

@@ -1,6 +1,5 @@
 use std::collections::{BinaryHeap, HashMap, HashSet};
 use std::fs;
-use std::mem::take;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::{Duration, Instant};
@@ -30,13 +29,11 @@ use super::{
 use crate::analytics::Analytics;
 use crate::option::{default_http_addr, IndexerOpts, MaxMemory, MaxThreads, ScheduleSnapshot};
 use crate::routes::indexes::documents::UpdateDocumentsQuery;
-use crate::routes::indexes::facet_search::FacetSearchQuery;
 use crate::routes::tasks::TasksFilterQuery;
 use crate::routes::{create_all_stats, Stats};
 use crate::search::{
-    FacetSearchResult, MatchingStrategy, SearchQuery, SearchQueryWithIndex, SearchResult,
-    DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG,
-    DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT,
+    SearchQuery, SearchQueryWithIndex, SearchResult, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER,
+    DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT,
 };
 use crate::Opt;
 
@@ -74,7 +71,6 @@ pub enum AnalyticsMsg {
     AggregateGetSearch(SearchAggregator),
     AggregatePostSearch(SearchAggregator),
     AggregatePostMultiSearch(MultiSearchAggregator),
-    AggregatePostFacetSearch(FacetSearchAggregator),
     AggregateAddDocuments(DocumentsAggregator),
     AggregateDeleteDocuments(DocumentsDeletionAggregator),
     AggregateUpdateDocuments(DocumentsAggregator),
@@ -143,7 +139,6 @@ impl SegmentAnalytics {
             batcher,
             post_search_aggregator: SearchAggregator::default(),
             post_multi_search_aggregator: MultiSearchAggregator::default(),
-            post_facet_search_aggregator: FacetSearchAggregator::default(),
             get_search_aggregator: SearchAggregator::default(),
             add_documents_aggregator: DocumentsAggregator::default(),
             delete_documents_aggregator: DocumentsDeletionAggregator::default(),
@@ -187,10 +182,6 @@ impl super::Analytics for SegmentAnalytics {
         let _ = self.sender.try_send(AnalyticsMsg::AggregatePostSearch(aggregate));
     }
 
-    fn post_facet_search(&self, aggregate: FacetSearchAggregator) {
-        let _ = self.sender.try_send(AnalyticsMsg::AggregatePostFacetSearch(aggregate));
-    }
-
     fn post_multi_search(&self, aggregate: MultiSearchAggregator) {
         let _ = self.sender.try_send(AnalyticsMsg::AggregatePostMultiSearch(aggregate));
     }
@@ -363,7 +354,6 @@ pub struct Segment {
     get_search_aggregator: SearchAggregator,
     post_search_aggregator: SearchAggregator,
     post_multi_search_aggregator: MultiSearchAggregator,
-    post_facet_search_aggregator: FacetSearchAggregator,
     add_documents_aggregator: DocumentsAggregator,
     delete_documents_aggregator: DocumentsDeletionAggregator,
     update_documents_aggregator: DocumentsAggregator,
@@ -428,7 +418,6 @@ impl Segment {
                         Some(AnalyticsMsg::AggregateGetSearch(agreg)) => self.get_search_aggregator.aggregate(agreg),
                         Some(AnalyticsMsg::AggregatePostSearch(agreg)) => self.post_search_aggregator.aggregate(agreg),
                         Some(AnalyticsMsg::AggregatePostMultiSearch(agreg)) => self.post_multi_search_aggregator.aggregate(agreg),
-                        Some(AnalyticsMsg::AggregatePostFacetSearch(agreg)) => self.post_facet_search_aggregator.aggregate(agreg),
                         Some(AnalyticsMsg::AggregateAddDocuments(agreg)) => self.add_documents_aggregator.aggregate(agreg),
                         Some(AnalyticsMsg::AggregateDeleteDocuments(agreg)) => self.delete_documents_aggregator.aggregate(agreg),
                         Some(AnalyticsMsg::AggregateUpdateDocuments(agreg)) => self.update_documents_aggregator.aggregate(agreg),
@@ -472,74 +461,55 @@ impl Segment {
                 })
                 .await;
         }
+        let get_search = std::mem::take(&mut self.get_search_aggregator)
+            .into_event(&self.user, "Documents Searched GET");
+        let post_search = std::mem::take(&mut self.post_search_aggregator)
+            .into_event(&self.user, "Documents Searched POST");
+        let post_multi_search = std::mem::take(&mut self.post_multi_search_aggregator)
+            .into_event(&self.user, "Documents Searched by Multi-Search POST");
+        let add_documents = std::mem::take(&mut self.add_documents_aggregator)
+            .into_event(&self.user, "Documents Added");
+        let delete_documents = std::mem::take(&mut self.delete_documents_aggregator)
+            .into_event(&self.user, "Documents Deleted");
+        let update_documents = std::mem::take(&mut self.update_documents_aggregator)
+            .into_event(&self.user, "Documents Updated");
+        let get_fetch_documents = std::mem::take(&mut self.get_fetch_documents_aggregator)
+            .into_event(&self.user, "Documents Fetched GET");
+        let post_fetch_documents = std::mem::take(&mut self.post_fetch_documents_aggregator)
+            .into_event(&self.user, "Documents Fetched POST");
+        let get_tasks =
+            std::mem::take(&mut self.get_tasks_aggregator).into_event(&self.user, "Tasks Seen");
+        let health =
+            std::mem::take(&mut self.health_aggregator).into_event(&self.user, "Health Seen");
 
-        let Segment {
-            inbox: _,
-            opt: _,
-            batcher: _,
-            user,
-            get_search_aggregator,
-            post_search_aggregator,
-            post_multi_search_aggregator,
-            post_facet_search_aggregator,
-            add_documents_aggregator,
-            delete_documents_aggregator,
-            update_documents_aggregator,
-            get_fetch_documents_aggregator,
-            post_fetch_documents_aggregator,
-            get_tasks_aggregator,
-            health_aggregator,
-        } = self;
-
-        if let Some(get_search) =
-            take(get_search_aggregator).into_event(&user, "Documents Searched GET")
-        {
+        if let Some(get_search) = get_search {
             let _ = self.batcher.push(get_search).await;
         }
-        if let Some(post_search) =
-            take(post_search_aggregator).into_event(&user, "Documents Searched POST")
-        {
+        if let Some(post_search) = post_search {
             let _ = self.batcher.push(post_search).await;
         }
-        if let Some(post_multi_search) = take(post_multi_search_aggregator)
-            .into_event(&user, "Documents Searched by Multi-Search POST")
-        {
+        if let Some(post_multi_search) = post_multi_search {
             let _ = self.batcher.push(post_multi_search).await;
         }
-        if let Some(post_facet_search) =
-            take(post_facet_search_aggregator).into_event(&user, "Facet Searched POST")
-        {
-            let _ = self.batcher.push(post_facet_search).await;
-        }
-        if let Some(add_documents) =
-            take(add_documents_aggregator).into_event(&user, "Documents Added")
-        {
+        if let Some(add_documents) = add_documents {
             let _ = self.batcher.push(add_documents).await;
         }
-        if let Some(delete_documents) =
-            take(delete_documents_aggregator).into_event(&user, "Documents Deleted")
-        {
+        if let Some(delete_documents) = delete_documents {
             let _ = self.batcher.push(delete_documents).await;
         }
-        if let Some(update_documents) =
-            take(update_documents_aggregator).into_event(&user, "Documents Updated")
-        {
+        if let Some(update_documents) = update_documents {
             let _ = self.batcher.push(update_documents).await;
         }
-        if let Some(get_fetch_documents) =
-            take(get_fetch_documents_aggregator).into_event(&user, "Documents Fetched GET")
-        {
+        if let Some(get_fetch_documents) = get_fetch_documents {
             let _ = self.batcher.push(get_fetch_documents).await;
         }
-        if let Some(post_fetch_documents) =
-            take(post_fetch_documents_aggregator).into_event(&user, "Documents Fetched POST")
-        {
+        if let Some(post_fetch_documents) = post_fetch_documents {
             let _ = self.batcher.push(post_fetch_documents).await;
         }
-        if let Some(get_tasks) = take(get_tasks_aggregator).into_event(&user, "Tasks Seen") {
+        if let Some(get_tasks) = get_tasks {
             let _ = self.batcher.push(get_tasks).await;
         }
-        if let Some(health) = take(health_aggregator).into_event(&user, "Health Seen") {
+        if let Some(health) = health {
             let _ = self.batcher.push(health).await;
         }
         let _ = self.batcher.flush().await;
@@ -578,10 +548,6 @@ pub struct SearchAggregator {
     // The maximum number of terms in a q request
     max_terms_number: usize,
 
-    // vector
-    // The maximum number of floats in a vector request
-    max_vector_size: usize,
-
     // every time a search is done, we increment the counter linked to the used settings
     matching_strategy: HashMap<String, usize>,
 
@@ -603,10 +569,6 @@ pub struct SearchAggregator {
     // facets
     facets_sum_of_terms: usize,
     facets_total_number_of_facets: usize,
-
-    // scoring
-    show_ranking_score: bool,
-    show_ranking_score_details: bool,
 }
 
 impl SearchAggregator {
@@ -651,10 +613,6 @@ impl SearchAggregator {
             ret.max_terms_number = q.split_whitespace().count();
         }
 
-        if let Some(ref vector) = query.vector {
-            ret.max_vector_size = vector.len();
-        }
-
         if query.is_finite_pagination() {
             let limit = query.hits_per_page.unwrap_or_else(DEFAULT_SEARCH_LIMIT);
             ret.max_limit = limit;
@@ -674,9 +632,6 @@ impl SearchAggregator {
         ret.crop_length = query.crop_length != DEFAULT_CROP_LENGTH();
         ret.show_matches_position = query.show_matches_position;
 
-        ret.show_ranking_score = query.show_ranking_score;
-        ret.show_ranking_score_details = query.show_ranking_score_details;
-
         ret
     }
 
@@ -751,10 +706,6 @@ impl SearchAggregator {
             let matching_strategy = self.matching_strategy.entry(key).or_insert(0);
             *matching_strategy = matching_strategy.saturating_add(value);
         }
-
-        // scoring
-        self.show_ranking_score |= other.show_ranking_score;
-        self.show_ranking_score_details |= other.show_ranking_score_details;
     }
 
     pub fn into_event(self, user: &User, event_name: &str) -> Option<Track> {
@@ -809,11 +760,7 @@ impl SearchAggregator {
                 },
                 "matching_strategy": {
                     "most_used_strategy": self.matching_strategy.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)),
-                },
-                "scoring": {
-                    "show_ranking_score": self.show_ranking_score,
-                    "show_ranking_score_details": self.show_ranking_score_details,
-                },
+                }
             });
 
             Some(Track {
@@ -939,120 +886,6 @@ impl MultiSearchAggregator {
     }
 }
 
-#[derive(Default)]
-pub struct FacetSearchAggregator {
-    timestamp: Option<OffsetDateTime>,
-
-    // context
-    user_agents: HashSet<String>,
-
-    // requests
-    total_received: usize,
-    total_succeeded: usize,
-    time_spent: BinaryHeap<usize>,
-
-    // The set of all facetNames that were used
-    facet_names: HashSet<String>,
-
-    // As there been any other parameter than the facetName or facetQuery ones?
-    additional_search_parameters_provided: bool,
-}
-
-impl FacetSearchAggregator {
-    pub fn from_query(query: &FacetSearchQuery, request: &HttpRequest) -> Self {
-        let FacetSearchQuery {
-            facet_query: _,
-            facet_name,
-            vector,
-            q,
-            filter,
-            matching_strategy,
-            attributes_to_search_on,
-        } = query;
-
-        let mut ret = Self::default();
-        ret.timestamp = Some(OffsetDateTime::now_utc());
-
-        ret.total_received = 1;
-        ret.user_agents = extract_user_agents(request).into_iter().collect();
-        ret.facet_names = Some(facet_name.clone()).into_iter().collect();
-
-        ret.additional_search_parameters_provided = q.is_some()
-            || vector.is_some()
-            || filter.is_some()
-            || *matching_strategy != MatchingStrategy::default()
-            || attributes_to_search_on.is_some();
-
-        ret
-    }
-
-    pub fn succeed(&mut self, result: &FacetSearchResult) {
-        self.total_succeeded = self.total_succeeded.saturating_add(1);
-        self.time_spent.push(result.processing_time_ms as usize);
-    }
-
-    /// Aggregate one [SearchAggregator] into another.
-    pub fn aggregate(&mut self, mut other: Self) {
-        if self.timestamp.is_none() {
-            self.timestamp = other.timestamp;
-        }
-
-        // context
-        for user_agent in other.user_agents.into_iter() {
-            self.user_agents.insert(user_agent);
-        }
-
-        // request
-        self.total_received = self.total_received.saturating_add(other.total_received);
-        self.total_succeeded = self.total_succeeded.saturating_add(other.total_succeeded);
-        self.time_spent.append(&mut other.time_spent);
-
-        // facet_names
-        for facet_name in other.facet_names.into_iter() {
-            self.facet_names.insert(facet_name);
-        }
-
-        // additional_search_parameters_provided
-        self.additional_search_parameters_provided = self.additional_search_parameters_provided
-            | other.additional_search_parameters_provided;
-    }
-
-    pub fn into_event(self, user: &User, event_name: &str) -> Option<Track> {
-        if self.total_received == 0 {
-            None
-        } else {
-            // the index of the 99th percentage of value
-            let percentile_99th = 0.99 * (self.total_succeeded as f64 - 1.) + 1.;
-            // we get all the values in a sorted manner
-            let time_spent = self.time_spent.into_sorted_vec();
-            // We are only interested by the slowest value of the 99th fastest results
-            let time_spent = time_spent.get(percentile_99th as usize);
-
-            let properties = json!({
-                "user-agent": self.user_agents,
-                "requests": {
-                    "99th_response_time":  time_spent.map(|t| format!("{:.2}", t)),
-                    "total_succeeded": self.total_succeeded,
-                    "total_failed": self.total_received.saturating_sub(self.total_succeeded), // just to be sure we never panics
-                    "total_received": self.total_received,
-                },
-                "facets": {
-                    "total_distinct_facet_count": self.facet_names.len(),
-                    "additional_search_parameters_provided": self.additional_search_parameters_provided,
-                },
-            });
-
-            Some(Track {
-                timestamp: self.timestamp,
-                user: user.clone(),
-                event: event_name.to_string(),
-                properties,
-                ..Default::default()
-            })
-        }
-    }
-}
-
 #[derive(Default)]
 pub struct DocumentsAggregator {
     timestamp: Option<OffsetDateTime>,

meilisearch/src/extractors/payload.rs

@@ -71,40 +71,3 @@ impl Stream for Payload {
         }
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use actix_http::encoding::Decoder as Decompress;
-    use actix_http::BoxedPayloadStream;
-    use bytes::Bytes;
-    use futures_util::StreamExt;
-    use meili_snap::snapshot;
-
-    use super::*;
-
-    #[actix_rt::test]
-    async fn payload_to_large() {
-        let stream = futures::stream::iter(vec![
-            Ok(Bytes::from("1")),
-            Ok(Bytes::from("2")),
-            Ok(Bytes::from("3")),
-            Ok(Bytes::from("4")),
-        ]);
-        let boxed_stream: BoxedPayloadStream = Box::pin(stream);
-        let actix_payload = dev::Payload::from(boxed_stream);
-
-        let payload = Payload {
-            limit: 3,
-            remaining: 3,
-            payload: Decompress::new(actix_payload, actix_http::ContentEncoding::Identity),
-        };
-
-        let mut enumerated_payload_stream = payload.enumerate();
-
-        while let Some((idx, chunk)) = enumerated_payload_stream.next().await {
-            if idx == 3 {
-                snapshot!(chunk.unwrap_err(), @"The provided payload reached the size limit. The maximum accepted payload size is 3 B.");
-            }
-        }
-    }
-}

meilisearch/src/lib.rs

@@ -111,7 +111,7 @@ pub fn create_app(
                 analytics.clone(),
             )
         })
-        .configure(routes::configure)
+        .configure(|cfg| routes::configure(cfg, opt.experimental_enable_metrics))
         .configure(|s| dashboard(s, enable_dashboard));
 
     let app = app.wrap(actix_web::middleware::Condition::new(
@@ -221,7 +221,6 @@ fn open_or_create_database_unchecked(
     // we don't want to create anything in the data.ms yet, thus we
     // wrap our two builders in a closure that'll be executed later.
     let auth_controller = AuthController::new(&opt.db_path, &opt.master_key);
-    let instance_features = opt.to_instance_features();
     let index_scheduler_builder = || -> anyhow::Result<_> {
         Ok(IndexScheduler::new(IndexSchedulerOptions {
             version_file_path: opt.db_path.join(VERSION_FILE_NAME),
@@ -239,7 +238,6 @@ fn open_or_create_database_unchecked(
             max_number_of_tasks: 1_000_000,
             index_growth_amount: byte_unit::Byte::from_str("10GiB").unwrap().get_bytes() as usize,
             index_count: DEFAULT_INDEX_COUNT,
-            instance_features,
         })?)
     };
 
@@ -309,16 +307,12 @@ fn import_dump(
         keys.push(key);
     }
 
-    // 3. Import the runtime features.
-    let features = dump_reader.features()?.unwrap_or_default();
-    index_scheduler.put_runtime_features(features)?;
-
     let indexer_config = index_scheduler.indexer_config();
 
     // /!\ The tasks must be imported AFTER importing the indexes or else the scheduler might
     // try to process tasks while we're trying to import the indexes.
 
-    // 4. Import the indexes.
+    // 3. Import the indexes.
     for index_reader in dump_reader.indexes()? {
         let mut index_reader = index_reader?;
         let metadata = index_reader.metadata();
@@ -330,19 +324,19 @@ fn import_dump(
         let mut wtxn = index.write_txn()?;
 
         let mut builder = milli::update::Settings::new(&mut wtxn, &index, indexer_config);
-        // 4.1 Import the primary key if there is one.
+        // 3.1 Import the primary key if there is one.
         if let Some(ref primary_key) = metadata.primary_key {
             builder.set_primary_key(primary_key.to_string());
         }
 
-        // 4.2 Import the settings.
+        // 3.2 Import the settings.
         log::info!("Importing the settings.");
         let settings = index_reader.settings()?;
         apply_settings_to_builder(&settings, &mut builder);
         builder.execute(|indexing_step| log::debug!("update: {:?}", indexing_step), || false)?;
 
-        // 4.3 Import the documents.
-        // 4.3.1 We need to recreate the grenad+obkv format accepted by the index.
+        // 3.3 Import the documents.
+        // 3.3.1 We need to recreate the grenad+obkv format accepted by the index.
         log::info!("Importing the documents.");
         let file = tempfile::tempfile()?;
         let mut builder = DocumentsBatchBuilder::new(BufWriter::new(file));
@@ -353,7 +347,7 @@ fn import_dump(
         // This flush the content of the batch builder.
         let file = builder.into_inner()?.into_inner()?;
 
-        // 4.3.2 We feed it to the milli index.
+        // 3.3.2 We feed it to the milli index.
         let reader = BufReader::new(file);
         let reader = DocumentsBatchReader::from_reader(reader)?;
 
@@ -378,7 +372,7 @@ fn import_dump(
 
     let mut index_scheduler_dump = index_scheduler.register_dumped_task()?;
 
-    // 5. Import the tasks.
+    // 4. Import the tasks.
     for ret in dump_reader.tasks()? {
         let (task, file) = ret?;
         index_scheduler_dump.register_dumped_task(task, file)?;

meilisearch/src/main.rs

@@ -29,10 +29,6 @@ fn setup(opt: &Opt) -> anyhow::Result<()> {
 async fn main() -> anyhow::Result<()> {
     let (opt, config_read_from) = Opt::try_build()?;
 
-    #[cfg(feature = "profile-with-puffin")]
-    let _server = puffin_http::Server::new(&format!("0.0.0.0:{}", puffin_http::DEFAULT_PORT))?;
-    puffin::set_scopes_on(cfg!(feature = "profile-with-puffin"));
-
     anyhow::ensure!(
         !(cfg!(windows) && opt.experimental_reduce_indexing_memory_usage),
         "The `experimental-reduce-indexing-memory-usage` flag is not supported on Windows"
@@ -190,10 +186,9 @@ Anonymous telemetry:\t\"Enabled\""
     }
 
     eprintln!();
-    eprintln!("Check out Meilisearch Cloud!\thttps://cloud.meilisearch.com/login?utm_campaign=oss&utm_source=engine&utm_medium=cli");
-    eprintln!("Documentation:\t\t\thttps://www.meilisearch.com/docs");
-    eprintln!("Source code:\t\t\thttps://github.com/meilisearch/meilisearch");
-    eprintln!("Discord:\t\t\thttps://discord.meilisearch.com");
+    eprintln!("Documentation:\t\thttps://www.meilisearch.com/docs");
+    eprintln!("Source code:\t\thttps://github.com/meilisearch/meilisearch");
+    eprintln!("Discord:\t\thttps://discord.meilisearch.com");
     eprintln!();
 }
 

meilisearch/src/metrics.rs

@@ -16,7 +16,7 @@ fn create_buckets() -> [f64; 29] {
 }
 
 lazy_static! {
-    pub static ref MEILISEARCH_HTTP_RESPONSE_TIME_CUSTOM_BUCKETS: [f64; 29] = create_buckets();
+    pub static ref HTTP_RESPONSE_TIME_CUSTOM_BUCKETS: [f64; 29] = create_buckets();
     pub static ref MEILISEARCH_HTTP_REQUESTS_TOTAL: IntCounterVec = register_int_counter_vec!(
         opts!("meilisearch_http_requests_total", "Meilisearch HTTP requests total"),
         &["method", "path"]
@@ -39,10 +39,10 @@ lazy_static! {
     )
     .expect("Can't create a metric");
     pub static ref MEILISEARCH_HTTP_RESPONSE_TIME_SECONDS: HistogramVec = register_histogram_vec!(
-        "meilisearch_http_response_time_seconds",
-        "Meilisearch HTTP response times",
+        "http_response_time_seconds",
+        "HTTP response times",
         &["method", "path"],
-        MEILISEARCH_HTTP_RESPONSE_TIME_CUSTOM_BUCKETS.to_vec()
+        HTTP_RESPONSE_TIME_CUSTOM_BUCKETS.to_vec()
     )
     .expect("Can't create a metric");
     pub static ref MEILISEARCH_NB_TASKS: IntGaugeVec = register_int_gauge_vec!(

meilisearch/src/option.rs

@@ -12,7 +12,6 @@ use std::{env, fmt, fs};
 
 use byte_unit::{Byte, ByteError};
 use clap::Parser;
-use meilisearch_types::features::InstanceTogglableFeatures;
 use meilisearch_types::milli::update::IndexerConfig;
 use rustls::server::{
     AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, ServerSessionMemoryCache,
@@ -487,10 +486,6 @@ impl Opt {
             Ok(None)
         }
     }
-
-    pub(crate) fn to_instance_features(&self) -> InstanceTogglableFeatures {
-        InstanceTogglableFeatures { metrics: self.experimental_enable_metrics }
-    }
 }
 
 #[derive(Debug, Default, Clone, Parser, Deserialize)]

meilisearch/src/routes/features.rs

@@ -1,70 +0,0 @@
-use actix_web::web::{self, Data};
-use actix_web::{HttpRequest, HttpResponse};
-use deserr::actix_web::AwebJson;
-use deserr::Deserr;
-use index_scheduler::IndexScheduler;
-use log::debug;
-use meilisearch_types::deserr::DeserrJsonError;
-use meilisearch_types::error::ResponseError;
-use meilisearch_types::keys::actions;
-use serde_json::json;
-
-use crate::analytics::Analytics;
-use crate::extractors::authentication::policies::ActionPolicy;
-use crate::extractors::authentication::GuardedData;
-use crate::extractors::sequential_extractor::SeqHandler;
-
-pub fn configure(cfg: &mut web::ServiceConfig) {
-    cfg.service(
-        web::resource("")
-            .route(web::get().to(SeqHandler(get_features)))
-            .route(web::patch().to(SeqHandler(patch_features))),
-    );
-}
-
-async fn get_features(
-    index_scheduler: GuardedData<
-        ActionPolicy<{ actions::EXPERIMENTAL_FEATURES_GET }>,
-        Data<IndexScheduler>,
-    >,
-    req: HttpRequest,
-    analytics: Data<dyn Analytics>,
-) -> Result<HttpResponse, ResponseError> {
-    let features = index_scheduler.features()?;
-
-    analytics.publish("Experimental features Seen".to_string(), json!(null), Some(&req));
-    debug!("returns: {:?}", features.runtime_features());
-    Ok(HttpResponse::Ok().json(features.runtime_features()))
-}
-
-#[derive(Debug, Deserr)]
-#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
-pub struct RuntimeTogglableFeatures {
-    #[deserr(default)]
-    pub score_details: Option<bool>,
-    #[deserr(default)]
-    pub vector_store: Option<bool>,
-}
-
-async fn patch_features(
-    index_scheduler: GuardedData<
-        ActionPolicy<{ actions::EXPERIMENTAL_FEATURES_UPDATE }>,
-        Data<IndexScheduler>,
-    >,
-    new_features: AwebJson<RuntimeTogglableFeatures, DeserrJsonError>,
-    req: HttpRequest,
-    analytics: Data<dyn Analytics>,
-) -> Result<HttpResponse, ResponseError> {
-    let features = index_scheduler.features()?;
-
-    let old_features = features.runtime_features();
-
-    let new_features = meilisearch_types::features::RuntimeTogglableFeatures {
-        score_details: new_features.0.score_details.unwrap_or(old_features.score_details),
-        vector_store: new_features.0.vector_store.unwrap_or(old_features.vector_store),
-    };
-
-    analytics.publish("Experimental features Updated".to_string(), json!(new_features), Some(&req));
-    index_scheduler.put_runtime_features(new_features)?;
-    Ok(HttpResponse::Ok().json(new_features))
-}

meilisearch/src/routes/indexes/facet_search.rs

@@ -1,124 +0,0 @@
-use actix_web::web::Data;
-use actix_web::{web, HttpRequest, HttpResponse};
-use deserr::actix_web::AwebJson;
-use index_scheduler::IndexScheduler;
-use log::debug;
-use meilisearch_types::deserr::DeserrJsonError;
-use meilisearch_types::error::deserr_codes::*;
-use meilisearch_types::error::ResponseError;
-use meilisearch_types::index_uid::IndexUid;
-use serde_json::Value;
-
-use crate::analytics::{Analytics, FacetSearchAggregator};
-use crate::extractors::authentication::policies::*;
-use crate::extractors::authentication::GuardedData;
-use crate::search::{
-    add_search_rules, perform_facet_search, MatchingStrategy, SearchQuery, DEFAULT_CROP_LENGTH,
-    DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG,
-    DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET,
-};
-
-pub fn configure(cfg: &mut web::ServiceConfig) {
-    cfg.service(web::resource("").route(web::post().to(search)));
-}
-
-/// # Important
-///
-/// Intentionally don't use `deny_unknown_fields` to ignore search parameters sent by user
-#[derive(Debug, Clone, Default, PartialEq, deserr::Deserr)]
-#[deserr(error = DeserrJsonError, rename_all = camelCase)]
-pub struct FacetSearchQuery {
-    #[deserr(default, error = DeserrJsonError<InvalidFacetSearchQuery>)]
-    pub facet_query: Option<String>,
-    #[deserr(error = DeserrJsonError<InvalidFacetSearchFacetName>, missing_field_error = DeserrJsonError::missing_facet_search_facet_name)]
-    pub facet_name: String,
-    #[deserr(default, error = DeserrJsonError<InvalidSearchQ>)]
-    pub q: Option<String>,
-    #[deserr(default, error = DeserrJsonError<InvalidSearchVector>)]
-    pub vector: Option<Vec<f32>>,
-    #[deserr(default, error = DeserrJsonError<InvalidSearchFilter>)]
-    pub filter: Option<Value>,
-    #[deserr(default, error = DeserrJsonError<InvalidSearchMatchingStrategy>, default)]
-    pub matching_strategy: MatchingStrategy,
-    #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToSearchOn>, default)]
-    pub attributes_to_search_on: Option<Vec<String>>,
-}
-
-pub async fn search(
-    index_scheduler: GuardedData<ActionPolicy<{ actions::SEARCH }>, Data<IndexScheduler>>,
-    index_uid: web::Path<String>,
-    params: AwebJson<FacetSearchQuery, DeserrJsonError>,
-    req: HttpRequest,
-    analytics: web::Data<dyn Analytics>,
-) -> Result<HttpResponse, ResponseError> {
-    let index_uid = IndexUid::try_from(index_uid.into_inner())?;
-
-    let query = params.into_inner();
-    debug!("facet search called with params: {:?}", query);
-
-    let mut aggregate = FacetSearchAggregator::from_query(&query, &req);
-
-    let facet_query = query.facet_query.clone();
-    let facet_name = query.facet_name.clone();
-    let mut search_query = SearchQuery::from(query);
-
-    // Tenant token search_rules.
-    if let Some(search_rules) = index_scheduler.filters().get_index_search_rules(&index_uid) {
-        add_search_rules(&mut search_query, search_rules);
-    }
-
-    let index = index_scheduler.index(&index_uid)?;
-    let features = index_scheduler.features()?;
-    let search_result = tokio::task::spawn_blocking(move || {
-        perform_facet_search(&index, search_query, facet_query, facet_name, features)
-    })
-    .await?;
-
-    if let Ok(ref search_result) = search_result {
-        aggregate.succeed(search_result);
-    }
-    analytics.post_facet_search(aggregate);
-
-    let search_result = search_result?;
-
-    debug!("returns: {:?}", search_result);
-    Ok(HttpResponse::Ok().json(search_result))
-}
-
-impl From<FacetSearchQuery> for SearchQuery {
-    fn from(value: FacetSearchQuery) -> Self {
-        let FacetSearchQuery {
-            facet_query: _,
-            facet_name: _,
-            q,
-            vector,
-            filter,
-            matching_strategy,
-            attributes_to_search_on,
-        } = value;
-
-        SearchQuery {
-            q,
-            offset: DEFAULT_SEARCH_OFFSET(),
-            limit: DEFAULT_SEARCH_LIMIT(),
-            page: None,
-            hits_per_page: None,
-            attributes_to_retrieve: None,
-            attributes_to_crop: None,
-            crop_length: DEFAULT_CROP_LENGTH(),
-            attributes_to_highlight: None,
-            show_matches_position: false,
-            show_ranking_score: false,
-            show_ranking_score_details: false,
-            filter,
-            sort: None,
-            facets: None,
-            highlight_pre_tag: DEFAULT_HIGHLIGHT_PRE_TAG(),
-            highlight_post_tag: DEFAULT_HIGHLIGHT_POST_TAG(),
-            crop_marker: DEFAULT_CROP_MARKER(),
-            matching_strategy,
-            vector,
-            attributes_to_search_on,
-        }
-    }
-}

meilisearch/src/routes/indexes/mod.rs

@@ -24,7 +24,6 @@ use crate::extractors::authentication::{AuthenticationError, GuardedData};
 use crate::extractors::sequential_extractor::SeqHandler;
 
 pub mod documents;
-pub mod facet_search;
 pub mod search;
 pub mod settings;
 
@@ -45,7 +44,6 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
             .service(web::resource("/stats").route(web::get().to(SeqHandler(get_index_stats))))
             .service(web::scope("/documents").configure(documents::configure))
             .service(web::scope("/search").configure(search::configure))
-            .service(web::scope("/facet-search").configure(facet_search::configure))
             .service(web::scope("/settings").configure(settings::configure)),
     );
 }

meilisearch/src/routes/indexes/search.rs

@@ -34,8 +34,6 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
 pub struct SearchQueryGet {
     #[deserr(default, error = DeserrQueryParamError<InvalidSearchQ>)]
     q: Option<String>,
-    #[deserr(default, error = DeserrQueryParamError<InvalidSearchVector>)]
-    vector: Option<Vec<f32>>,
     #[deserr(default = Param(DEFAULT_SEARCH_OFFSET()), error = DeserrQueryParamError<InvalidSearchOffset>)]
     offset: Param<usize>,
     #[deserr(default = Param(DEFAULT_SEARCH_LIMIT()), error = DeserrQueryParamError<InvalidSearchLimit>)]
@@ -72,8 +70,6 @@ pub struct SearchQueryGet {
     crop_marker: String,
     #[deserr(default, error = DeserrQueryParamError<InvalidSearchMatchingStrategy>)]
     matching_strategy: MatchingStrategy,
-    #[deserr(default, error = DeserrQueryParamError<InvalidSearchAttributesToSearchOn>)]
-    pub attributes_to_search_on: Option<CS<String>>,
 }
 
 impl From<SearchQueryGet> for SearchQuery {
@@ -88,7 +84,6 @@ impl From<SearchQueryGet> for SearchQuery {
 
         Self {
             q: other.q,
-            vector: other.vector,
             offset: other.offset.0,
             limit: other.limit.0,
             page: other.page.as_deref().copied(),
@@ -107,7 +102,6 @@ impl From<SearchQueryGet> for SearchQuery {
             highlight_post_tag: other.highlight_post_tag,
             crop_marker: other.crop_marker,
             matching_strategy: other.matching_strategy,
-            attributes_to_search_on: other.attributes_to_search_on.map(|o| o.into_iter().collect()),
         }
     }
 }
@@ -157,9 +151,7 @@ pub async fn search_with_url_query(
     let mut aggregate = SearchAggregator::from_query(&query, &req);
 
     let index = index_scheduler.index(&index_uid)?;
-    let features = index_scheduler.features()?;
-    let search_result =
-        tokio::task::spawn_blocking(move || perform_search(&index, query, features)).await?;
+    let search_result = tokio::task::spawn_blocking(move || perform_search(&index, query)).await?;
     if let Ok(ref search_result) = search_result {
         aggregate.succeed(search_result);
     }
@@ -191,10 +183,7 @@ pub async fn search_with_post(
     let mut aggregate = SearchAggregator::from_query(&query, &req);
 
     let index = index_scheduler.index(&index_uid)?;
-
-    let features = index_scheduler.features()?;
-    let search_result =
-        tokio::task::spawn_blocking(move || perform_search(&index, query, features)).await?;
+    let search_result = tokio::task::spawn_blocking(move || perform_search(&index, query)).await?;
     if let Ok(ref search_result) = search_result {
         aggregate.succeed(search_result);
     }

meilisearch/src/routes/indexes/settings.rs

@@ -309,81 +309,6 @@ make_setting_route!(
     }
 );
 
-make_setting_route!(
-    "/non-separator-tokens",
-    put,
-    std::collections::BTreeSet<String>,
-    meilisearch_types::deserr::DeserrJsonError<
-        meilisearch_types::error::deserr_codes::InvalidSettingsNonSeparatorTokens,
-    >,
-    non_separator_tokens,
-    "nonSeparatorTokens",
-    analytics,
-    |non_separator_tokens: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
-        use serde_json::json;
-
-        analytics.publish(
-            "nonSeparatorTokens Updated".to_string(),
-            json!({
-                "non_separator_tokens": {
-                    "total": non_separator_tokens.as_ref().map(|non_separator_tokens| non_separator_tokens.len()),
-                },
-            }),
-            Some(req),
-        );
-    }
-);
-
-make_setting_route!(
-    "/separator-tokens",
-    put,
-    std::collections::BTreeSet<String>,
-    meilisearch_types::deserr::DeserrJsonError<
-        meilisearch_types::error::deserr_codes::InvalidSettingsSeparatorTokens,
-    >,
-    separator_tokens,
-    "separatorTokens",
-    analytics,
-    |separator_tokens: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
-        use serde_json::json;
-
-        analytics.publish(
-            "separatorTokens Updated".to_string(),
-            json!({
-                "separator_tokens": {
-                    "total": separator_tokens.as_ref().map(|separator_tokens| separator_tokens.len()),
-                },
-            }),
-            Some(req),
-        );
-    }
-);
-
-make_setting_route!(
-    "/dictionary",
-    put,
-    std::collections::BTreeSet<String>,
-    meilisearch_types::deserr::DeserrJsonError<
-        meilisearch_types::error::deserr_codes::InvalidSettingsDictionary,
-    >,
-    dictionary,
-    "dictionary",
-    analytics,
-    |dictionary: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
-        use serde_json::json;
-
-        analytics.publish(
-            "dictionary Updated".to_string(),
-            json!({
-                "dictionary": {
-                    "total": dictionary.as_ref().map(|dictionary| dictionary.len()),
-                },
-            }),
-            Some(req),
-        );
-    }
-);
-
 make_setting_route!(
     "/synonyms",
     put,
@@ -476,17 +401,12 @@ make_setting_route!(
     analytics,
     |setting: &Option<meilisearch_types::settings::FacetingSettings>, req: &HttpRequest| {
         use serde_json::json;
-        use meilisearch_types::facet_values_sort::FacetValuesSort;
 
         analytics.publish(
             "Faceting Updated".to_string(),
             json!({
                 "faceting": {
                     "max_values_per_facet": setting.as_ref().and_then(|s| s.max_values_per_facet.set()),
-                    "sort_facet_values_by_star_count": setting.as_ref().and_then(|s| {
-                        s.sort_facet_values_by.as_ref().set().map(|s| s.iter().any(|(k, v)| k == "*" && v == &FacetValuesSort::Count))
-                    }),
-                    "sort_facet_values_by_total": setting.as_ref().and_then(|s| s.sort_facet_values_by.as_ref().set().map(|s| s.len())),
                 },
             }),
             Some(req),
@@ -625,10 +545,6 @@ pub async fn update_all(
                     .as_ref()
                     .set()
                     .and_then(|s| s.max_values_per_facet.as_ref().set()),
-                "sort_facet_values_by": new_settings.faceting
-                    .as_ref()
-                    .set()
-                    .and_then(|s| s.sort_facet_values_by.as_ref().set()),
             },
             "pagination": {
                 "max_total_hits": new_settings.pagination

meilisearch/src/routes/metrics.rs

@@ -19,7 +19,6 @@ pub async fn get_metrics(
     index_scheduler: GuardedData<ActionPolicy<{ actions::METRICS_GET }>, Data<IndexScheduler>>,
     auth_controller: Data<AuthController>,
 ) -> Result<HttpResponse, ResponseError> {
-    index_scheduler.features()?.check_metrics()?;
     let auth_filters = index_scheduler.filters();
     if !auth_filters.all_indexes_authorized() {
         let mut error = ResponseError::from(AuthenticationError::InvalidToken);

meilisearch/src/routes/mod.rs

@@ -20,14 +20,13 @@ const PAGINATION_DEFAULT_LIMIT: usize = 20;
 
 mod api_key;
 mod dump;
-pub mod features;
 pub mod indexes;
 mod metrics;
 mod multi_search;
 mod swap_indexes;
 pub mod tasks;
 
-pub fn configure(cfg: &mut web::ServiceConfig) {
+pub fn configure(cfg: &mut web::ServiceConfig, enable_metrics: bool) {
     cfg.service(web::scope("/tasks").configure(tasks::configure))
         .service(web::resource("/health").route(web::get().to(get_health)))
         .service(web::scope("/keys").configure(api_key::configure))
@@ -36,9 +35,11 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
         .service(web::resource("/version").route(web::get().to(get_version)))
         .service(web::scope("/indexes").configure(indexes::configure))
         .service(web::scope("/multi-search").configure(multi_search::configure))
-        .service(web::scope("/swap-indexes").configure(swap_indexes::configure))
-        .service(web::scope("/metrics").configure(metrics::configure))
-        .service(web::scope("/experimental-features").configure(features::configure));
+        .service(web::scope("/swap-indexes").configure(swap_indexes::configure));
+
+    if enable_metrics {
+        cfg.service(web::scope("/metrics").configure(metrics::configure));
+    }
 }
 
 #[derive(Debug, Serialize)]

meilisearch/src/routes/multi_search.rs

@@ -41,7 +41,6 @@ pub async fn multi_search_with_post(
     let queries = params.into_inner().queries;
 
     let mut multi_aggregate = MultiSearchAggregator::from_queries(&queries, &req);
-    let features = index_scheduler.features()?;
 
     // Explicitly expect a `(ResponseError, usize)` for the error type rather than `ResponseError` only,
     // so that `?` doesn't work if it doesn't use `with_index`, ensuring that it is not forgotten in case of code
@@ -75,9 +74,8 @@ pub async fn multi_search_with_post(
                         err
                     })
                     .with_index(query_index)?;
-
                 let search_result =
-                    tokio::task::spawn_blocking(move || perform_search(&index, query, features))
+                    tokio::task::spawn_blocking(move || perform_search(&index, query))
                         .await
                         .with_index(query_index)?;
 

meilisearch/src/search.rs

@@ -5,26 +5,18 @@ use std::time::Instant;
 
 use deserr::Deserr;
 use either::Either;
-use index_scheduler::RoFeatures;
-use indexmap::IndexMap;
-use log::warn;
 use meilisearch_auth::IndexSearchRules;
 use meilisearch_types::deserr::DeserrJsonError;
 use meilisearch_types::error::deserr_codes::*;
-use meilisearch_types::heed::RoTxn;
 use meilisearch_types::index_uid::IndexUid;
-use meilisearch_types::milli::score_details::{ScoreDetails, ScoringStrategy};
-use meilisearch_types::milli::{
-    dot_product_similarity, FacetValueHit, InternalError, OrderBy, SearchForFacetValues,
-};
+use meilisearch_types::milli::score_details::ScoreDetails;
 use meilisearch_types::settings::DEFAULT_PAGINATION_MAX_TOTAL_HITS;
 use meilisearch_types::{milli, Document};
 use milli::tokenizer::TokenizerBuilder;
 use milli::{
     AscDesc, FieldId, FieldsIdsMap, Filter, FormatOptions, Index, MatchBounds, MatcherBuilder,
-    SortError, TermsMatchingStrategy, VectorOrArrayOfVectors, DEFAULT_VALUES_PER_FACET,
+    SortError, TermsMatchingStrategy, DEFAULT_VALUES_PER_FACET,
 };
-use ordered_float::OrderedFloat;
 use regex::Regex;
 use serde::Serialize;
 use serde_json::{json, Value};
@@ -40,13 +32,11 @@ pub const DEFAULT_CROP_MARKER: fn() -> String = || "…".to_string();
 pub const DEFAULT_HIGHLIGHT_PRE_TAG: fn() -> String = || "<em>".to_string();
 pub const DEFAULT_HIGHLIGHT_POST_TAG: fn() -> String = || "</em>".to_string();
 
-#[derive(Debug, Clone, Default, PartialEq, Deserr)]
+#[derive(Debug, Clone, Default, PartialEq, Eq, Deserr)]
 #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
 pub struct SearchQuery {
     #[deserr(default, error = DeserrJsonError<InvalidSearchQ>)]
     pub q: Option<String>,
-    #[deserr(default, error = DeserrJsonError<InvalidSearchVector>)]
-    pub vector: Option<Vec<f32>>,
     #[deserr(default = DEFAULT_SEARCH_OFFSET(), error = DeserrJsonError<InvalidSearchOffset>)]
     pub offset: usize,
     #[deserr(default = DEFAULT_SEARCH_LIMIT(), error = DeserrJsonError<InvalidSearchLimit>)]
@@ -83,8 +73,6 @@ pub struct SearchQuery {
     pub crop_marker: String,
     #[deserr(default, error = DeserrJsonError<InvalidSearchMatchingStrategy>, default)]
     pub matching_strategy: MatchingStrategy,
-    #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToSearchOn>, default)]
-    pub attributes_to_search_on: Option<Vec<String>>,
 }
 
 impl SearchQuery {
@@ -97,15 +85,13 @@ impl SearchQuery {
 // This struct contains the fields of `SearchQuery` inline.
 // This is because neither deserr nor serde support `flatten` when using `deny_unknown_fields.
 // The `From<SearchQueryWithIndex>` implementation ensures both structs remain up to date.
-#[derive(Debug, Clone, PartialEq, Deserr)]
+#[derive(Debug, Clone, PartialEq, Eq, Deserr)]
 #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
 pub struct SearchQueryWithIndex {
     #[deserr(error = DeserrJsonError<InvalidIndexUid>, missing_field_error = DeserrJsonError::missing_index_uid)]
     pub index_uid: IndexUid,
     #[deserr(default, error = DeserrJsonError<InvalidSearchQ>)]
     pub q: Option<String>,
-    #[deserr(default, error = DeserrJsonError<InvalidSearchQ>)]
-    pub vector: Option<Vec<f32>>,
     #[deserr(default = DEFAULT_SEARCH_OFFSET(), error = DeserrJsonError<InvalidSearchOffset>)]
     pub offset: usize,
     #[deserr(default = DEFAULT_SEARCH_LIMIT(), error = DeserrJsonError<InvalidSearchLimit>)]
@@ -142,8 +128,6 @@ pub struct SearchQueryWithIndex {
     pub crop_marker: String,
     #[deserr(default, error = DeserrJsonError<InvalidSearchMatchingStrategy>, default)]
     pub matching_strategy: MatchingStrategy,
-    #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToSearchOn>, default)]
-    pub attributes_to_search_on: Option<Vec<String>>,
 }
 
 impl SearchQueryWithIndex {
@@ -151,7 +135,6 @@ impl SearchQueryWithIndex {
         let SearchQueryWithIndex {
             index_uid,
             q,
-            vector,
             offset,
             limit,
             page,
@@ -170,13 +153,11 @@ impl SearchQueryWithIndex {
             highlight_post_tag,
             crop_marker,
             matching_strategy,
-            attributes_to_search_on,
         } = self;
         (
             index_uid,
             SearchQuery {
                 q,
-                vector,
                 offset,
                 limit,
                 page,
@@ -195,7 +176,6 @@ impl SearchQueryWithIndex {
                 highlight_post_tag,
                 crop_marker,
                 matching_strategy,
-                attributes_to_search_on,
                 // do not use ..Default::default() here,
                 // rather add any missing field from `SearchQuery` to `SearchQueryWithIndex`
             },
@@ -203,7 +183,7 @@ impl SearchQueryWithIndex {
     }
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr)]
+#[derive(Debug, Clone, PartialEq, Eq, Deserr)]
 #[deserr(rename_all = camelCase)]
 pub enum MatchingStrategy {
     /// Remove query words from last to first
@@ -227,26 +207,6 @@ impl From<MatchingStrategy> for TermsMatchingStrategy {
     }
 }
 
-#[derive(Debug, Default, Clone, PartialEq, Eq, Deserr)]
-#[deserr(rename_all = camelCase)]
-pub enum FacetValuesSort {
-    /// Facet values are sorted in alphabetical order, ascending from A to Z.
-    #[default]
-    Alpha,
-    /// Facet values are sorted by decreasing count.
-    /// The count is the number of records containing this facet value in the results of the query.
-    Count,
-}
-
-impl From<FacetValuesSort> for OrderBy {
-    fn from(val: FacetValuesSort) -> Self {
-        match val {
-            FacetValuesSort::Alpha => OrderBy::Lexicographic,
-            FacetValuesSort::Count => OrderBy::Count,
-        }
-    }
-}
-
 #[derive(Debug, Clone, Serialize, PartialEq)]
 pub struct SearchHit {
     #[serde(flatten)]
@@ -256,11 +216,9 @@ pub struct SearchHit {
     #[serde(rename = "_matchesPosition", skip_serializing_if = "Option::is_none")]
     pub matches_position: Option<MatchesPosition>,
     #[serde(rename = "_rankingScore", skip_serializing_if = "Option::is_none")]
-    pub ranking_score: Option<f64>,
+    pub ranking_score: Option<u64>,
     #[serde(rename = "_rankingScoreDetails", skip_serializing_if = "Option::is_none")]
     pub ranking_score_details: Option<serde_json::Map<String, serde_json::Value>>,
-    #[serde(rename = "_semanticScore", skip_serializing_if = "Option::is_none")]
-    pub semantic_score: Option<f32>,
 }
 
 #[derive(Serialize, Debug, Clone, PartialEq)]
@@ -268,13 +226,11 @@ pub struct SearchHit {
 pub struct SearchResult {
     pub hits: Vec<SearchHit>,
     pub query: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub vector: Option<Vec<f32>>,
     pub processing_time_ms: u128,
     #[serde(flatten)]
     pub hits_info: HitsInfo,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub facet_distribution: Option<BTreeMap<String, IndexMap<String, u64>>>,
+    pub facet_distribution: Option<BTreeMap<String, BTreeMap<String, u64>>>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub facet_stats: Option<BTreeMap<String, FacetStats>>,
 }
@@ -302,14 +258,6 @@ pub struct FacetStats {
     pub max: f64,
 }
 
-#[derive(Serialize, Debug, Clone, PartialEq)]
-#[serde(rename_all = "camelCase")]
-pub struct FacetSearchResult {
-    pub facet_hits: Vec<FacetValueHit>,
-    pub facet_query: Option<String>,
-    pub processing_time_ms: u128,
-}
-
 /// Incorporate search rules in search query
 pub fn add_search_rules(query: &mut SearchQuery, rules: IndexSearchRules) {
     query.filter = match (query.filter.take(), rules.filter) {
@@ -330,52 +278,28 @@ pub fn add_search_rules(query: &mut SearchQuery, rules: IndexSearchRules) {
     }
 }
 
-fn prepare_search<'t>(
-    index: &'t Index,
-    rtxn: &'t RoTxn,
-    query: &'t SearchQuery,
-    features: RoFeatures,
-) -> Result<(milli::Search<'t>, bool, usize, usize), MeilisearchHttpError> {
-    let mut search = index.search(rtxn);
+pub fn perform_search(
+    index: &Index,
+    query: SearchQuery,
+) -> Result<SearchResult, MeilisearchHttpError> {
+    let before_search = Instant::now();
+    let rtxn = index.read_txn()?;
 
-    if query.vector.is_some() && query.q.is_some() {
-        warn!("Ignoring the query string `q` when used with the `vector` parameter.");
-    }
-
-    if let Some(ref vector) = query.vector {
-        search.vector(vector.clone());
-    }
+    let mut search = index.search(&rtxn);
 
     if let Some(ref query) = query.q {
         search.query(query);
     }
 
-    if let Some(ref searchable) = query.attributes_to_search_on {
-        search.searchable_attributes(searchable);
-    }
-
     let is_finite_pagination = query.is_finite_pagination();
     search.terms_matching_strategy(query.matching_strategy.into());
 
     let max_total_hits = index
-        .pagination_max_total_hits(rtxn)
+        .pagination_max_total_hits(&rtxn)
         .map_err(milli::Error::from)?
         .unwrap_or(DEFAULT_PAGINATION_MAX_TOTAL_HITS);
 
     search.exhaustive_number_hits(is_finite_pagination);
-    search.scoring_strategy(if query.show_ranking_score || query.show_ranking_score_details {
-        ScoringStrategy::Detailed
-    } else {
-        ScoringStrategy::Skip
-    });
-
-    if query.show_ranking_score_details {
-        features.check_score_details()?;
-    }
-
-    if query.vector.is_some() {
-        features.check_vector()?;
-    }
 
     // compute the offset on the limit depending on the pagination mode.
     let (offset, limit) = if is_finite_pagination {
@@ -413,20 +337,6 @@ fn prepare_search<'t>(
         search.sort_criteria(sort);
     }
 
-    Ok((search, is_finite_pagination, max_total_hits, offset))
-}
-
-pub fn perform_search(
-    index: &Index,
-    query: SearchQuery,
-    features: RoFeatures,
-) -> Result<SearchResult, MeilisearchHttpError> {
-    let before_search = Instant::now();
-    let rtxn = index.read_txn()?;
-
-    let (search, is_finite_pagination, max_total_hits, offset) =
-        prepare_search(index, &rtxn, &query, features)?;
-
     let milli::SearchResult { documents_ids, matching_words, candidates, document_scores, .. } =
         search.execute()?;
 
@@ -491,26 +401,13 @@ pub fn perform_search(
         tokenizer_builder.allow_list(&script_lang_map);
     }
 
-    let separators = index.allowed_separators(&rtxn)?;
-    let separators: Option<Vec<_>> =
-        separators.as_ref().map(|x| x.iter().map(String::as_str).collect());
-    if let Some(ref separators) = separators {
-        tokenizer_builder.separators(separators);
-    }
-
-    let dictionary = index.dictionary(&rtxn)?;
-    let dictionary: Option<Vec<_>> =
-        dictionary.as_ref().map(|x| x.iter().map(String::as_str).collect());
-    if let Some(ref dictionary) = dictionary {
-        tokenizer_builder.words_dict(dictionary);
-    }
-
     let mut formatter_builder = MatcherBuilder::new(matching_words, tokenizer_builder.build());
     formatter_builder.crop_marker(query.crop_marker);
     formatter_builder.highlight_prefix(query.highlight_pre_tag);
     formatter_builder.highlight_suffix(query.highlight_post_tag);
 
     let mut documents = Vec::new();
+
     let documents_iter = index.documents(&rtxn, documents_ids)?;
 
     for ((_id, obkv), score) in documents_iter.into_iter().zip(document_scores.into_iter()) {
@@ -537,16 +434,8 @@ pub fn perform_search(
             insert_geo_distance(sort, &mut document);
         }
 
-        let semantic_score = match query.vector.as_ref() {
-            Some(vector) => match extract_field("_vectors", &fields_ids_map, obkv)? {
-                Some(vectors) => compute_semantic_score(vector, vectors)?,
-                None => None,
-            },
-            None => None,
-        };
-
         let ranking_score =
-            query.show_ranking_score.then(|| ScoreDetails::global_score(score.iter()));
+            query.show_ranking_score.then(|| ScoreDetails::global_score_linear_scale(score.iter()));
         let ranking_score_details =
             query.show_ranking_score_details.then(|| ScoreDetails::to_json_map(score.iter()));
 
@@ -556,7 +445,6 @@ pub fn perform_search(
             matches_position,
             ranking_score_details,
             ranking_score,
-            semantic_score,
         };
         documents.push(hit);
     }
@@ -589,30 +477,10 @@ pub fn perform_search(
                 .unwrap_or(DEFAULT_VALUES_PER_FACET);
             facet_distribution.max_values_per_facet(max_values_by_facet);
 
-            let sort_facet_values_by =
-                index.sort_facet_values_by(&rtxn).map_err(milli::Error::from)?;
-            let default_sort_facet_values_by =
-                sort_facet_values_by.get("*").copied().unwrap_or_default();
-
             if fields.iter().all(|f| f != "*") {
-                let fields: Vec<_> = fields
-                    .iter()
-                    .map(|n| {
-                        (
-                            n,
-                            sort_facet_values_by
-                                .get(n)
-                                .copied()
-                                .unwrap_or(default_sort_facet_values_by),
-                        )
-                    })
-                    .collect();
                 facet_distribution.facets(fields);
             }
-            let distribution = facet_distribution
-                .candidates(candidates)
-                .default_order_by(default_sort_facet_values_by)
-                .execute()?;
+            let distribution = facet_distribution.candidates(candidates).execute()?;
             let stats = facet_distribution.compute_stats()?;
             (Some(distribution), Some(stats))
         }
@@ -626,8 +494,7 @@ pub fn perform_search(
     let result = SearchResult {
         hits: documents,
         hits_info,
-        query: query.q.unwrap_or_default(),
-        vector: query.vector,
+        query: query.q.clone().unwrap_or_default(),
         processing_time_ms: before_search.elapsed().as_millis(),
         facet_distribution,
         facet_stats,
@@ -635,29 +502,6 @@ pub fn perform_search(
     Ok(result)
 }
 
-pub fn perform_facet_search(
-    index: &Index,
-    search_query: SearchQuery,
-    facet_query: Option<String>,
-    facet_name: String,
-    features: RoFeatures,
-) -> Result<FacetSearchResult, MeilisearchHttpError> {
-    let before_search = Instant::now();
-    let rtxn = index.read_txn()?;
-
-    let (search, _, _, _) = prepare_search(index, &rtxn, &search_query, features)?;
-    let mut facet_search = SearchForFacetValues::new(facet_name, search);
-    if let Some(facet_query) = &facet_query {
-        facet_search.query(facet_query);
-    }
-
-    Ok(FacetSearchResult {
-        facet_hits: facet_search.execute()?,
-        facet_query,
-        processing_time_ms: before_search.elapsed().as_millis(),
-    })
-}
-
 fn insert_geo_distance(sorts: &[String], document: &mut Document) {
     lazy_static::lazy_static! {
         static ref GEO_REGEX: Regex =
@@ -674,17 +518,6 @@ fn insert_geo_distance(sorts: &[String], document: &mut Document) {
     }
 }
 
-fn compute_semantic_score(query: &[f32], vectors: Value) -> milli::Result<Option<f32>> {
-    let vectors = serde_json::from_value(vectors)
-        .map(VectorOrArrayOfVectors::into_array_of_vectors)
-        .map_err(InternalError::SerdeJson)?;
-    Ok(vectors
-        .into_iter()
-        .map(|v| OrderedFloat(dot_product_similarity(query, &v)))
-        .max()
-        .map(OrderedFloat::into_inner))
-}
-
 fn compute_formatted_options(
     attr_to_highlight: &HashSet<String>,
     attr_to_crop: &[String],
@@ -812,26 +645,10 @@ fn make_document(
     Ok(document)
 }
 
-/// Extract the JSON value under the field name specified
-/// but doesn't support nested objects.
-fn extract_field(
-    field_name: &str,
-    field_ids_map: &FieldsIdsMap,
-    obkv: obkv::KvReaderU16,
-) -> Result<Option<serde_json::Value>, MeilisearchHttpError> {
-    match field_ids_map.id(field_name) {
-        Some(fid) => match obkv.get(fid) {
-            Some(value) => Ok(serde_json::from_slice(value).map(Some)?),
-            None => Ok(None),
-        },
-        None => Ok(None),
-    }
-}
-
-fn format_fields<'a>(
+fn format_fields<A: AsRef<[u8]>>(
     document: &Document,
     field_ids_map: &FieldsIdsMap,
-    builder: &'a MatcherBuilder<'a>,
+    builder: &MatcherBuilder<'_, A>,
     formatted_options: &BTreeMap<FieldId, FormatOptions>,
     compute_matches: bool,
     displayable_ids: &BTreeSet<FieldId>,
@@ -876,9 +693,9 @@ fn format_fields<'a>(
     Ok((matches_position, document))
 }
 
-fn format_value<'a>(
+fn format_value<A: AsRef<[u8]>>(
     value: Value,
-    builder: &'a MatcherBuilder<'a>,
+    builder: &MatcherBuilder<'_, A>,
     format_options: Option<FormatOptions>,
     infos: &mut Vec<MatchBounds>,
     compute_matches: bool,

meilisearch/tests/auth/api_keys.rs

@@ -422,7 +422,7 @@ async fn error_add_api_key_invalid_parameters_actions() {
     meili_snap::snapshot!(code, @"400 Bad Request");
     meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]" }), @r###"
     {
-      "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`",
+      "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`",
       "code": "invalid_api_key_actions",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions"

meilisearch/tests/auth/errors.rs

@@ -90,7 +90,7 @@ async fn create_api_key_bad_actions() {
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`",
+      "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`",
       "code": "invalid_api_key_actions",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions"

meilisearch/tests/common/index.rs

@@ -346,24 +346,17 @@ impl Index<'_> {
         query: Value,
         test: impl Fn(Value, StatusCode) + UnwindSafe + Clone,
     ) {
-        let post = self.search_post(query.clone()).await;
-
+        let (response, code) = self.search_post(query.clone()).await;
+        let t = test.clone();
+        if let Err(e) = catch_unwind(move || t(response, code)) {
+            eprintln!("Error with post search");
+            resume_unwind(e);
+        }
         let query = yaup::to_string(&query).unwrap();
-        let get = self.search_get(&query).await;
-
-        insta::allow_duplicates! {
-            let (response, code) = post;
-            let t = test.clone();
-            if let Err(e) = catch_unwind(move || t(response, code)) {
-                eprintln!("Error with post search");
-                resume_unwind(e);
-            }
-
-            let (response, code) = get;
-            if let Err(e) = catch_unwind(move || test(response, code)) {
-                eprintln!("Error with get search");
-                resume_unwind(e);
-            }
+        let (response, code) = self.search_get(&query).await;
+        if let Err(e) = catch_unwind(move || test(response, code)) {
+            eprintln!("Error with get search");
+            resume_unwind(e);
         }
     }
 
@@ -377,11 +370,6 @@ impl Index<'_> {
         self.service.get(url).await
     }
 
-    pub async fn facet_search(&self, query: Value) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/facet-search", urlencode(self.uid.as_ref()));
-        self.service.post_encoded(url, query, self.encoder).await
-    }
-
     pub async fn update_distinct_attribute(&self, value: Value) -> (Value, StatusCode) {
         let url =
             format!("/indexes/{}/settings/{}", urlencode(self.uid.as_ref()), "distinct-attribute");

meilisearch/tests/search/errors.rs

@@ -963,29 +963,3 @@ async fn sort_unset_ranking_rule() {
         )
         .await;
 }
-
-#[actix_rt::test]
-async fn search_on_unknown_field() {
-    let server = Server::new().await;
-    let index = server.index("test");
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
-    index
-        .search(
-            json!({"q": "Captain Marvel", "attributesToSearchOn": ["unknown"]}),
-            |response, code| {
-                snapshot!(code, @"400 Bad Request");
-                snapshot!(json_string!(response), @r###"
-                {
-                  "message": "Attribute `unknown` is not searchable. Available searchable attributes are: `id, title`.",
-                  "code": "invalid_search_attributes_to_search_on",
-                  "type": "invalid_request",
-                  "link": "https://docs.meilisearch.com/errors#invalid_search_attributes_to_search_on"
-                }
-                "###);
-            },
-        )
-        .await;
-}

meilisearch/tests/search/facet_search.rs

@@ -1,92 +0,0 @@
-use once_cell::sync::Lazy;
-use serde_json::{json, Value};
-
-use crate::common::Server;
-
-pub(self) static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
-    json!([
-        {
-            "title": "Shazam!",
-            "genres": ["Action", "Adventure"],
-            "id": "287947",
-        },
-        {
-            "title": "Captain Marvel",
-            "genres": ["Action", "Adventure"],
-            "id": "299537",
-        },
-        {
-            "title": "Escape Room",
-            "genres": ["Horror", "Thriller", "Multiple Words"],
-            "id": "522681",
-        },
-        {
-            "title": "How to Train Your Dragon: The Hidden World",
-            "genres": ["Action", "Comedy"],
-            "id": "166428",
-        },
-        {
-            "title": "Gläss",
-            "genres": ["Thriller"],
-            "id": "450465",
-        }
-    ])
-});
-
-#[actix_rt::test]
-async fn simple_facet_search() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let documents = DOCUMENTS.clone();
-    index.update_settings_filterable_attributes(json!(["genres"])).await;
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
-    let (response, code) =
-        index.facet_search(json!({"facetName": "genres", "facetQuery": "a"})).await;
-
-    assert_eq!(code, 200, "{}", response);
-    assert_eq!(dbg!(response)["facetHits"].as_array().unwrap().len(), 2);
-
-    let (response, code) =
-        index.facet_search(json!({"facetName": "genres", "facetQuery": "adventure"})).await;
-
-    assert_eq!(code, 200, "{}", response);
-    assert_eq!(response["facetHits"].as_array().unwrap().len(), 1);
-}
-
-#[actix_rt::test]
-async fn non_filterable_facet_search_error() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
-    let (response, code) =
-        index.facet_search(json!({"facetName": "genres", "facetQuery": "a"})).await;
-    assert_eq!(code, 400, "{}", response);
-
-    let (response, code) =
-        index.facet_search(json!({"facetName": "genres", "facetQuery": "adv"})).await;
-    assert_eq!(code, 400, "{}", response);
-}
-
-#[actix_rt::test]
-async fn facet_search_dont_support_words() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let documents = DOCUMENTS.clone();
-    index.update_settings_filterable_attributes(json!(["genres"])).await;
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
-    let (response, code) =
-        index.facet_search(json!({"facetName": "genres", "facetQuery": "words"})).await;
-
-    assert_eq!(code, 200, "{}", response);
-    assert_eq!(response["facetHits"].as_array().unwrap().len(), 0);
-}

meilisearch/tests/search/mod.rs

@@ -2,11 +2,9 @@
 // should be tested in its own module to isolate tests and keep the tests readable.
 
 mod errors;
-mod facet_search;
 mod formatted;
 mod multi;
 mod pagination;
-mod restrict_searchable;
 
 use once_cell::sync::Lazy;
 use serde_json::{json, Value};

meilisearch/tests/search/restrict_searchable.rs

@@ -1,267 +0,0 @@
-use meili_snap::{json_string, snapshot};
-use once_cell::sync::Lazy;
-use serde_json::{json, Value};
-
-use crate::common::index::Index;
-use crate::common::Server;
-
-async fn index_with_documents<'a>(server: &'a Server, documents: &Value) -> Index<'a> {
-    let index = server.index("test");
-
-    index.add_documents(documents.clone(), None).await;
-    index.wait_task(0).await;
-    index
-}
-
-static SIMPLE_SEARCH_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
-    json!([
-    {
-        "title": "Shazam!",
-        "desc": "a Captain Marvel ersatz",
-        "id": "1",
-    },
-    {
-        "title": "Captain Planet",
-        "desc": "He's not part of the Marvel Cinematic Universe",
-        "id": "2",
-    },
-    {
-        "title": "Captain Marvel",
-        "desc": "a Shazam ersatz",
-        "id": "3",
-    }])
-});
-
-#[actix_rt::test]
-async fn simple_search_on_title() {
-    let server = Server::new().await;
-    let index = index_with_documents(&server, &SIMPLE_SEARCH_DOCUMENTS).await;
-
-    // simple search should return 2 documents (ids: 2 and 3).
-    index
-        .search(
-            json!({"q": "Captain Marvel", "attributesToSearchOn": ["title"]}),
-            |response, code| {
-                snapshot!(code, @"200 OK");
-                snapshot!(response["hits"].as_array().unwrap().len(), @"2");
-            },
-        )
-        .await;
-}
-
-#[actix_rt::test]
-async fn simple_prefix_search_on_title() {
-    let server = Server::new().await;
-    let index = index_with_documents(&server, &SIMPLE_SEARCH_DOCUMENTS).await;
-
-    // simple search should return 2 documents (ids: 2 and 3).
-    index
-        .search(json!({"q": "Captain Mar", "attributesToSearchOn": ["title"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(response["hits"].as_array().unwrap().len(), @"2");
-        })
-        .await;
-}
-
-#[actix_rt::test]
-async fn simple_search_on_title_matching_strategy_all() {
-    let server = Server::new().await;
-    let index = index_with_documents(&server, &SIMPLE_SEARCH_DOCUMENTS).await;
-    // simple search matching strategy all should only return 1 document (ids: 2).
-    index
-        .search(json!({"q": "Captain Marvel", "attributesToSearchOn": ["title"], "matchingStrategy": "all"}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(response["hits"].as_array().unwrap().len(), @"1");
-        })
-        .await;
-}
-
-#[actix_rt::test]
-async fn simple_search_on_no_field() {
-    let server = Server::new().await;
-    let index = index_with_documents(&server, &SIMPLE_SEARCH_DOCUMENTS).await;
-    // simple search on no field shouldn't return any document.
-    index
-        .search(json!({"q": "Captain Marvel", "attributesToSearchOn": []}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(response["hits"].as_array().unwrap().len(), @"0");
-        })
-        .await;
-}
-
-#[actix_rt::test]
-async fn word_ranking_rule_order() {
-    let server = Server::new().await;
-    let index = index_with_documents(&server, &SIMPLE_SEARCH_DOCUMENTS).await;
-
-    // Document 3 should appear before document 2.
-    index
-        .search(
-            json!({"q": "Captain Marvel", "attributesToSearchOn": ["title"], "attributesToRetrieve": ["id"]}),
-            |response, code| {
-                snapshot!(code, @"200 OK");
-                snapshot!(json_string!(response["hits"]),
-                    @r###"
-                [
-                  {
-                    "id": "3"
-                  },
-                  {
-                    "id": "2"
-                  }
-                ]
-                "###
-                );
-            },
-        )
-        .await;
-}
-
-#[actix_rt::test]
-async fn word_ranking_rule_order_exact_words() {
-    let server = Server::new().await;
-    let index = index_with_documents(&server, &SIMPLE_SEARCH_DOCUMENTS).await;
-    index.update_settings_typo_tolerance(json!({"disableOnWords": ["Captain", "Marvel"]})).await;
-    index.wait_task(1).await;
-
-    // simple search should return 2 documents (ids: 2 and 3).
-    index
-        .search(
-            json!({"q": "Captain Marvel", "attributesToSearchOn": ["title"], "attributesToRetrieve": ["id"]}),
-            |response, code| {
-                snapshot!(code, @"200 OK");
-                snapshot!(json_string!(response["hits"]),
-                    @r###"
-                [
-                  {
-                    "id": "3"
-                  },
-                  {
-                    "id": "2"
-                  }
-                ]
-                "###
-                );
-            },
-        )
-        .await;
-}
-
-#[actix_rt::test]
-async fn typo_ranking_rule_order() {
-    let server = Server::new().await;
-    let index = index_with_documents(
-        &server,
-        &json!([
-        {
-            "title": "Capitain Marivel",
-            "desc": "Captain Marvel",
-            "id": "1",
-        },
-        {
-            "title": "Captain Marivel",
-            "desc": "a Shazam ersatz",
-            "id": "2",
-        }]),
-    )
-    .await;
-
-    // Document 2 should appear before document 1.
-    index
-        .search(json!({"q": "Captain Marvel", "attributesToSearchOn": ["title"], "attributesToRetrieve": ["id"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]),
-                @r###"
-            [
-              {
-                "id": "2"
-              },
-              {
-                "id": "1"
-              }
-            ]
-            "###
-            );
-        })
-        .await;
-}
-
-#[actix_rt::test]
-async fn attributes_ranking_rule_order() {
-    let server = Server::new().await;
-    let index = index_with_documents(
-        &server,
-        &json!([
-        {
-            "title": "Captain Marvel",
-            "desc": "a Shazam ersatz",
-            "footer": "The story of Captain Marvel",
-            "id": "1",
-        },
-        {
-            "title": "The Avengers",
-            "desc": "Captain Marvel is far from the earth",
-            "footer": "A super hero team",
-            "id": "2",
-        }]),
-    )
-    .await;
-
-    // Document 2 should appear before document 1.
-    index
-        .search(json!({"q": "Captain Marvel", "attributesToSearchOn": ["desc", "footer"], "attributesToRetrieve": ["id"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]),
-                @r###"
-            [
-              {
-                "id": "2"
-              },
-              {
-                "id": "1"
-              }
-            ]
-            "###
-            );
-        })
-        .await;
-}
-
-#[actix_rt::test]
-async fn exactness_ranking_rule_order() {
-    let server = Server::new().await;
-    let index = index_with_documents(
-        &server,
-        &json!([
-        {
-            "title": "Captain Marvel",
-            "desc": "Captain Marivel",
-            "id": "1",
-        },
-        {
-            "title": "Captain Marvel",
-            "desc": "CaptainMarvel",
-            "id": "2",
-        }]),
-    )
-    .await;
-
-    // Document 2 should appear before document 1.
-    index
-        .search(json!({"q": "Captain Marvel", "attributesToRetrieve": ["id"], "attributesToSearchOn": ["desc"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]),
-                @r###"
-            [
-              {
-                "id": "2"
-              },
-              {
-                "id": "1"
-              }
-            ]
-            "###
-            );
-        })
-        .await;
-}

meilisearch/tests/settings/get_settings.rs

@@ -16,17 +16,11 @@ static DEFAULT_SETTINGS_VALUES: Lazy<HashMap<&'static str, Value>> = Lazy::new(|
         json!(["words", "typo", "proximity", "attribute", "sort", "exactness"]),
     );
     map.insert("stop_words", json!([]));
-    map.insert("non_separator_tokens", json!([]));
-    map.insert("separator_tokens", json!([]));
-    map.insert("dictionary", json!([]));
     map.insert("synonyms", json!({}));
     map.insert(
         "faceting",
         json!({
             "maxValuesPerFacet": json!(100),
-            "sortFacetValuesBy": {
-                "*": "alpha"
-            }
         }),
     );
     map.insert(
@@ -54,7 +48,7 @@ async fn get_settings() {
     let (response, code) = index.settings().await;
     assert_eq!(code, 200);
     let settings = response.as_object().unwrap();
-    assert_eq!(settings.keys().len(), 14);
+    assert_eq!(settings.keys().len(), 11);
     assert_eq!(settings["displayedAttributes"], json!(["*"]));
     assert_eq!(settings["searchableAttributes"], json!(["*"]));
     assert_eq!(settings["filterableAttributes"], json!([]));
@@ -65,16 +59,10 @@ async fn get_settings() {
         json!(["words", "typo", "proximity", "attribute", "sort", "exactness"])
     );
     assert_eq!(settings["stopWords"], json!([]));
-    assert_eq!(settings["nonSeparatorTokens"], json!([]));
-    assert_eq!(settings["separatorTokens"], json!([]));
-    assert_eq!(settings["dictionary"], json!([]));
     assert_eq!(
         settings["faceting"],
         json!({
             "maxValuesPerFacet": 100,
-            "sortFacetValuesBy": {
-                "*": "alpha"
-            }
         })
     );
     assert_eq!(

meilisearch/tests/settings/mod.rs

@@ -1,4 +1,3 @@
 mod distinct;
 mod errors;
 mod get_settings;
-mod tokenizer_customization;

meilisearch/tests/settings/tokenizer_customization.rs

@@ -1,459 +0,0 @@
-use meili_snap::{json_string, snapshot};
-use serde_json::json;
-
-use crate::common::Server;
-
-#[actix_rt::test]
-async fn set_and_reset() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let (_response, _code) = index
-        .update_settings(json!({
-            "nonSeparatorTokens": ["#", "&"],
-            "separatorTokens": ["&sep", "<br/>"],
-            "dictionary": ["J.R.R.", "J. R. R."],
-        }))
-        .await;
-    index.wait_task(0).await;
-
-    let (response, _) = index.settings().await;
-    snapshot!(json_string!(response["nonSeparatorTokens"]), @r###"
-    [
-      "#",
-      "&"
-    ]
-    "###);
-    snapshot!(json_string!(response["separatorTokens"]), @r###"
-    [
-      "&sep",
-      "<br/>"
-    ]
-    "###);
-    snapshot!(json_string!(response["dictionary"]), @r###"
-    [
-      "J. R. R.",
-      "J.R.R."
-    ]
-    "###);
-
-    index
-        .update_settings(json!({
-            "nonSeparatorTokens": null,
-            "separatorTokens": null,
-            "dictionary": null,
-        }))
-        .await;
-
-    index.wait_task(1).await;
-
-    let (response, _) = index.settings().await;
-    snapshot!(json_string!(response["nonSeparatorTokens"]), @"[]");
-    snapshot!(json_string!(response["separatorTokens"]), @"[]");
-    snapshot!(json_string!(response["dictionary"]), @"[]");
-}
-
-#[actix_rt::test]
-async fn set_and_search() {
-    let documents = json!([
-        {
-            "id": 1,
-            "content": "Mac & cheese",
-        },
-        {
-            "id": 2,
-            "content": "G#D#G#D#G#C#D#G#C#",
-        },
-        {
-            "id": 3,
-            "content": "Mac&sep&&sepcheese",
-        },
-    ]);
-
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
-    let (_response, _code) = index
-        .update_settings(json!({
-            "nonSeparatorTokens": ["#", "&"],
-            "separatorTokens": ["<br/>", "&sep"],
-            "dictionary": ["#", "A#", "B#", "C#", "D#", "E#", "F#", "G#"],
-        }))
-        .await;
-    index.wait_task(1).await;
-
-    index
-        .search(json!({"q": "&", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 1,
-                "content": "Mac & cheese",
-                "_formatted": {
-                  "id": "1",
-                  "content": "Mac <em>&</em> cheese"
-                }
-              },
-              {
-                "id": 3,
-                "content": "Mac&sep&&sepcheese",
-                "_formatted": {
-                  "id": "3",
-                  "content": "Mac&sep<em>&</em>&sepcheese"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(
-            json!({"q": "Mac & cheese", "attributesToHighlight": ["content"]}),
-            |response, code| {
-                snapshot!(code, @"200 OK");
-                snapshot!(json_string!(response["hits"]), @r###"
-                [
-                  {
-                    "id": 1,
-                    "content": "Mac & cheese",
-                    "_formatted": {
-                      "id": "1",
-                      "content": "<em>Mac</em> <em>&</em> <em>cheese</em>"
-                    }
-                  },
-                  {
-                    "id": 3,
-                    "content": "Mac&sep&&sepcheese",
-                    "_formatted": {
-                      "id": "3",
-                      "content": "<em>Mac</em>&sep<em>&</em>&sep<em>cheese</em>"
-                    }
-                  }
-                ]
-                "###);
-            },
-        )
-        .await;
-
-    index
-        .search(
-            json!({"q": "Mac&sep&&sepcheese", "attributesToHighlight": ["content"]}),
-            |response, code| {
-                snapshot!(code, @"200 OK");
-                snapshot!(json_string!(response["hits"]), @r###"
-                [
-                  {
-                    "id": 1,
-                    "content": "Mac & cheese",
-                    "_formatted": {
-                      "id": "1",
-                      "content": "<em>Mac</em> <em>&</em> <em>cheese</em>"
-                    }
-                  },
-                  {
-                    "id": 3,
-                    "content": "Mac&sep&&sepcheese",
-                    "_formatted": {
-                      "id": "3",
-                      "content": "<em>Mac</em>&sep<em>&</em>&sep<em>cheese</em>"
-                    }
-                  }
-                ]
-                "###);
-            },
-        )
-        .await;
-
-    index
-        .search(json!({"q": "C#D#G", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 2,
-                "content": "G#D#G#D#G#C#D#G#C#",
-                "_formatted": {
-                  "id": "2",
-                  "content": "<em>G</em>#<em>D#</em><em>G</em>#<em>D#</em><em>G</em>#<em>C#</em><em>D#</em><em>G</em>#<em>C#</em>"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "#", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @"[]");
-        })
-        .await;
-}
-
-#[actix_rt::test]
-async fn advanced_synergies() {
-    let documents = json!([
-        {
-            "id": 1,
-            "content": "J.R.R. Tolkien",
-        },
-        {
-            "id": 2,
-            "content": "J. R. R. Tolkien",
-        },
-        {
-            "id": 3,
-            "content": "jrr Tolkien",
-        },
-        {
-            "id": 4,
-            "content": "J.K. Rowlings",
-        },
-        {
-            "id": 5,
-            "content": "J. K. Rowlings",
-        },
-        {
-            "id": 6,
-            "content": "jk Rowlings",
-        },
-    ]);
-
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
-    let (_response, _code) = index
-        .update_settings(json!({
-            "dictionary": ["J.R.R.", "J. R. R.", "J.K.", "J. K."],
-            "synonyms": {
-                "J.R.R.": ["jrr", "J. R. R."],
-                "J. R. R.": ["jrr", "J.R.R."],
-                "jrr": ["J.R.R.", "J. R. R."],
-                "J.K.": ["jk", "J. K."],
-                "J. K.": ["jk", "J.K."],
-                "jk": ["J.K.", "J. K."],
-            }
-        }))
-        .await;
-    index.wait_task(1).await;
-
-    index
-        .search(json!({"q": "J.R.R.", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 1,
-                "content": "J.R.R. Tolkien",
-                "_formatted": {
-                  "id": "1",
-                  "content": "<em>J.R.R.</em> Tolkien"
-                }
-              },
-              {
-                "id": 2,
-                "content": "J. R. R. Tolkien",
-                "_formatted": {
-                  "id": "2",
-                  "content": "<em>J. R. R.</em> Tolkien"
-                }
-              },
-              {
-                "id": 3,
-                "content": "jrr Tolkien",
-                "_formatted": {
-                  "id": "3",
-                  "content": "<em>jrr</em> Tolkien"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "jrr", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 3,
-                "content": "jrr Tolkien",
-                "_formatted": {
-                  "id": "3",
-                  "content": "<em>jrr</em> Tolkien"
-                }
-              },
-              {
-                "id": 1,
-                "content": "J.R.R. Tolkien",
-                "_formatted": {
-                  "id": "1",
-                  "content": "<em>J.R.R.</em> Tolkien"
-                }
-              },
-              {
-                "id": 2,
-                "content": "J. R. R. Tolkien",
-                "_formatted": {
-                  "id": "2",
-                  "content": "<em>J. R. R.</em> Tolkien"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "J. R. R.", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 2,
-                "content": "J. R. R. Tolkien",
-                "_formatted": {
-                  "id": "2",
-                  "content": "<em>J. R. R.</em> Tolkien"
-                }
-              },
-              {
-                "id": 1,
-                "content": "J.R.R. Tolkien",
-                "_formatted": {
-                  "id": "1",
-                  "content": "<em>J.R.R.</em> Tolkien"
-                }
-              },
-              {
-                "id": 3,
-                "content": "jrr Tolkien",
-                "_formatted": {
-                  "id": "3",
-                  "content": "<em>jrr</em> Tolkien"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "jk", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 6,
-                "content": "jk Rowlings",
-                "_formatted": {
-                  "id": "6",
-                  "content": "<em>jk</em> Rowlings"
-                }
-              },
-              {
-                "id": 4,
-                "content": "J.K. Rowlings",
-                "_formatted": {
-                  "id": "4",
-                  "content": "<em>J.K.</em> Rowlings"
-                }
-              },
-              {
-                "id": 5,
-                "content": "J. K. Rowlings",
-                "_formatted": {
-                  "id": "5",
-                  "content": "<em>J. K.</em> Rowlings"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "J.K.", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 4,
-                "content": "J.K. Rowlings",
-                "_formatted": {
-                  "id": "4",
-                  "content": "<em>J.K.</em> Rowlings"
-                }
-              },
-              {
-                "id": 5,
-                "content": "J. K. Rowlings",
-                "_formatted": {
-                  "id": "5",
-                  "content": "<em>J. K.</em> Rowlings"
-                }
-              },
-              {
-                "id": 6,
-                "content": "jk Rowlings",
-                "_formatted": {
-                  "id": "6",
-                  "content": "<em>jk</em> Rowlings"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "J. K.", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 5,
-                "content": "J. K. Rowlings",
-                "_formatted": {
-                  "id": "5",
-                  "content": "<em>J. K.</em> Rowlings"
-                }
-              },
-              {
-                "id": 4,
-                "content": "J.K. Rowlings",
-                "_formatted": {
-                  "id": "4",
-                  "content": "<em>J.K.</em> Rowlings"
-                }
-              },
-              {
-                "id": 6,
-                "content": "jk Rowlings",
-                "_formatted": {
-                  "id": "6",
-                  "content": "<em>jk</em> Rowlings"
-                }
-              },
-              {
-                "id": 2,
-                "content": "J. R. R. Tolkien",
-                "_formatted": {
-                  "id": "2",
-                  "content": "<em>J. R.</em> R. Tolkien"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-}

milli/Cargo.toml

@@ -15,9 +15,8 @@ license.workspace = true
 bimap = { version = "0.6.3", features = ["serde"] }
 bincode = "1.3.3"
 bstr = "1.4.0"
-bytemuck = { version = "1.13.1", features = ["extern_crate_alloc"] }
 byteorder = "1.4.3"
-charabia = { version = "0.8.1", default-features = false }
+charabia = { version = "0.7.2", default-features = false }
 concat-arrays = "0.1.2"
 crossbeam-channel = "0.5.8"
 deserr = "0.5.0"
@@ -33,22 +32,18 @@ heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.6", default-f
     "lmdb",
     "sync-read-txn",
 ] }
-hnsw = { version = "0.11.0", features = ["serde1"] }
-indexmap = { version = "1.9.3", features = ["serde"] }
 json-depth-checker = { path = "../json-depth-checker" }
 levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] }
 memmap2 = "0.5.10"
 obkv = "0.2.0"
 once_cell = "1.17.1"
 ordered-float = "3.6.0"
-rand_pcg = { version = "0.3.1", features = ["serde1"] }
 rayon = "1.7.0"
 roaring = "0.10.1"
 rstar = { version = "0.10.0", features = ["serde"] }
 serde = { version = "1.0.160", features = ["derive"] }
 serde_json = { version = "1.0.95", features = ["preserve_order"] }
 slice-group-by = "0.3.0"
-space = "0.17.0"
 smallstr = { version = "0.3.0", features = ["serde"] }
 smallvec = "1.10.0"
 smartstring = "1.0.1"
@@ -67,9 +62,6 @@ filter-parser = { path = "../filter-parser" }
 # documents words self-join
 itertools = "0.10.5"
 
-# profiling
-puffin = "0.16.0"
-
 # logging
 log = "0.4.17"
 logging_timer = "1.1.0"
@@ -83,6 +75,9 @@ maplit = "1.0.2"
 md5 = "0.7.0"
 rand = { version = "0.8.5", features = ["small_rng"] }
 
+[target.'cfg(fuzzing)'.dev-dependencies]
+fuzzcheck = "0.12.1"
+
 [features]
 all-tokenizations = ["charabia/default"]
 

milli/examples/search.rs

@@ -52,9 +52,7 @@ fn main() -> Result<(), Box<dyn Error>> {
             let docs = execute_search(
                 &mut ctx,
                 &(!query.trim().is_empty()).then(|| query.trim().to_owned()),
-                &None,
                 TermsMatchingStrategy::Last,
-                milli::score_details::ScoringStrategy::Skip,
                 false,
                 &None,
                 &None,

milli/src/distance.rs

@@ -1,25 +0,0 @@
-use serde::{Deserialize, Serialize};
-use space::Metric;
-
-#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)]
-pub struct DotProduct;
-
-impl Metric<Vec<f32>> for DotProduct {
-    type Unit = u32;
-
-    // Following <https://docs.rs/space/0.17.0/space/trait.Metric.html>.
-    //
-    // Here is a playground that validate the ordering of the bit representation of floats in range 0.0..=1.0:
-    // <https://play.rust-lang.org/?version=stable&mode=debug&edition=2021&gist=6c59e31a3cc5036b32edf51e8937b56e>
-    fn distance(&self, a: &Vec<f32>, b: &Vec<f32>) -> Self::Unit {
-        let dist = 1.0 - dot_product_similarity(a, b);
-        debug_assert!(!dist.is_nan());
-        dist.to_bits()
-    }
-}
-
-/// Returns the dot product similarity score that will between 0.0 and 1.0
-/// if both vectors are normalized. The higher the more similar the vectors are.
-pub fn dot_product_similarity(a: &[f32], b: &[f32]) -> f32 {
-    a.iter().zip(b).map(|(a, b)| a * b).sum()
-}

milli/src/documents/mod.rs

@@ -111,6 +111,7 @@ pub enum Error {
     Io(#[from] io::Error),
 }
 
+#[cfg(test)]
 pub fn objects_from_json_value(json: serde_json::Value) -> Vec<crate::Object> {
     let documents = match json {
         object @ serde_json::Value::Object(_) => vec![object],
@@ -140,6 +141,7 @@ macro_rules! documents {
     }};
 }
 
+#[cfg(test)]
 pub fn documents_batch_reader_from_objects(
     objects: impl IntoIterator<Item = Object>,
 ) -> DocumentsBatchReader<std::io::Cursor<Vec<u8>>> {

milli/src/error.rs

@@ -110,13 +110,9 @@ only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and undersco
     },
     #[error(transparent)]
     InvalidGeoField(#[from] GeoError),
-    #[error("Invalid vector dimensions: expected: `{}`, found: `{}`.", .expected, .found)]
-    InvalidVectorDimensions { expected: usize, found: usize },
-    #[error("The `_vectors` field in the document with the id: `{document_id}` is not an array. Was expecting an array of floats or an array of arrays of floats but instead got `{value}`.")]
-    InvalidVectorsType { document_id: Value, value: Value },
     #[error("{0}")]
     InvalidFilter(String),
-    #[error("Invalid type for filter subexpression: expected: {}, found: {1}.", .0.join(", "))]
+    #[error("Invalid type for filter subexpression: `expected {}, found: {1}`.", .0.join(", "))]
     InvalidFilterExpression(&'static [&'static str], Value),
     #[error("Attribute `{}` is not sortable. {}",
         .field,
@@ -128,26 +124,6 @@ only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and undersco
         }
     )]
     InvalidSortableAttribute { field: String, valid_fields: BTreeSet<String> },
-    #[error("Attribute `{}` is not facet-searchable. {}",
-        .field,
-        match .valid_fields.is_empty() {
-            true => "This index does not have configured facet-searchable attributes. To make it facet-searchable add it to the `filterableAttributes` index settings.".to_string(),
-            false => format!("Available facet-searchable attributes are: `{}`. To make it facet-searchable add it to the `filterableAttributes` index settings.",
-                    valid_fields.iter().map(AsRef::as_ref).collect::<Vec<&str>>().join(", ")
-                ),
-        }
-    )]
-    InvalidFacetSearchFacetName { field: String, valid_fields: BTreeSet<String> },
-    #[error("Attribute `{}` is not searchable. Available searchable attributes are: `{}{}`.",
-        .field,
-        .valid_fields.iter().map(AsRef::as_ref).collect::<Vec<&str>>().join(", "),
-        .hidden_fields.then_some(", <..hidden-attributes>").unwrap_or(""),
-    )]
-    InvalidSearchableAttribute {
-        field: String,
-        valid_fields: BTreeSet<String>,
-        hidden_fields: bool,
-    },
     #[error("{}", HeedError::BadOpenOptions)]
     InvalidLmdbOpenOptions,
     #[error("You must specify where `sort` is listed in the rankingRules setting to use the sort parameter at search time.")]

milli/src/external_documents_ids.rs

@@ -106,30 +106,22 @@ impl<'a> ExternalDocumentsIds<'a> {
         map
     }
 
-    /// Return an fst of the combined hard and soft deleted ID.
-    pub fn to_fst<'b>(&'b self) -> fst::Result<Cow<'b, fst::Map<Cow<'a, [u8]>>>> {
-        if self.soft.is_empty() {
-            return Ok(Cow::Borrowed(&self.hard));
-        }
-        let union_op = self.hard.op().add(&self.soft).r#union();
-
-        let mut iter = union_op.into_stream();
-        let mut new_hard_builder = fst::MapBuilder::memory();
-        while let Some((external_id, marked_docids)) = iter.next() {
-            let value = indexed_last_value(marked_docids).unwrap();
-            if value != DELETED_ID {
-                new_hard_builder.insert(external_id, value)?;
-            }
-        }
-
-        drop(iter);
-
-        Ok(Cow::Owned(new_hard_builder.into_map().map_data(Cow::Owned)?))
-    }
-
     fn merge_soft_into_hard(&mut self) -> fst::Result<()> {
         if self.soft.len() >= self.hard.len() / 2 {
-            self.hard = self.to_fst()?.into_owned();
+            let union_op = self.hard.op().add(&self.soft).r#union();
+
+            let mut iter = union_op.into_stream();
+            let mut new_hard_builder = fst::MapBuilder::memory();
+            while let Some((external_id, marked_docids)) = iter.next() {
+                let value = indexed_last_value(marked_docids).unwrap();
+                if value != DELETED_ID {
+                    new_hard_builder.insert(external_id, value)?;
+                }
+            }
+
+            drop(iter);
+
+            self.hard = new_hard_builder.into_map().map_data(Cow::Owned)?;
             self.soft = fst::Map::default().map_data(Cow::Owned)?;
         }
 

milli/src/heed_codec/fst_set_codec.rs

@@ -1,23 +0,0 @@
-use std::borrow::Cow;
-
-use fst::Set;
-use heed::{BytesDecode, BytesEncode};
-
-/// A codec for values of type `Set<&[u8]>`.
-pub struct FstSetCodec;
-
-impl<'a> BytesEncode<'a> for FstSetCodec {
-    type EItem = Set<Vec<u8>>;
-
-    fn bytes_encode(item: &'a Self::EItem) -> Option<Cow<'a, [u8]>> {
-        Some(Cow::Borrowed(item.as_fst().as_bytes()))
-    }
-}
-
-impl<'a> BytesDecode<'a> for FstSetCodec {
-    type DItem = Set<&'a [u8]>;
-
-    fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
-        Set::new(bytes).ok()
-    }
-}

milli/src/heed_codec/mod.rs

@@ -2,7 +2,6 @@ mod beu32_str_codec;
 mod byte_slice_ref;
 pub mod facet;
 mod field_id_word_count_codec;
-mod fst_set_codec;
 mod obkv_codec;
 mod roaring_bitmap;
 mod roaring_bitmap_length;
@@ -16,7 +15,6 @@ pub use str_ref::StrRefCodec;
 
 pub use self::beu32_str_codec::BEU32StrCodec;
 pub use self::field_id_word_count_codec::FieldIdWordCountCodec;
-pub use self::fst_set_codec::FstSetCodec;
 pub use self::obkv_codec::ObkvCodec;
 pub use self::roaring_bitmap::{BoRoaringBitmapCodec, CboRoaringBitmapCodec, RoaringBitmapCodec};
 pub use self::roaring_bitmap_length::{
@@ -25,9 +23,3 @@ pub use self::roaring_bitmap_length::{
 pub use self::script_language_codec::ScriptLanguageCodec;
 pub use self::str_beu32_codec::{StrBEU16Codec, StrBEU32Codec};
 pub use self::str_str_u8_codec::{U8StrStrCodec, UncheckedU8StrStrCodec};
-
-pub trait BytesDecodeOwned {
-    type DItem;
-
-    fn bytes_decode_owned(bytes: &[u8]) -> Option<Self::DItem>;
-}

milli/src/heed_codec/roaring_bitmap/bo_roaring_bitmap_codec.rs

@@ -2,11 +2,8 @@ use std::borrow::Cow;
 use std::convert::TryInto;
 use std::mem::size_of;
 
-use heed::BytesDecode;
 use roaring::RoaringBitmap;
 
-use crate::heed_codec::BytesDecodeOwned;
-
 pub struct BoRoaringBitmapCodec;
 
 impl BoRoaringBitmapCodec {
@@ -16,7 +13,7 @@ impl BoRoaringBitmapCodec {
     }
 }
 
-impl BytesDecode<'_> for BoRoaringBitmapCodec {
+impl heed::BytesDecode<'_> for BoRoaringBitmapCodec {
     type DItem = RoaringBitmap;
 
     fn bytes_decode(bytes: &[u8]) -> Option<Self::DItem> {
@@ -31,14 +28,6 @@ impl BytesDecode<'_> for BoRoaringBitmapCodec {
     }
 }
 
-impl BytesDecodeOwned for BoRoaringBitmapCodec {
-    type DItem = RoaringBitmap;
-
-    fn bytes_decode_owned(bytes: &[u8]) -> Option<Self::DItem> {
-        Self::bytes_decode(bytes)
-    }
-}
-
 impl heed::BytesEncode<'_> for BoRoaringBitmapCodec {
     type EItem = RoaringBitmap;
 

milli/src/heed_codec/roaring_bitmap/cbo_roaring_bitmap_codec.rs

@@ -5,8 +5,6 @@ use std::mem::size_of;
 use byteorder::{NativeEndian, ReadBytesExt, WriteBytesExt};
 use roaring::RoaringBitmap;
 
-use crate::heed_codec::BytesDecodeOwned;
-
 /// This is the limit where using a byteorder became less size efficient
 /// than using a direct roaring encoding, it is also the point where we are able
 /// to determine the encoding used only by using the array of bytes length.
@@ -105,14 +103,6 @@ impl heed::BytesDecode<'_> for CboRoaringBitmapCodec {
     }
 }
 
-impl BytesDecodeOwned for CboRoaringBitmapCodec {
-    type DItem = RoaringBitmap;
-
-    fn bytes_decode_owned(bytes: &[u8]) -> Option<Self::DItem> {
-        Self::deserialize_from(bytes).ok()
-    }
-}
-
 impl heed::BytesEncode<'_> for CboRoaringBitmapCodec {
     type EItem = RoaringBitmap;
 

milli/src/heed_codec/roaring_bitmap/roaring_bitmap_codec.rs

@@ -2,8 +2,6 @@ use std::borrow::Cow;
 
 use roaring::RoaringBitmap;
 
-use crate::heed_codec::BytesDecodeOwned;
-
 pub struct RoaringBitmapCodec;
 
 impl heed::BytesDecode<'_> for RoaringBitmapCodec {
@@ -14,14 +12,6 @@ impl heed::BytesDecode<'_> for RoaringBitmapCodec {
     }
 }
 
-impl BytesDecodeOwned for RoaringBitmapCodec {
-    type DItem = RoaringBitmap;
-
-    fn bytes_decode_owned(bytes: &[u8]) -> Option<Self::DItem> {
-        RoaringBitmap::deserialize_from(bytes).ok()
-    }
-}
-
 impl heed::BytesEncode<'_> for RoaringBitmapCodec {
     type EItem = RoaringBitmap;
 

milli/src/heed_codec/roaring_bitmap_length/bo_roaring_bitmap_len_codec.rs

@@ -1,23 +1,11 @@
 use std::mem;
 
-use heed::BytesDecode;
-
-use crate::heed_codec::BytesDecodeOwned;
-
 pub struct BoRoaringBitmapLenCodec;
 
-impl BytesDecode<'_> for BoRoaringBitmapLenCodec {
+impl heed::BytesDecode<'_> for BoRoaringBitmapLenCodec {
     type DItem = u64;
 
     fn bytes_decode(bytes: &[u8]) -> Option<Self::DItem> {
         Some((bytes.len() / mem::size_of::<u32>()) as u64)
     }
 }
-
-impl BytesDecodeOwned for BoRoaringBitmapLenCodec {
-    type DItem = u64;
-
-    fn bytes_decode_owned(bytes: &[u8]) -> Option<Self::DItem> {
-        Self::bytes_decode(bytes)
-    }
-}

milli/src/heed_codec/roaring_bitmap_length/cbo_roaring_bitmap_len_codec.rs

@@ -1,14 +1,11 @@
 use std::mem;
 
-use heed::BytesDecode;
-
 use super::{BoRoaringBitmapLenCodec, RoaringBitmapLenCodec};
 use crate::heed_codec::roaring_bitmap::cbo_roaring_bitmap_codec::THRESHOLD;
-use crate::heed_codec::BytesDecodeOwned;
 
 pub struct CboRoaringBitmapLenCodec;
 
-impl BytesDecode<'_> for CboRoaringBitmapLenCodec {
+impl heed::BytesDecode<'_> for CboRoaringBitmapLenCodec {
     type DItem = u64;
 
     fn bytes_decode(bytes: &[u8]) -> Option<Self::DItem> {
@@ -23,11 +20,3 @@ impl BytesDecode<'_> for CboRoaringBitmapLenCodec {
         }
     }
 }
-
-impl BytesDecodeOwned for CboRoaringBitmapLenCodec {
-    type DItem = u64;
-
-    fn bytes_decode_owned(bytes: &[u8]) -> Option<Self::DItem> {
-        Self::bytes_decode(bytes)
-    }
-}

milli/src/heed_codec/roaring_bitmap_length/roaring_bitmap_len_codec.rs

@@ -3,8 +3,6 @@ use std::mem;
 
 use byteorder::{LittleEndian, ReadBytesExt};
 
-use crate::heed_codec::BytesDecodeOwned;
-
 const SERIAL_COOKIE_NO_RUNCONTAINER: u32 = 12346;
 const SERIAL_COOKIE: u16 = 12347;
 
@@ -61,14 +59,6 @@ impl heed::BytesDecode<'_> for RoaringBitmapLenCodec {
     }
 }
 
-impl BytesDecodeOwned for RoaringBitmapLenCodec {
-    type DItem = u64;
-
-    fn bytes_decode_owned(bytes: &[u8]) -> Option<Self::DItem> {
-        RoaringBitmapLenCodec::deserialize_from_slice(bytes).ok()
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use heed::BytesEncode;

milli/src/index.rs

@@ -1,5 +1,5 @@
 use std::borrow::Cow;
-use std::collections::{BTreeSet, HashMap, HashSet};
+use std::collections::{HashMap, HashSet};
 use std::fs::File;
 use std::mem::size_of;
 use std::path::Path;
@@ -8,12 +8,10 @@ use charabia::{Language, Script};
 use heed::flags::Flags;
 use heed::types::*;
 use heed::{CompactionOption, Database, PolyDatabase, RoTxn, RwTxn};
-use rand_pcg::Pcg32;
 use roaring::RoaringBitmap;
 use rstar::RTree;
 use time::OffsetDateTime;
 
-use crate::distance::DotProduct;
 use crate::error::{InternalError, UserError};
 use crate::facet::FacetType;
 use crate::fields_ids_map::FieldsIdsMap;
@@ -21,18 +19,13 @@ use crate::heed_codec::facet::{
     FacetGroupKeyCodec, FacetGroupValueCodec, FieldDocIdFacetF64Codec, FieldDocIdFacetStringCodec,
     FieldIdCodec, OrderedF64Codec,
 };
-use crate::heed_codec::{FstSetCodec, ScriptLanguageCodec, StrBEU16Codec, StrRefCodec};
-use crate::readable_slices::ReadableSlices;
+use crate::heed_codec::{ScriptLanguageCodec, StrBEU16Codec, StrRefCodec};
 use crate::{
     default_criteria, CboRoaringBitmapCodec, Criterion, DocumentId, ExternalDocumentsIds,
     FacetDistribution, FieldDistribution, FieldId, FieldIdWordCountCodec, GeoPoint, ObkvCodec,
-    OrderBy, Result, RoaringBitmapCodec, RoaringBitmapLenCodec, Search, U8StrStrCodec, BEU16,
-    BEU32,
+    Result, RoaringBitmapCodec, RoaringBitmapLenCodec, Search, U8StrStrCodec, BEU16, BEU32,
 };
 
-/// The HNSW data-structure that we serialize, fill and search in.
-pub type Hnsw = hnsw::Hnsw<DotProduct, Vec<f32>, Pcg32, 12, 24>;
-
 pub const DEFAULT_MIN_WORD_LEN_ONE_TYPO: u8 = 5;
 pub const DEFAULT_MIN_WORD_LEN_TWO_TYPOS: u8 = 9;
 
@@ -49,10 +42,6 @@ pub mod main_key {
     pub const FIELDS_IDS_MAP_KEY: &str = "fields-ids-map";
     pub const GEO_FACETED_DOCUMENTS_IDS_KEY: &str = "geo-faceted-documents-ids";
     pub const GEO_RTREE_KEY: &str = "geo-rtree";
-    /// The prefix of the key that is used to store the, potential big, HNSW structure.
-    /// It is concatenated with a big-endian encoded number (non-human readable).
-    /// e.g. vector-hnsw0x0032.
-    pub const VECTOR_HNSW_KEY_PREFIX: &str = "vector-hnsw";
     pub const HARD_EXTERNAL_DOCUMENTS_IDS_KEY: &str = "hard-external-documents-ids";
     pub const NUMBER_FACETED_DOCUMENTS_IDS_PREFIX: &str = "number-faceted-documents-ids";
     pub const PRIMARY_KEY_KEY: &str = "primary-key";
@@ -60,9 +49,6 @@ pub mod main_key {
     pub const USER_DEFINED_SEARCHABLE_FIELDS_KEY: &str = "user-defined-searchable-fields";
     pub const SOFT_EXTERNAL_DOCUMENTS_IDS_KEY: &str = "soft-external-documents-ids";
     pub const STOP_WORDS_KEY: &str = "stop-words";
-    pub const NON_SEPARATOR_TOKENS_KEY: &str = "non-separator-tokens";
-    pub const SEPARATOR_TOKENS_KEY: &str = "separator-tokens";
-    pub const DICTIONARY_KEY: &str = "dictionary";
     pub const STRING_FACETED_DOCUMENTS_IDS_PREFIX: &str = "string-faceted-documents-ids";
     pub const SYNONYMS_KEY: &str = "synonyms";
     pub const WORDS_FST_KEY: &str = "words-fst";
@@ -75,7 +61,6 @@ pub mod main_key {
     pub const EXACT_WORDS: &str = "exact-words";
     pub const EXACT_ATTRIBUTES: &str = "exact-attributes";
     pub const MAX_VALUES_PER_FACET: &str = "max-values-per-facet";
-    pub const SORT_FACET_VALUES_BY: &str = "sort-facet-values-by";
     pub const PAGINATION_MAX_TOTAL_HITS: &str = "pagination-max-total-hits";
 }
 
@@ -99,10 +84,8 @@ pub mod db_name {
     pub const FACET_ID_IS_NULL_DOCIDS: &str = "facet-id-is-null-docids";
     pub const FACET_ID_IS_EMPTY_DOCIDS: &str = "facet-id-is-empty-docids";
     pub const FACET_ID_STRING_DOCIDS: &str = "facet-id-string-docids";
-    pub const FACET_ID_STRING_FST: &str = "facet-id-string-fst";
     pub const FIELD_ID_DOCID_FACET_F64S: &str = "field-id-docid-facet-f64s";
     pub const FIELD_ID_DOCID_FACET_STRINGS: &str = "field-id-docid-facet-strings";
-    pub const VECTOR_ID_DOCID: &str = "vector-id-docids";
     pub const DOCUMENTS: &str = "documents";
     pub const SCRIPT_LANGUAGE_DOCIDS: &str = "script_language_docids";
 }
@@ -160,17 +143,12 @@ pub struct Index {
     pub facet_id_f64_docids: Database<FacetGroupKeyCodec<OrderedF64Codec>, FacetGroupValueCodec>,
     /// Maps the facet field id and ranges of strings with the docids that corresponds to them.
     pub facet_id_string_docids: Database<FacetGroupKeyCodec<StrRefCodec>, FacetGroupValueCodec>,
-    /// Maps the facet field id of the string facets with an FST containing all the facets values.
-    pub facet_id_string_fst: Database<OwnedType<BEU16>, FstSetCodec>,
 
     /// Maps the document id, the facet field id and the numbers.
     pub field_id_docid_facet_f64s: Database<FieldDocIdFacetF64Codec, Unit>,
     /// Maps the document id, the facet field id and the strings.
     pub field_id_docid_facet_strings: Database<FieldDocIdFacetStringCodec, Str>,
 
-    /// Maps a vector id to the document id that have it.
-    pub vector_id_docid: Database<OwnedType<BEU32>, OwnedType<BEU32>>,
-
     /// Maps the document id to the document as an obkv store.
     pub(crate) documents: Database<OwnedType<BEU32>, ObkvCodec>,
 }
@@ -184,7 +162,7 @@ impl Index {
     ) -> Result<Index> {
         use db_name::*;
 
-        options.max_dbs(24);
+        options.max_dbs(23);
         unsafe { options.flag(Flags::MdbAlwaysFreePages) };
 
         let env = options.open(path)?;
@@ -214,18 +192,17 @@ impl Index {
         let facet_id_f64_docids = env.create_database(&mut wtxn, Some(FACET_ID_F64_DOCIDS))?;
         let facet_id_string_docids =
             env.create_database(&mut wtxn, Some(FACET_ID_STRING_DOCIDS))?;
-        let facet_id_string_fst = env.create_database(&mut wtxn, Some(FACET_ID_STRING_FST))?;
         let facet_id_exists_docids =
             env.create_database(&mut wtxn, Some(FACET_ID_EXISTS_DOCIDS))?;
         let facet_id_is_null_docids =
             env.create_database(&mut wtxn, Some(FACET_ID_IS_NULL_DOCIDS))?;
         let facet_id_is_empty_docids =
             env.create_database(&mut wtxn, Some(FACET_ID_IS_EMPTY_DOCIDS))?;
+
         let field_id_docid_facet_f64s =
             env.create_database(&mut wtxn, Some(FIELD_ID_DOCID_FACET_F64S))?;
         let field_id_docid_facet_strings =
             env.create_database(&mut wtxn, Some(FIELD_ID_DOCID_FACET_STRINGS))?;
-        let vector_id_docid = env.create_database(&mut wtxn, Some(VECTOR_ID_DOCID))?;
         let documents = env.create_database(&mut wtxn, Some(DOCUMENTS))?;
         wtxn.commit()?;
 
@@ -249,13 +226,11 @@ impl Index {
             field_id_word_count_docids,
             facet_id_f64_docids,
             facet_id_string_docids,
-            facet_id_string_fst,
             facet_id_exists_docids,
             facet_id_is_null_docids,
             facet_id_is_empty_docids,
             field_id_docid_facet_f64s,
             field_id_docid_facet_strings,
-            vector_id_docid,
             documents,
         })
     }
@@ -527,56 +502,6 @@ impl Index {
         }
     }
 
-    /* vector HNSW */
-
-    /// Writes the provided `hnsw`.
-    pub(crate) fn put_vector_hnsw(&self, wtxn: &mut RwTxn, hnsw: &Hnsw) -> heed::Result<()> {
-        // We must delete all the chunks before we write the new HNSW chunks.
-        self.delete_vector_hnsw(wtxn)?;
-
-        let chunk_size = 1024 * 1024 * (1024 + 512); // 1.5 GiB
-        let bytes = bincode::serialize(hnsw).map_err(|_| heed::Error::Encoding)?;
-        for (i, chunk) in bytes.chunks(chunk_size).enumerate() {
-            let i = i as u32;
-            let mut key = main_key::VECTOR_HNSW_KEY_PREFIX.as_bytes().to_vec();
-            key.extend_from_slice(&i.to_be_bytes());
-            self.main.put::<_, ByteSlice, ByteSlice>(wtxn, &key, chunk)?;
-        }
-        Ok(())
-    }
-
-    /// Delete the `hnsw`.
-    pub(crate) fn delete_vector_hnsw(&self, wtxn: &mut RwTxn) -> heed::Result<bool> {
-        let mut iter = self.main.prefix_iter_mut::<_, ByteSlice, DecodeIgnore>(
-            wtxn,
-            main_key::VECTOR_HNSW_KEY_PREFIX.as_bytes(),
-        )?;
-        let mut deleted = false;
-        while iter.next().transpose()?.is_some() {
-            // We do not keep a reference to the key or the value.
-            unsafe { deleted |= iter.del_current()? };
-        }
-        Ok(deleted)
-    }
-
-    /// Returns the `hnsw`.
-    pub fn vector_hnsw(&self, rtxn: &RoTxn) -> Result<Option<Hnsw>> {
-        let mut slices = Vec::new();
-        for result in
-            self.main.prefix_iter::<_, Str, ByteSlice>(rtxn, main_key::VECTOR_HNSW_KEY_PREFIX)?
-        {
-            let (_, slice) = result?;
-            slices.push(slice);
-        }
-
-        if slices.is_empty() {
-            Ok(None)
-        } else {
-            let readable_slices: ReadableSlices<_> = slices.into_iter().collect();
-            Ok(Some(bincode::deserialize_from(readable_slices).map_err(|_| heed::Error::Decoding)?))
-        }
-    }
-
     /* field distribution */
 
     /// Writes the field distribution which associates every field name with
@@ -1051,87 +976,6 @@ impl Index {
         }
     }
 
-    /* non separator tokens */
-
-    pub(crate) fn put_non_separator_tokens(
-        &self,
-        wtxn: &mut RwTxn,
-        set: &BTreeSet<String>,
-    ) -> heed::Result<()> {
-        self.main.put::<_, Str, SerdeBincode<_>>(wtxn, main_key::NON_SEPARATOR_TOKENS_KEY, set)
-    }
-
-    pub(crate) fn delete_non_separator_tokens(&self, wtxn: &mut RwTxn) -> heed::Result<bool> {
-        self.main.delete::<_, Str>(wtxn, main_key::NON_SEPARATOR_TOKENS_KEY)
-    }
-
-    pub fn non_separator_tokens(&self, rtxn: &RoTxn) -> Result<Option<BTreeSet<String>>> {
-        Ok(self.main.get::<_, Str, SerdeBincode<BTreeSet<String>>>(
-            rtxn,
-            main_key::NON_SEPARATOR_TOKENS_KEY,
-        )?)
-    }
-
-    /* separator tokens */
-
-    pub(crate) fn put_separator_tokens(
-        &self,
-        wtxn: &mut RwTxn,
-        set: &BTreeSet<String>,
-    ) -> heed::Result<()> {
-        self.main.put::<_, Str, SerdeBincode<_>>(wtxn, main_key::SEPARATOR_TOKENS_KEY, set)
-    }
-
-    pub(crate) fn delete_separator_tokens(&self, wtxn: &mut RwTxn) -> heed::Result<bool> {
-        self.main.delete::<_, Str>(wtxn, main_key::SEPARATOR_TOKENS_KEY)
-    }
-
-    pub fn separator_tokens(&self, rtxn: &RoTxn) -> Result<Option<BTreeSet<String>>> {
-        Ok(self
-            .main
-            .get::<_, Str, SerdeBincode<BTreeSet<String>>>(rtxn, main_key::SEPARATOR_TOKENS_KEY)?)
-    }
-
-    /* separators easing method */
-
-    pub fn allowed_separators(&self, rtxn: &RoTxn) -> Result<Option<BTreeSet<String>>> {
-        let default_separators =
-            charabia::separators::DEFAULT_SEPARATORS.iter().map(|s| s.to_string());
-        let mut separators: Option<BTreeSet<_>> = None;
-        if let Some(mut separator_tokens) = self.separator_tokens(rtxn)? {
-            separator_tokens.extend(default_separators.clone());
-            separators = Some(separator_tokens);
-        }
-
-        if let Some(non_separator_tokens) = self.non_separator_tokens(rtxn)? {
-            separators = separators
-                .or_else(|| Some(default_separators.collect()))
-                .map(|separators| &separators - &non_separator_tokens);
-        }
-
-        Ok(separators)
-    }
-
-    /* dictionary */
-
-    pub(crate) fn put_dictionary(
-        &self,
-        wtxn: &mut RwTxn,
-        set: &BTreeSet<String>,
-    ) -> heed::Result<()> {
-        self.main.put::<_, Str, SerdeBincode<_>>(wtxn, main_key::DICTIONARY_KEY, set)
-    }
-
-    pub(crate) fn delete_dictionary(&self, wtxn: &mut RwTxn) -> heed::Result<bool> {
-        self.main.delete::<_, Str>(wtxn, main_key::DICTIONARY_KEY)
-    }
-
-    pub fn dictionary(&self, rtxn: &RoTxn) -> Result<Option<BTreeSet<String>>> {
-        Ok(self
-            .main
-            .get::<_, Str, SerdeBincode<BTreeSet<String>>>(rtxn, main_key::DICTIONARY_KEY)?)
-    }
-
     /* synonyms */
 
     pub(crate) fn put_synonyms(
@@ -1384,31 +1228,6 @@ impl Index {
         self.main.delete::<_, Str>(txn, main_key::MAX_VALUES_PER_FACET)
     }
 
-    pub fn sort_facet_values_by(&self, txn: &RoTxn) -> heed::Result<HashMap<String, OrderBy>> {
-        let mut orders = self
-            .main
-            .get::<_, Str, SerdeJson<HashMap<String, OrderBy>>>(
-                txn,
-                main_key::SORT_FACET_VALUES_BY,
-            )?
-            .unwrap_or_default();
-        // Insert the default ordering if it is not already overwritten by the user.
-        orders.entry("*".to_string()).or_insert(OrderBy::Lexicographic);
-        Ok(orders)
-    }
-
-    pub(crate) fn put_sort_facet_values_by(
-        &self,
-        txn: &mut RwTxn,
-        val: &HashMap<String, OrderBy>,
-    ) -> heed::Result<()> {
-        self.main.put::<_, Str, SerdeJson<_>>(txn, main_key::SORT_FACET_VALUES_BY, &val)
-    }
-
-    pub(crate) fn delete_sort_facet_values_by(&self, txn: &mut RwTxn) -> heed::Result<bool> {
-        self.main.delete::<_, Str>(txn, main_key::SORT_FACET_VALUES_BY)
-    }
-
     pub fn pagination_max_total_hits(&self, txn: &RoTxn) -> heed::Result<Option<usize>> {
         self.main.get::<_, Str, OwnedType<usize>>(txn, main_key::PAGINATION_MAX_TOTAL_HITS)
     }
@@ -1647,9 +1466,9 @@ pub(crate) mod tests {
 
         db_snap!(index, field_distribution,
             @r###"
-        age              1      |
-        id               2      |
-        name             2      |
+        age              1     
+        id               2     
+        name             2     
         "###
         );
 
@@ -1667,9 +1486,9 @@ pub(crate) mod tests {
 
         db_snap!(index, field_distribution,
             @r###"
-        age              1      |
-        id               2      |
-        name             2      |
+        age              1     
+        id               2     
+        name             2     
         "###
         );
 
@@ -1683,9 +1502,9 @@ pub(crate) mod tests {
 
         db_snap!(index, field_distribution,
             @r###"
-        has_dog          1      |
-        id               2      |
-        name             2      |
+        has_dog          1     
+        id               2     
+        name             2     
         "###
         );
     }

milli/src/lib.rs

@@ -10,7 +10,6 @@ pub mod documents;
 
 mod asc_desc;
 mod criterion;
-pub mod distance;
 mod error;
 mod external_documents_ids;
 pub mod facet;
@@ -18,7 +17,6 @@ mod fields_ids_map;
 pub mod heed_codec;
 pub mod index;
 pub mod proximity;
-mod readable_slices;
 pub mod score_details;
 mod search;
 pub mod update;
@@ -32,7 +30,6 @@ use std::convert::{TryFrom, TryInto};
 use std::hash::BuildHasherDefault;
 
 use charabia::normalizer::{CharNormalizer, CompatibilityDecompositionNormalizer};
-pub use distance::dot_product_similarity;
 pub use filter_parser::{Condition, FilterCondition, Span, Token};
 use fxhash::{FxHasher32, FxHasher64};
 pub use grenad::CompressionType;
@@ -57,9 +54,8 @@ pub use self::heed_codec::{
 };
 pub use self::index::Index;
 pub use self::search::{
-    FacetDistribution, FacetValueHit, Filter, FormatOptions, MatchBounds, MatcherBuilder,
-    MatchingWords, OrderBy, Search, SearchForFacetValues, SearchResult, TermsMatchingStrategy,
-    DEFAULT_VALUES_PER_FACET,
+    FacetDistribution, Filter, FormatOptions, MatchBounds, MatcherBuilder, MatchingWords, Search,
+    SearchResult, TermsMatchingStrategy, DEFAULT_VALUES_PER_FACET,
 };
 
 pub type Result<T> = std::result::Result<T, error::Error>;
@@ -288,35 +284,6 @@ pub fn normalize_facet(original: &str) -> String {
     CompatibilityDecompositionNormalizer.normalize_str(original.trim()).to_lowercase()
 }
 
-/// Represents either a vector or an array of multiple vectors.
-#[derive(serde::Serialize, serde::Deserialize, Debug)]
-#[serde(transparent)]
-pub struct VectorOrArrayOfVectors {
-    #[serde(with = "either::serde_untagged")]
-    inner: either::Either<Vec<f32>, Vec<Vec<f32>>>,
-}
-
-impl VectorOrArrayOfVectors {
-    pub fn into_array_of_vectors(self) -> Vec<Vec<f32>> {
-        match self.inner {
-            either::Either::Left(vector) => vec![vector],
-            either::Either::Right(vectors) => vectors,
-        }
-    }
-}
-
-/// Normalize a vector by dividing the dimensions by the length of it.
-pub fn normalize_vector(mut vector: Vec<f32>) -> Vec<f32> {
-    let squared: f32 = vector.iter().map(|x| x * x).sum();
-    let length = squared.sqrt();
-    if length <= f32::EPSILON {
-        vector
-    } else {
-        vector.iter_mut().for_each(|x| *x /= length);
-        vector
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use serde_json::json;

milli/src/readable_slices.rs

@@ -1,85 +0,0 @@
-use std::io::{self, Read};
-use std::iter::FromIterator;
-
-pub struct ReadableSlices<A> {
-    inner: Vec<A>,
-    pos: u64,
-}
-
-impl<A> FromIterator<A> for ReadableSlices<A> {
-    fn from_iter<T: IntoIterator<Item = A>>(iter: T) -> Self {
-        ReadableSlices { inner: iter.into_iter().collect(), pos: 0 }
-    }
-}
-
-impl<A: AsRef<[u8]>> Read for ReadableSlices<A> {
-    fn read(&mut self, mut buf: &mut [u8]) -> io::Result<usize> {
-        let original_buf_len = buf.len();
-
-        // We explore the list of slices to find the one where we must start reading.
-        let mut pos = self.pos;
-        let index = match self
-            .inner
-            .iter()
-            .map(|s| s.as_ref().len() as u64)
-            .position(|size| pos.checked_sub(size).map(|p| pos = p).is_none())
-        {
-            Some(index) => index,
-            None => return Ok(0),
-        };
-
-        let mut inner_pos = pos as usize;
-        for slice in &self.inner[index..] {
-            let slice = &slice.as_ref()[inner_pos..];
-
-            if buf.len() > slice.len() {
-                // We must exhaust the current slice and go to the next one there is not enough here.
-                buf[..slice.len()].copy_from_slice(slice);
-                buf = &mut buf[slice.len()..];
-                inner_pos = 0;
-            } else {
-                // There is enough in this slice to fill the remaining bytes of the buffer.
-                // Let's break just after filling it.
-                buf.copy_from_slice(&slice[..buf.len()]);
-                buf = &mut [];
-                break;
-            }
-        }
-
-        let written = original_buf_len - buf.len();
-        self.pos += written as u64;
-        Ok(written)
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use std::io::Read;
-
-    use super::ReadableSlices;
-
-    #[test]
-    fn basic() {
-        let data: Vec<_> = (0..100).collect();
-        let splits: Vec<_> = data.chunks(3).collect();
-        let mut rdslices: ReadableSlices<_> = splits.into_iter().collect();
-
-        let mut output = Vec::new();
-        let length = rdslices.read_to_end(&mut output).unwrap();
-        assert_eq!(length, data.len());
-        assert_eq!(output, data);
-    }
-
-    #[test]
-    fn small_reads() {
-        let data: Vec<_> = (0..u8::MAX).collect();
-        let splits: Vec<_> = data.chunks(27).collect();
-        let mut rdslices: ReadableSlices<_> = splits.into_iter().collect();
-
-        let buffer = &mut [0; 45];
-        let length = rdslices.read(buffer).unwrap();
-        let expected: Vec<_> = (0..buffer.len() as u8).collect();
-        assert_eq!(length, buffer.len());
-        assert_eq!(buffer, &expected[..]);
-    }
-}

milli/src/score_details.rs

@@ -38,15 +38,19 @@ impl ScoreDetails {
         Rank::global_score(details.filter_map(Self::rank))
     }
 
+    pub fn global_score_linear_scale<'a>(details: impl Iterator<Item = &'a Self>) -> u64 {
+        (Self::global_score(details) * LINEAR_SCALE_FACTOR).round() as u64
+    }
+
     /// Panics
     ///
     /// - If Position is not preceded by Fid
     /// - If Exactness is not preceded by ExactAttribute
+    /// - If a sort fid is not contained in the passed `fields_ids_map`.
     pub fn to_json_map<'a>(
         details: impl Iterator<Item = &'a Self>,
     ) -> serde_json::Map<String, serde_json::Value> {
         let mut order = 0;
-        let mut fid_details = None;
         let mut details_map = serde_json::Map::default();
         for details in details {
             match details {
@@ -55,7 +59,7 @@ impl ScoreDetails {
                             "order": order,
                             "matchingWords": words.matching_words,
                             "maxMatchingWords": words.max_matching_words,
-                            "score": words.rank().local_score(),
+                            "score": words.rank().local_score_linear_scale(),
                     });
                     details_map.insert("words".into(), words_details);
                     order += 1;
@@ -65,7 +69,7 @@ impl ScoreDetails {
                         "order": order,
                         "typoCount": typo.typo_count,
                         "maxTypoCount": typo.max_typo_count,
-                        "score": typo.rank().local_score(),
+                        "score": typo.rank().local_score_linear_scale(),
                     });
                     details_map.insert("typo".into(), typo_details);
                     order += 1;
@@ -73,18 +77,16 @@ impl ScoreDetails {
                 ScoreDetails::Proximity(proximity) => {
                     let proximity_details = serde_json::json!({
                         "order": order,
-                        "score": proximity.local_score(),
+                        "score": proximity.local_score_linear_scale(),
                     });
                     details_map.insert("proximity".into(), proximity_details);
                     order += 1;
                 }
                 ScoreDetails::Fid(fid) => {
-                    // copy the rank for future use in Position.
-                    fid_details = Some(*fid);
                     // For now, fid is a virtual rule always followed by the "position" rule
                     let fid_details = serde_json::json!({
                         "order": order,
-                        "attribute_ranking_order_score": fid.local_score(),
+                        "attributes_ranking_order": fid.local_score_linear_scale(),
                     });
                     details_map.insert("attribute".into(), fid_details);
                     order += 1;
@@ -97,22 +99,17 @@ impl ScoreDetails {
                     let attribute_details = attribute_details
                         .as_object_mut()
                         .expect("attribute details was not an object");
-                    let Some(fid_details) = fid_details else {
-                        unimplemented!("position not preceded by attribute");
-                    };
-
-                    attribute_details
-                        .insert("query_word_distance_score".into(), position.local_score().into());
-                    let score = Rank::global_score([fid_details, *position].iter().copied());
-                    attribute_details.insert("score".into(), score.into());
-
+                    attribute_details.insert(
+                        "attributes_query_word_order".into(),
+                        position.local_score_linear_scale().into(),
+                    );
                     // do not update the order since this was already done by fid
                 }
                 ScoreDetails::ExactAttribute(exact_attribute) => {
                     let exactness_details = serde_json::json!({
                         "order": order,
-                        "matchType": exact_attribute,
-                        "score": exact_attribute.rank().local_score(),
+                        "exactIn": exact_attribute,
+                        "score": exact_attribute.rank().local_score_linear_scale(),
                     });
                     details_map.insert("exactness".into(), exactness_details);
                     order += 1;
@@ -125,10 +122,10 @@ impl ScoreDetails {
                     let exactness_details = exactness_details
                         .as_object_mut()
                         .expect("exactness details was not an object");
-                    if exactness_details.get("matchType").expect("missing 'matchType'")
+                    if exactness_details.get("exactIn").expect("missing 'exactIn'")
                         == &serde_json::json!(ExactAttribute::NoExactMatch)
                     {
-                        let score = Rank::global_score(
+                        let score = Rank::global_score_linear_scale(
                             [ExactAttribute::NoExactMatch.rank(), *details].iter().copied(),
                         );
                         *exactness_details.get_mut("score").expect("missing score") = score.into();
@@ -136,20 +133,14 @@ impl ScoreDetails {
                     // do not update the order since this was already done by exactAttribute
                 }
                 ScoreDetails::Sort(details) => {
-                    let sort = if details.redacted {
-                        format!("<hidden-rule-{order}>")
-                    } else {
-                        format!(
-                            "{}:{}",
-                            details.field_name,
-                            if details.ascending { "asc" } else { "desc" }
-                        )
-                    };
-                    let value =
-                        if details.redacted { "<hidden>".into() } else { details.value.clone() };
+                    let sort = format!(
+                        "{}:{}",
+                        details.field_name,
+                        if details.ascending { "asc" } else { "desc" }
+                    );
                     let sort_details = serde_json::json!({
                         "order": order,
-                        "value": value,
+                        "value": details.value,
                     });
                     details_map.insert(sort, sort_details);
                     order += 1;
@@ -180,24 +171,6 @@ impl ScoreDetails {
     }
 }
 
-/// The strategy to compute scores.
-///
-/// It makes sense to pass down this strategy to the internals of the search, because
-/// some optimizations (today, mainly skipping ranking rules for universes of a single document)
-/// are not correct to do when computing the scores.
-///
-/// This strategy could feasibly be extended to differentiate between the normalized score and the
-/// detailed scores, but it is not useful today as the normalized score is *derived from* the
-/// detailed scores.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
-pub enum ScoringStrategy {
-    /// Don't compute scores
-    #[default]
-    Skip,
-    /// Compute detailed scores
-    Detailed,
-}
-
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct Words {
     pub matching_words: u32,
@@ -258,6 +231,10 @@ impl Rank {
         self.rank as f64 / self.max_rank as f64
     }
 
+    pub fn local_score_linear_scale(self) -> u64 {
+        (self.local_score() * LINEAR_SCALE_FACTOR).round() as u64
+    }
+
     pub fn global_score(details: impl Iterator<Item = Self>) -> f64 {
         let mut rank = Rank { rank: 1, max_rank: 1 };
         for inner_rank in details {
@@ -270,12 +247,16 @@ impl Rank {
         }
         rank.local_score()
     }
+
+    pub fn global_score_linear_scale(details: impl Iterator<Item = Self>) -> u64 {
+        (Self::global_score(details) * LINEAR_SCALE_FACTOR).round() as u64
+    }
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub enum ExactAttribute {
-    ExactMatch,
+    MatchesFull,
     MatchesStart,
     NoExactMatch,
 }
@@ -283,7 +264,7 @@ pub enum ExactAttribute {
 impl ExactAttribute {
     pub fn rank(&self) -> Rank {
         let rank = match self {
-            ExactAttribute::ExactMatch => 3,
+            ExactAttribute::MatchesFull => 3,
             ExactAttribute::MatchesStart => 2,
             ExactAttribute::NoExactMatch => 1,
         };
@@ -295,7 +276,6 @@ impl ExactAttribute {
 pub struct Sort {
     pub field_name: String,
     pub ascending: bool,
-    pub redacted: bool,
     pub value: serde_json::Value,
 }
 
@@ -311,3 +291,5 @@ impl GeoSort {
         self.value.map(|value| distance_between_two_points(&self.target_point, &value))
     }
 }
+
+const LINEAR_SCALE_FACTOR: f64 = 1000.0;

milli/src/search/facet/facet_distribution.rs

@@ -1,22 +1,19 @@
-use std::collections::{BTreeMap, HashMap, HashSet};
+use std::collections::{BTreeMap, HashSet};
 use std::ops::ControlFlow;
 use std::{fmt, mem};
 
 use heed::types::ByteSlice;
 use heed::BytesDecode;
-use indexmap::IndexMap;
 use roaring::RoaringBitmap;
-use serde::{Deserialize, Serialize};
 
 use crate::error::UserError;
 use crate::facet::FacetType;
 use crate::heed_codec::facet::{
-    FacetGroupKeyCodec, FieldDocIdFacetF64Codec, FieldDocIdFacetStringCodec, OrderedF64Codec,
+    FacetGroupKeyCodec, FacetGroupValueCodec, FieldDocIdFacetF64Codec, FieldDocIdFacetStringCodec,
+    OrderedF64Codec,
 };
 use crate::heed_codec::{ByteSliceRefCodec, StrRefCodec};
-use crate::search::facet::facet_distribution_iter::{
-    count_iterate_over_facet_distribution, lexicographically_iterate_over_facet_distribution,
-};
+use crate::search::facet::facet_distribution_iter;
 use crate::{FieldId, Index, Result};
 
 /// The default number of values by facets that will
@@ -27,21 +24,10 @@ pub const DEFAULT_VALUES_PER_FACET: usize = 100;
 /// the system to choose between one algorithm or another.
 const CANDIDATES_THRESHOLD: u64 = 3000;
 
-/// How should we fetch the facets?
-#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
-pub enum OrderBy {
-    /// By lexicographic order...
-    #[default]
-    Lexicographic,
-    /// Or by number of docids in common?
-    Count,
-}
-
 pub struct FacetDistribution<'a> {
-    facets: Option<HashMap<String, OrderBy>>,
+    facets: Option<HashSet<String>>,
     candidates: Option<RoaringBitmap>,
     max_values_per_facet: usize,
-    default_order_by: OrderBy,
     rtxn: &'a heed::RoTxn<'a>,
     index: &'a Index,
 }
@@ -52,22 +38,13 @@ impl<'a> FacetDistribution<'a> {
             facets: None,
             candidates: None,
             max_values_per_facet: DEFAULT_VALUES_PER_FACET,
-            default_order_by: OrderBy::default(),
             rtxn,
             index,
         }
     }
 
-    pub fn facets<I: IntoIterator<Item = (A, OrderBy)>, A: AsRef<str>>(
-        &mut self,
-        names_ordered_by: I,
-    ) -> &mut Self {
-        self.facets = Some(
-            names_ordered_by
-                .into_iter()
-                .map(|(name, order_by)| (name.as_ref().to_string(), order_by))
-                .collect(),
-        );
+    pub fn facets<I: IntoIterator<Item = A>, A: AsRef<str>>(&mut self, names: I) -> &mut Self {
+        self.facets = Some(names.into_iter().map(|s| s.as_ref().to_string()).collect());
         self
     }
 
@@ -76,11 +53,6 @@ impl<'a> FacetDistribution<'a> {
         self
     }
 
-    pub fn default_order_by(&mut self, order_by: OrderBy) -> &mut Self {
-        self.default_order_by = order_by;
-        self
-    }
-
     pub fn candidates(&mut self, candidates: RoaringBitmap) -> &mut Self {
         self.candidates = Some(candidates);
         self
@@ -93,7 +65,7 @@ impl<'a> FacetDistribution<'a> {
         field_id: FieldId,
         facet_type: FacetType,
         candidates: &RoaringBitmap,
-        distribution: &mut IndexMap<String, u64>,
+        distribution: &mut BTreeMap<String, u64>,
     ) -> heed::Result<()> {
         match facet_type {
             FacetType::Number => {
@@ -162,15 +134,9 @@ impl<'a> FacetDistribution<'a> {
         &self,
         field_id: FieldId,
         candidates: &RoaringBitmap,
-        order_by: OrderBy,
-        distribution: &mut IndexMap<String, u64>,
+        distribution: &mut BTreeMap<String, u64>,
     ) -> heed::Result<()> {
-        let search_function = match order_by {
-            OrderBy::Lexicographic => lexicographically_iterate_over_facet_distribution,
-            OrderBy::Count => count_iterate_over_facet_distribution,
-        };
-
-        search_function(
+        facet_distribution_iter::iterate_over_facet_distribution(
             self.rtxn,
             self.index
                 .facet_id_f64_docids
@@ -193,15 +159,9 @@ impl<'a> FacetDistribution<'a> {
         &self,
         field_id: FieldId,
         candidates: &RoaringBitmap,
-        order_by: OrderBy,
-        distribution: &mut IndexMap<String, u64>,
+        distribution: &mut BTreeMap<String, u64>,
     ) -> heed::Result<()> {
-        let search_function = match order_by {
-            OrderBy::Lexicographic => lexicographically_iterate_over_facet_distribution,
-            OrderBy::Count => count_iterate_over_facet_distribution,
-        };
-
-        search_function(
+        facet_distribution_iter::iterate_over_facet_distribution(
             self.rtxn,
             self.index
                 .facet_id_string_docids
@@ -229,49 +189,95 @@ impl<'a> FacetDistribution<'a> {
         )
     }
 
-    fn facet_values(
+    /// Placeholder search, a.k.a. no candidates were specified. We iterate throught the
+    /// facet values one by one and iterate on the facet level 0 for numbers.
+    fn facet_values_from_raw_facet_database(
         &self,
         field_id: FieldId,
-        order_by: OrderBy,
-    ) -> heed::Result<IndexMap<String, u64>> {
-        use FacetType::{Number, String};
+    ) -> heed::Result<BTreeMap<String, u64>> {
+        let mut distribution = BTreeMap::new();
 
-        let mut distribution = IndexMap::new();
-        match (order_by, &self.candidates) {
-            (OrderBy::Lexicographic, Some(cnd)) if cnd.len() <= CANDIDATES_THRESHOLD => {
-                // Classic search, candidates were specified, we must return facet values only related
-                // to those candidates. We also enter here for facet strings for performance reasons.
-                self.facet_distribution_from_documents(field_id, Number, cnd, &mut distribution)?;
-                self.facet_distribution_from_documents(field_id, String, cnd, &mut distribution)?;
-            }
-            _ => {
-                let universe;
-                let candidates = match &self.candidates {
-                    Some(cnd) => cnd,
-                    None => {
-                        universe = self.index.documents_ids(self.rtxn)?;
-                        &universe
-                    }
-                };
+        let db = self.index.facet_id_f64_docids;
+        let mut prefix = vec![];
+        prefix.extend_from_slice(&field_id.to_be_bytes());
+        prefix.push(0); // read values from level 0 only
 
-                self.facet_numbers_distribution_from_facet_levels(
-                    field_id,
-                    candidates,
-                    order_by,
-                    &mut distribution,
-                )?;
-                self.facet_strings_distribution_from_facet_levels(
-                    field_id,
-                    candidates,
-                    order_by,
-                    &mut distribution,
-                )?;
+        let iter = db
+            .as_polymorph()
+            .prefix_iter::<_, ByteSlice, ByteSlice>(self.rtxn, prefix.as_slice())?
+            .remap_types::<FacetGroupKeyCodec<OrderedF64Codec>, FacetGroupValueCodec>();
+
+        for result in iter {
+            let (key, value) = result?;
+            distribution.insert(key.left_bound.to_string(), value.bitmap.len());
+            if distribution.len() == self.max_values_per_facet {
+                break;
             }
-        };
+        }
+
+        let iter = self
+            .index
+            .facet_id_string_docids
+            .as_polymorph()
+            .prefix_iter::<_, ByteSlice, ByteSlice>(self.rtxn, prefix.as_slice())?
+            .remap_types::<FacetGroupKeyCodec<StrRefCodec>, FacetGroupValueCodec>();
+
+        for result in iter {
+            let (key, value) = result?;
+
+            let docid = value.bitmap.iter().next().unwrap();
+            let key: (FieldId, _, &'a str) = (field_id, docid, key.left_bound);
+            let original_string =
+                self.index.field_id_docid_facet_strings.get(self.rtxn, &key)?.unwrap().to_owned();
+
+            distribution.insert(original_string, value.bitmap.len());
+            if distribution.len() == self.max_values_per_facet {
+                break;
+            }
+        }
 
         Ok(distribution)
     }
 
+    fn facet_values(&self, field_id: FieldId) -> heed::Result<BTreeMap<String, u64>> {
+        use FacetType::{Number, String};
+
+        match self.candidates {
+            Some(ref candidates) => {
+                // Classic search, candidates were specified, we must return facet values only related
+                // to those candidates. We also enter here for facet strings for performance reasons.
+                let mut distribution = BTreeMap::new();
+                if candidates.len() <= CANDIDATES_THRESHOLD {
+                    self.facet_distribution_from_documents(
+                        field_id,
+                        Number,
+                        candidates,
+                        &mut distribution,
+                    )?;
+                    self.facet_distribution_from_documents(
+                        field_id,
+                        String,
+                        candidates,
+                        &mut distribution,
+                    )?;
+                } else {
+                    self.facet_numbers_distribution_from_facet_levels(
+                        field_id,
+                        candidates,
+                        &mut distribution,
+                    )?;
+                    self.facet_strings_distribution_from_facet_levels(
+                        field_id,
+                        candidates,
+                        &mut distribution,
+                    )?;
+                }
+                Ok(distribution)
+            }
+            None => self.facet_values_from_raw_facet_database(field_id),
+        }
+    }
+
     pub fn compute_stats(&self) -> Result<BTreeMap<String, (f64, f64)>> {
         let fields_ids_map = self.index.fields_ids_map(self.rtxn)?;
         let filterable_fields = self.index.filterable_fields(self.rtxn)?;
@@ -285,7 +291,6 @@ impl<'a> FacetDistribution<'a> {
             Some(facets) => {
                 let invalid_fields: HashSet<_> = facets
                     .iter()
-                    .map(|(name, _)| name)
                     .filter(|facet| !crate::is_faceted(facet, &filterable_fields))
                     .collect();
                 if !invalid_fields.is_empty() {
@@ -295,7 +300,7 @@ impl<'a> FacetDistribution<'a> {
                     }
                     .into());
                 } else {
-                    facets.iter().map(|(name, _)| name).cloned().collect()
+                    facets.clone()
                 }
             }
             None => filterable_fields,
@@ -332,7 +337,7 @@ impl<'a> FacetDistribution<'a> {
         Ok(distribution)
     }
 
-    pub fn execute(&self) -> Result<BTreeMap<String, IndexMap<String, u64>>> {
+    pub fn execute(&self) -> Result<BTreeMap<String, BTreeMap<String, u64>>> {
         let fields_ids_map = self.index.fields_ids_map(self.rtxn)?;
         let filterable_fields = self.index.filterable_fields(self.rtxn)?;
 
@@ -340,7 +345,6 @@ impl<'a> FacetDistribution<'a> {
             Some(ref facets) => {
                 let invalid_fields: HashSet<_> = facets
                     .iter()
-                    .map(|(name, _)| name)
                     .filter(|facet| !crate::is_faceted(facet, &filterable_fields))
                     .collect();
                 if !invalid_fields.is_empty() {
@@ -350,7 +354,7 @@ impl<'a> FacetDistribution<'a> {
                     }
                     .into());
                 } else {
-                    facets.iter().map(|(name, _)| name).cloned().collect()
+                    facets.clone()
                 }
             }
             None => filterable_fields,
@@ -359,12 +363,7 @@ impl<'a> FacetDistribution<'a> {
         let mut distribution = BTreeMap::new();
         for (fid, name) in fields_ids_map.iter() {
             if crate::is_faceted(name, &fields) {
-                let order_by = self
-                    .facets
-                    .as_ref()
-                    .and_then(|facets| facets.get(name).copied())
-                    .unwrap_or(self.default_order_by);
-                let values = self.facet_values(fid, order_by)?;
+                let values = self.facet_values(fid)?;
                 distribution.insert(name.to_string(), values);
             }
         }
@@ -375,34 +374,25 @@ impl<'a> FacetDistribution<'a> {
 
 impl fmt::Debug for FacetDistribution<'_> {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let FacetDistribution {
-            facets,
-            candidates,
-            max_values_per_facet,
-            default_order_by,
-            rtxn: _,
-            index: _,
-        } = self;
+        let FacetDistribution { facets, candidates, max_values_per_facet, rtxn: _, index: _ } =
+            self;
 
         f.debug_struct("FacetDistribution")
             .field("facets", facets)
             .field("candidates", candidates)
             .field("max_values_per_facet", max_values_per_facet)
-            .field("default_order_by", default_order_by)
             .finish()
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use std::iter;
-
     use big_s::S;
     use maplit::hashset;
 
     use crate::documents::documents_batch_reader_from_objects;
     use crate::index::tests::TempIndex;
-    use crate::{milli_snap, FacetDistribution, OrderBy};
+    use crate::{milli_snap, FacetDistribution};
 
     #[test]
     fn few_candidates_few_facet_values() {
@@ -427,14 +417,14 @@ mod tests {
         let txn = index.read_txn().unwrap();
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .execute()
             .unwrap();
 
         milli_snap!(format!("{map:?}"), @r###"{"colour": {"Blue": 2, "RED": 1}}"###);
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .candidates([0, 1, 2].iter().copied().collect())
             .execute()
             .unwrap();
@@ -442,7 +432,7 @@ mod tests {
         milli_snap!(format!("{map:?}"), @r###"{"colour": {"Blue": 2, "RED": 1}}"###);
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .candidates([1, 2].iter().copied().collect())
             .execute()
             .unwrap();
@@ -453,7 +443,7 @@ mod tests {
         milli_snap!(format!("{map:?}"), @r###"{"colour": {"  blue": 1, "RED": 1}}"###);
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .candidates([2].iter().copied().collect())
             .execute()
             .unwrap();
@@ -461,22 +451,13 @@ mod tests {
         milli_snap!(format!("{map:?}"), @r###"{"colour": {"RED": 1}}"###);
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .candidates([0, 1, 2].iter().copied().collect())
             .max_values_per_facet(1)
             .execute()
             .unwrap();
 
         milli_snap!(format!("{map:?}"), @r###"{"colour": {"Blue": 1}}"###);
-
-        let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::Count)))
-            .candidates([0, 1, 2].iter().copied().collect())
-            .max_values_per_facet(1)
-            .execute()
-            .unwrap();
-
-        milli_snap!(format!("{map:?}"), @r###"{"colour": {"Blue": 2}}"###);
     }
 
     #[test]
@@ -508,14 +489,14 @@ mod tests {
         let txn = index.read_txn().unwrap();
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .execute()
             .unwrap();
 
         milli_snap!(format!("{map:?}"), @r###"{"colour": {"Blue": 4000, "Red": 6000}}"###);
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .max_values_per_facet(1)
             .execute()
             .unwrap();
@@ -523,7 +504,7 @@ mod tests {
         milli_snap!(format!("{map:?}"), @r###"{"colour": {"Blue": 4000}}"###);
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .candidates((0..10_000).collect())
             .execute()
             .unwrap();
@@ -531,7 +512,7 @@ mod tests {
         milli_snap!(format!("{map:?}"), @r###"{"colour": {"Blue": 4000, "Red": 6000}}"###);
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .candidates((0..5_000).collect())
             .execute()
             .unwrap();
@@ -539,7 +520,7 @@ mod tests {
         milli_snap!(format!("{map:?}"), @r###"{"colour": {"Blue": 2000, "Red": 3000}}"###);
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .candidates((0..5_000).collect())
             .execute()
             .unwrap();
@@ -547,22 +528,13 @@ mod tests {
         milli_snap!(format!("{map:?}"), @r###"{"colour": {"Blue": 2000, "Red": 3000}}"###);
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .candidates((0..5_000).collect())
             .max_values_per_facet(1)
             .execute()
             .unwrap();
 
         milli_snap!(format!("{map:?}"), @r###"{"colour": {"Blue": 2000}}"###);
-
-        let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::Count)))
-            .candidates((0..5_000).collect())
-            .max_values_per_facet(1)
-            .execute()
-            .unwrap();
-
-        milli_snap!(format!("{map:?}"), @r###"{"colour": {"Red": 3000}}"###);
     }
 
     #[test]
@@ -594,14 +566,14 @@ mod tests {
         let txn = index.read_txn().unwrap();
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .execute()
             .unwrap();
 
         milli_snap!(format!("{map:?}"), "no_candidates", @"ac9229ed5964d893af96a7076e2f8af5");
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .max_values_per_facet(2)
             .execute()
             .unwrap();
@@ -609,7 +581,7 @@ mod tests {
         milli_snap!(format!("{map:?}"), "no_candidates_with_max_2", @r###"{"colour": {"0": 10, "1": 10}}"###);
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .candidates((0..10_000).collect())
             .execute()
             .unwrap();
@@ -617,7 +589,7 @@ mod tests {
         milli_snap!(format!("{map:?}"), "candidates_0_10_000", @"ac9229ed5964d893af96a7076e2f8af5");
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .candidates((0..5_000).collect())
             .execute()
             .unwrap();
@@ -654,14 +626,14 @@ mod tests {
         let txn = index.read_txn().unwrap();
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .compute_stats()
             .unwrap();
 
         milli_snap!(format!("{map:?}"), "no_candidates", @"{}");
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .candidates((0..1000).collect())
             .compute_stats()
             .unwrap();
@@ -669,7 +641,7 @@ mod tests {
         milli_snap!(format!("{map:?}"), "candidates_0_1000", @r###"{"colour": (0.0, 999.0)}"###);
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .candidates((217..777).collect())
             .compute_stats()
             .unwrap();
@@ -706,14 +678,14 @@ mod tests {
         let txn = index.read_txn().unwrap();
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .compute_stats()
             .unwrap();
 
         milli_snap!(format!("{map:?}"), "no_candidates", @"{}");
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .candidates((0..1000).collect())
             .compute_stats()
             .unwrap();
@@ -721,7 +693,7 @@ mod tests {
         milli_snap!(format!("{map:?}"), "candidates_0_1000", @r###"{"colour": (0.0, 1999.0)}"###);
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .candidates((217..777).collect())
             .compute_stats()
             .unwrap();
@@ -758,14 +730,14 @@ mod tests {
         let txn = index.read_txn().unwrap();
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .compute_stats()
             .unwrap();
 
         milli_snap!(format!("{map:?}"), "no_candidates", @"{}");
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .candidates((0..1000).collect())
             .compute_stats()
             .unwrap();
@@ -773,7 +745,7 @@ mod tests {
         milli_snap!(format!("{map:?}"), "candidates_0_1000", @r###"{"colour": (0.0, 999.0)}"###);
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .candidates((217..777).collect())
             .compute_stats()
             .unwrap();
@@ -814,14 +786,14 @@ mod tests {
         let txn = index.read_txn().unwrap();
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .compute_stats()
             .unwrap();
 
         milli_snap!(format!("{map:?}"), "no_candidates", @"{}");
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .candidates((0..1000).collect())
             .compute_stats()
             .unwrap();
@@ -829,7 +801,7 @@ mod tests {
         milli_snap!(format!("{map:?}"), "candidates_0_1000", @r###"{"colour": (0.0, 1998.0)}"###);
 
         let map = FacetDistribution::new(&txn, &index)
-            .facets(iter::once(("colour", OrderBy::default())))
+            .facets(std::iter::once("colour"))
             .candidates((217..777).collect())
             .compute_stats()
             .unwrap();

milli/src/search/facet/facet_distribution_iter.rs

@@ -1,5 +1,3 @@
-use std::cmp::Reverse;
-use std::collections::BinaryHeap;
 use std::ops::ControlFlow;
 
 use heed::Result;
@@ -21,7 +19,7 @@ use crate::DocumentId;
 ///
 /// The return value of the closure is a `ControlFlow<()>` which indicates whether we should
 /// keep iterating over the different facet values or stop.
-pub fn lexicographically_iterate_over_facet_distribution<'t, CB>(
+pub fn iterate_over_facet_distribution<'t, CB>(
     rtxn: &'t heed::RoTxn<'t>,
     db: heed::Database<FacetGroupKeyCodec<ByteSliceRefCodec>, FacetGroupValueCodec>,
     field_id: u16,
@@ -31,7 +29,7 @@ pub fn lexicographically_iterate_over_facet_distribution<'t, CB>(
 where
     CB: FnMut(&'t [u8], u64, DocumentId) -> Result<ControlFlow<()>>,
 {
-    let mut fd = LexicographicFacetDistribution { rtxn, db, field_id, callback };
+    let mut fd = FacetDistribution { rtxn, db, field_id, callback };
     let highest_level = get_highest_level(
         rtxn,
         db.remap_key_type::<FacetGroupKeyCodec<ByteSliceRefCodec>>(),
@@ -46,102 +44,7 @@ where
     }
 }
 
-pub fn count_iterate_over_facet_distribution<'t, CB>(
-    rtxn: &'t heed::RoTxn<'t>,
-    db: heed::Database<FacetGroupKeyCodec<ByteSliceRefCodec>, FacetGroupValueCodec>,
-    field_id: u16,
-    candidates: &RoaringBitmap,
-    mut callback: CB,
-) -> Result<()>
-where
-    CB: FnMut(&'t [u8], u64, DocumentId) -> Result<ControlFlow<()>>,
-{
-    /// # Important
-    /// The order of the fields determines the order in which the facet values will be returned.
-    /// This struct is inserted in a BinaryHeap and popped later on.
-    #[derive(Debug, PartialOrd, Ord, PartialEq, Eq)]
-    struct LevelEntry<'t> {
-        /// The number of candidates in this entry.
-        count: u64,
-        /// The key level of the entry.
-        level: Reverse<u8>,
-        /// The left bound key.
-        left_bound: &'t [u8],
-        /// The number of keys we must look for after `left_bound`.
-        group_size: u8,
-        /// Any docid in the set of matching documents. Used to find the original facet string.
-        any_docid: u32,
-    }
-
-    // Represents the list of keys that we must explore.
-    let mut heap = BinaryHeap::new();
-    let highest_level = get_highest_level(
-        rtxn,
-        db.remap_key_type::<FacetGroupKeyCodec<ByteSliceRefCodec>>(),
-        field_id,
-    )?;
-
-    if let Some(first_bound) = get_first_facet_value::<ByteSliceRefCodec>(rtxn, db, field_id)? {
-        // We first fill the heap with values from the highest level
-        let starting_key =
-            FacetGroupKey { field_id, level: highest_level, left_bound: first_bound };
-        for el in db.range(rtxn, &(&starting_key..))?.take(usize::MAX) {
-            let (key, value) = el?;
-            // The range is unbounded on the right and the group size for the highest level is MAX,
-            // so we need to check that we are not iterating over the next field id
-            if key.field_id != field_id {
-                break;
-            }
-            let intersection = value.bitmap & candidates;
-            let count = intersection.len();
-            if count != 0 {
-                heap.push(LevelEntry {
-                    count,
-                    level: Reverse(key.level),
-                    left_bound: key.left_bound,
-                    group_size: value.size,
-                    any_docid: intersection.min().unwrap(),
-                });
-            }
-        }
-
-        while let Some(LevelEntry { count, level, left_bound, group_size, any_docid }) = heap.pop()
-        {
-            if let Reverse(0) = level {
-                match (callback)(left_bound, count, any_docid)? {
-                    ControlFlow::Continue(_) => (),
-                    ControlFlow::Break(_) => return Ok(()),
-                }
-            } else {
-                let starting_key = FacetGroupKey { field_id, level: level.0 - 1, left_bound };
-                for el in db.range(rtxn, &(&starting_key..))?.take(group_size as usize) {
-                    let (key, value) = el?;
-                    // The range is unbounded on the right and the group size for the highest level is MAX,
-                    // so we need to check that we are not iterating over the next field id
-                    if key.field_id != field_id {
-                        break;
-                    }
-                    let intersection = value.bitmap & candidates;
-                    let count = intersection.len();
-                    if count != 0 {
-                        heap.push(LevelEntry {
-                            count,
-                            level: Reverse(key.level),
-                            left_bound: key.left_bound,
-                            group_size: value.size,
-                            any_docid: intersection.min().unwrap(),
-                        });
-                    }
-                }
-            }
-        }
-    }
-
-    Ok(())
-}
-
-/// Iterate over the facets values by lexicographic order.
-struct LexicographicFacetDistribution<'t, CB>
+struct FacetDistribution<'t, CB>
 where
     CB: FnMut(&'t [u8], u64, DocumentId) -> Result<ControlFlow<()>>,
 {
@@ -151,7 +54,7 @@ where
     callback: CB,
 }
 
-impl<'t, CB> LexicographicFacetDistribution<'t, CB>
+impl<'t, CB> FacetDistribution<'t, CB>
 where
     CB: FnMut(&'t [u8], u64, DocumentId) -> Result<ControlFlow<()>>,
 {
@@ -183,7 +86,6 @@ where
         }
         Ok(ControlFlow::Continue(()))
     }
-
     fn iterate(
         &mut self,
         candidates: &RoaringBitmap,
@@ -196,10 +98,10 @@ where
         }
         let starting_key =
             FacetGroupKey { field_id: self.field_id, level, left_bound: starting_bound };
-        let iter = self.db.range(self.rtxn, &(&starting_key..))?.take(group_size);
+        let iter = self.db.range(self.rtxn, &(&starting_key..)).unwrap().take(group_size);
 
         for el in iter {
-            let (key, value) = el?;
+            let (key, value) = el.unwrap();
             // The range is unbounded on the right and the group size for the highest level is MAX,
             // so we need to check that we are not iterating over the next field id
             if key.field_id != self.field_id {
@@ -214,7 +116,7 @@ where
                     value.size as usize,
                 )?;
                 match cf {
-                    ControlFlow::Continue(_) => (),
+                    ControlFlow::Continue(_) => {}
                     ControlFlow::Break(_) => return Ok(ControlFlow::Break(())),
                 }
             }
@@ -230,7 +132,7 @@ mod tests {
     use heed::BytesDecode;
     use roaring::RoaringBitmap;
 
-    use super::lexicographically_iterate_over_facet_distribution;
+    use super::iterate_over_facet_distribution;
     use crate::heed_codec::facet::OrderedF64Codec;
     use crate::milli_snap;
     use crate::search::facet::tests::{get_random_looking_index, get_simple_index};
@@ -242,7 +144,7 @@ mod tests {
             let txn = index.env.read_txn().unwrap();
             let candidates = (0..=255).collect::<RoaringBitmap>();
             let mut results = String::new();
-            lexicographically_iterate_over_facet_distribution(
+            iterate_over_facet_distribution(
                 &txn,
                 index.content,
                 0,
@@ -259,7 +161,6 @@ mod tests {
             txn.commit().unwrap();
         }
     }
-
     #[test]
     fn filter_distribution_all_stop_early() {
         let indexes = [get_simple_index(), get_random_looking_index()];
@@ -268,7 +169,7 @@ mod tests {
             let candidates = (0..=255).collect::<RoaringBitmap>();
             let mut results = String::new();
             let mut nbr_facets = 0;
-            lexicographically_iterate_over_facet_distribution(
+            iterate_over_facet_distribution(
                 &txn,
                 index.content,
                 0,

milli/src/search/facet/mod.rs

@@ -4,7 +4,7 @@ use heed::types::{ByteSlice, DecodeIgnore};
 use heed::{BytesDecode, RoTxn};
 use roaring::RoaringBitmap;
 
-pub use self::facet_distribution::{FacetDistribution, OrderBy, DEFAULT_VALUES_PER_FACET};
+pub use self::facet_distribution::{FacetDistribution, DEFAULT_VALUES_PER_FACET};
 pub use self::filter::{BadGeoError, Filter};
 use crate::heed_codec::facet::{FacetGroupKeyCodec, FacetGroupValueCodec, OrderedF64Codec};
 use crate::heed_codec::ByteSliceRefCodec;

milli/src/search/mod.rs

@@ -1,21 +1,15 @@
 use std::fmt;
 
-use fst::automaton::{Automaton, Str};
-use fst::{IntoStreamer, Streamer};
 use levenshtein_automata::{LevenshteinAutomatonBuilder as LevBuilder, DFA};
-use log::error;
 use once_cell::sync::Lazy;
 use roaring::bitmap::RoaringBitmap;
 
-pub use self::facet::{FacetDistribution, Filter, OrderBy, DEFAULT_VALUES_PER_FACET};
+pub use self::facet::{FacetDistribution, Filter, DEFAULT_VALUES_PER_FACET};
 pub use self::new::matches::{FormatOptions, MatchBounds, Matcher, MatcherBuilder, MatchingWords};
 use self::new::PartialSearchResult;
-use crate::error::UserError;
-use crate::heed_codec::facet::{FacetGroupKey, FacetGroupValue};
-use crate::score_details::{ScoreDetails, ScoringStrategy};
+use crate::score_details::ScoreDetails;
 use crate::{
-    execute_search, normalize_facet, AscDesc, DefaultSearchLogger, DocumentId, FieldId, Index,
-    Result, SearchContext, BEU16,
+    execute_search, AscDesc, DefaultSearchLogger, DocumentId, Index, Result, SearchContext,
 };
 
 // Building these factories is not free.
@@ -23,25 +17,19 @@ static LEVDIST0: Lazy<LevBuilder> = Lazy::new(|| LevBuilder::new(0, true));
 static LEVDIST1: Lazy<LevBuilder> = Lazy::new(|| LevBuilder::new(1, true));
 static LEVDIST2: Lazy<LevBuilder> = Lazy::new(|| LevBuilder::new(2, true));
 
-/// The maximum number of facets returned by the facet search route.
-const MAX_NUMBER_OF_FACETS: usize = 100;
-
 pub mod facet;
 mod fst_utils;
 pub mod new;
 
 pub struct Search<'a> {
     query: Option<String>,
-    vector: Option<Vec<f32>>,
     // this should be linked to the String in the query
     filter: Option<Filter<'a>>,
     offset: usize,
     limit: usize,
     sort_criteria: Option<Vec<AscDesc>>,
-    searchable_attributes: Option<&'a [String]>,
     geo_strategy: new::GeoSortStrategy,
     terms_matching_strategy: TermsMatchingStrategy,
-    scoring_strategy: ScoringStrategy,
     words_limit: usize,
     exhaustive_number_hits: bool,
     rtxn: &'a heed::RoTxn<'a>,
@@ -52,15 +40,12 @@ impl<'a> Search<'a> {
     pub fn new(rtxn: &'a heed::RoTxn, index: &'a Index) -> Search<'a> {
         Search {
             query: None,
-            vector: None,
             filter: None,
             offset: 0,
             limit: 20,
             sort_criteria: None,
-            searchable_attributes: None,
             geo_strategy: new::GeoSortStrategy::default(),
             terms_matching_strategy: TermsMatchingStrategy::default(),
-            scoring_strategy: Default::default(),
             exhaustive_number_hits: false,
             words_limit: 10,
             rtxn,
@@ -73,11 +58,6 @@ impl<'a> Search<'a> {
         self
     }
 
-    pub fn vector(&mut self, vector: impl Into<Vec<f32>>) -> &mut Search<'a> {
-        self.vector = Some(vector.into());
-        self
-    }
-
     pub fn offset(&mut self, offset: usize) -> &mut Search<'a> {
         self.offset = offset;
         self
@@ -93,21 +73,11 @@ impl<'a> Search<'a> {
         self
     }
 
-    pub fn searchable_attributes(&mut self, searchable: &'a [String]) -> &mut Search<'a> {
-        self.searchable_attributes = Some(searchable);
-        self
-    }
-
     pub fn terms_matching_strategy(&mut self, value: TermsMatchingStrategy) -> &mut Search<'a> {
         self.terms_matching_strategy = value;
         self
     }
 
-    pub fn scoring_strategy(&mut self, value: ScoringStrategy) -> &mut Search<'a> {
-        self.scoring_strategy = value;
-        self
-    }
-
     pub fn words_limit(&mut self, value: usize) -> &mut Search<'a> {
         self.words_limit = value;
         self
@@ -133,18 +103,11 @@ impl<'a> Search<'a> {
 
     pub fn execute(&self) -> Result<SearchResult> {
         let mut ctx = SearchContext::new(self.index, self.rtxn);
-
-        if let Some(searchable_attributes) = self.searchable_attributes {
-            ctx.searchable_attributes(searchable_attributes)?;
-        }
-
         let PartialSearchResult { located_query_terms, candidates, documents_ids, document_scores } =
             execute_search(
                 &mut ctx,
                 &self.query,
-                &self.vector,
                 self.terms_matching_strategy,
-                self.scoring_strategy,
                 self.exhaustive_number_hits,
                 &self.filter,
                 &self.sort_criteria,
@@ -170,15 +133,12 @@ impl fmt::Debug for Search<'_> {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         let Search {
             query,
-            vector: _,
             filter,
             offset,
             limit,
             sort_criteria,
-            searchable_attributes,
             geo_strategy: _,
             terms_matching_strategy,
-            scoring_strategy,
             words_limit,
             exhaustive_number_hits,
             rtxn: _,
@@ -186,14 +146,11 @@ impl fmt::Debug for Search<'_> {
         } = self;
         f.debug_struct("Search")
             .field("query", query)
-            .field("vector", &"[...]")
             .field("filter", filter)
             .field("offset", offset)
             .field("limit", limit)
             .field("sort_criteria", sort_criteria)
-            .field("searchable_attributes", searchable_attributes)
             .field("terms_matching_strategy", terms_matching_strategy)
-            .field("scoring_strategy", scoring_strategy)
             .field("exhaustive_number_hits", exhaustive_number_hits)
             .field("words_limit", words_limit)
             .finish()
@@ -243,195 +200,6 @@ pub fn build_dfa(word: &str, typos: u8, is_prefix: bool) -> DFA {
     }
 }
 
-pub struct SearchForFacetValues<'a> {
-    query: Option<String>,
-    facet: String,
-    search_query: Search<'a>,
-}
-
-impl<'a> SearchForFacetValues<'a> {
-    pub fn new(facet: String, search_query: Search<'a>) -> SearchForFacetValues<'a> {
-        SearchForFacetValues { query: None, facet, search_query }
-    }
-
-    pub fn query(&mut self, query: impl Into<String>) -> &mut Self {
-        self.query = Some(query.into());
-        self
-    }
-
-    fn one_original_value_of(
-        &self,
-        field_id: FieldId,
-        facet_str: &str,
-        any_docid: DocumentId,
-    ) -> Result<Option<String>> {
-        let index = self.search_query.index;
-        let rtxn = self.search_query.rtxn;
-        let key: (FieldId, _, &str) = (field_id, any_docid, facet_str);
-        Ok(index.field_id_docid_facet_strings.get(rtxn, &key)?.map(|v| v.to_owned()))
-    }
-
-    pub fn execute(&self) -> Result<Vec<FacetValueHit>> {
-        let index = self.search_query.index;
-        let rtxn = self.search_query.rtxn;
-
-        let filterable_fields = index.filterable_fields(rtxn)?;
-        if !filterable_fields.contains(&self.facet) {
-            return Err(UserError::InvalidFacetSearchFacetName {
-                field: self.facet.clone(),
-                valid_fields: filterable_fields.into_iter().collect(),
-            }
-            .into());
-        }
-
-        let fields_ids_map = index.fields_ids_map(rtxn)?;
-        let fid = match fields_ids_map.id(&self.facet) {
-            Some(fid) => fid,
-            // we return an empty list of results when the attribute has been
-            // set as filterable but no document contains this field (yet).
-            None => return Ok(Vec::new()),
-        };
-
-        let fst = match self.search_query.index.facet_id_string_fst.get(rtxn, &BEU16::new(fid))? {
-            Some(fst) => fst,
-            None => return Ok(vec![]),
-        };
-
-        let search_candidates = self.search_query.execute()?.candidates;
-
-        match self.query.as_ref() {
-            Some(query) => {
-                let query = normalize_facet(query);
-                let query = query.as_str();
-                let authorize_typos = self.search_query.index.authorize_typos(rtxn)?;
-                let field_authorizes_typos =
-                    !self.search_query.index.exact_attributes_ids(rtxn)?.contains(&fid);
-
-                if authorize_typos && field_authorizes_typos {
-                    let mut results = vec![];
-
-                    let exact_words_fst = self.search_query.index.exact_words(rtxn)?;
-                    if exact_words_fst.map_or(false, |fst| fst.contains(query)) {
-                        let key = FacetGroupKey { field_id: fid, level: 0, left_bound: query };
-                        if let Some(FacetGroupValue { bitmap, .. }) =
-                            index.facet_id_string_docids.get(rtxn, &key)?
-                        {
-                            let count = search_candidates.intersection_len(&bitmap);
-                            if count != 0 {
-                                let value = self
-                                    .one_original_value_of(fid, query, bitmap.min().unwrap())?
-                                    .unwrap_or_else(|| query.to_string());
-                                results.push(FacetValueHit { value, count });
-                            }
-                        }
-                    } else {
-                        let one_typo = self.search_query.index.min_word_len_one_typo(rtxn)?;
-                        let two_typos = self.search_query.index.min_word_len_two_typos(rtxn)?;
-
-                        let is_prefix = true;
-                        let automaton = if query.len() < one_typo as usize {
-                            build_dfa(query, 0, is_prefix)
-                        } else if query.len() < two_typos as usize {
-                            build_dfa(query, 1, is_prefix)
-                        } else {
-                            build_dfa(query, 2, is_prefix)
-                        };
-
-                        let mut stream = fst.search(automaton).into_stream();
-                        let mut length = 0;
-                        while let Some(facet_value) = stream.next() {
-                            let value = std::str::from_utf8(facet_value)?;
-                            let key = FacetGroupKey { field_id: fid, level: 0, left_bound: value };
-                            let docids = match index.facet_id_string_docids.get(rtxn, &key)? {
-                                Some(FacetGroupValue { bitmap, .. }) => bitmap,
-                                None => {
-                                    error!(
-                                        "the facet value is missing from the facet database: {key:?}"
-                                    );
-                                    continue;
-                                }
-                            };
-                            let count = search_candidates.intersection_len(&docids);
-                            if count != 0 {
-                                let value = self
-                                    .one_original_value_of(fid, value, docids.min().unwrap())?
-                                    .unwrap_or_else(|| query.to_string());
-                                results.push(FacetValueHit { value, count });
-                                length += 1;
-                            }
-                            if length >= MAX_NUMBER_OF_FACETS {
-                                break;
-                            }
-                        }
-                    }
-
-                    Ok(results)
-                } else {
-                    let automaton = Str::new(query).starts_with();
-                    let mut stream = fst.search(automaton).into_stream();
-                    let mut results = vec![];
-                    let mut length = 0;
-                    while let Some(facet_value) = stream.next() {
-                        let value = std::str::from_utf8(facet_value)?;
-                        let key = FacetGroupKey { field_id: fid, level: 0, left_bound: value };
-                        let docids = match index.facet_id_string_docids.get(rtxn, &key)? {
-                            Some(FacetGroupValue { bitmap, .. }) => bitmap,
-                            None => {
-                                error!(
-                                    "the facet value is missing from the facet database: {key:?}"
-                                );
-                                continue;
-                            }
-                        };
-                        let count = search_candidates.intersection_len(&docids);
-                        if count != 0 {
-                            let value = self
-                                .one_original_value_of(fid, value, docids.min().unwrap())?
-                                .unwrap_or_else(|| query.to_string());
-                            results.push(FacetValueHit { value, count });
-                            length += 1;
-                        }
-                        if length >= MAX_NUMBER_OF_FACETS {
-                            break;
-                        }
-                    }
-
-                    Ok(results)
-                }
-            }
-            None => {
-                let mut results = vec![];
-                let mut length = 0;
-                let prefix = FacetGroupKey { field_id: fid, level: 0, left_bound: "" };
-                for result in index.facet_id_string_docids.prefix_iter(rtxn, &prefix)? {
-                    let (FacetGroupKey { left_bound, .. }, FacetGroupValue { bitmap, .. }) =
-                        result?;
-                    let count = search_candidates.intersection_len(&bitmap);
-                    if count != 0 {
-                        let value = self
-                            .one_original_value_of(fid, left_bound, bitmap.min().unwrap())?
-                            .unwrap_or_else(|| left_bound.to_string());
-                        results.push(FacetValueHit { value, count });
-                        length += 1;
-                    }
-                    if length >= MAX_NUMBER_OF_FACETS {
-                        break;
-                    }
-                }
-                Ok(results)
-            }
-        }
-    }
-}
-
-#[derive(Debug, Clone, serde::Serialize, PartialEq)]
-pub struct FacetValueHit {
-    /// The original facet value
-    pub value: String,
-    /// The number of documents associated to this facet
-    pub count: u64,
-}
-
 #[cfg(test)]
 mod test {
     #[allow(unused_imports)]

milli/src/search/new/bucket_sort.rs

@@ -3,7 +3,7 @@ use roaring::RoaringBitmap;
 use super::logger::SearchLogger;
 use super::ranking_rules::{BoxRankingRule, RankingRuleQueryTrait};
 use super::SearchContext;
-use crate::score_details::{ScoreDetails, ScoringStrategy};
+use crate::score_details::ScoreDetails;
 use crate::search::new::distinct::{apply_distinct_rule, distinct_single_docid, DistinctOutput};
 use crate::Result;
 
@@ -13,8 +13,6 @@ pub struct BucketSortOutput {
     pub all_candidates: RoaringBitmap,
 }
 
-// TODO: would probably be good to regroup some of these inside of a struct?
-#[allow(clippy::too_many_arguments)]
 pub fn bucket_sort<'ctx, Q: RankingRuleQueryTrait>(
     ctx: &mut SearchContext<'ctx>,
     mut ranking_rules: Vec<BoxRankingRule<'ctx, Q>>,
@@ -22,7 +20,6 @@ pub fn bucket_sort<'ctx, Q: RankingRuleQueryTrait>(
     universe: &RoaringBitmap,
     from: usize,
     length: usize,
-    scoring_strategy: ScoringStrategy,
     logger: &mut dyn SearchLogger<Q>,
 ) -> Result<BucketSortOutput> {
     logger.initial_query(query);
@@ -108,6 +105,7 @@ pub fn bucket_sort<'ctx, Q: RankingRuleQueryTrait>(
             } else {
                 cur_ranking_rule_index -= 1;
             }
+            // FIXME: check off by one
             if ranking_rule_scores.len() > cur_ranking_rule_index {
                 ranking_rule_scores.pop();
             }
@@ -143,22 +141,12 @@ pub fn bucket_sort<'ctx, Q: RankingRuleQueryTrait>(
     while valid_docids.len() < length {
         // The universe for this bucket is zero, so we don't need to sort
         // anything, just go back to the parent ranking rule.
-        if ranking_rule_universes[cur_ranking_rule_index].is_empty()
-            || (scoring_strategy == ScoringStrategy::Skip
-                && ranking_rule_universes[cur_ranking_rule_index].len() == 1)
-        {
-            let bucket = std::mem::take(&mut ranking_rule_universes[cur_ranking_rule_index]);
-            maybe_add_to_results!(bucket);
+        if ranking_rule_universes[cur_ranking_rule_index].is_empty() {
             back!();
             continue;
         }
 
-        let Some(next_bucket) = ranking_rules[cur_ranking_rule_index].next_bucket(
-            ctx,
-            logger,
-            &ranking_rule_universes[cur_ranking_rule_index],
-        )?
-        else {
+        let Some(next_bucket) = ranking_rules[cur_ranking_rule_index].next_bucket(ctx, logger, &ranking_rule_universes[cur_ranking_rule_index])? else {
             back!();
             continue;
         };
@@ -178,10 +166,10 @@ pub fn bucket_sort<'ctx, Q: RankingRuleQueryTrait>(
         ranking_rule_universes[cur_ranking_rule_index] -= &next_bucket.candidates;
 
         if cur_ranking_rule_index == ranking_rules_len - 1
-            || (scoring_strategy == ScoringStrategy::Skip && next_bucket.candidates.len() <= 1)
             || cur_offset + (next_bucket.candidates.len() as usize) < from
         {
             maybe_add_to_results!(next_bucket.candidates);
+            // FIXME: use index based logic like all the other rules so that you don't have to maintain the pop/push?
             ranking_rule_scores.pop();
             continue;
         }

milli/src/search/new/db_cache.rs

@@ -4,13 +4,12 @@ use std::hash::Hash;
 
 use fxhash::FxHashMap;
 use heed::types::ByteSlice;
-use heed::{BytesEncode, Database, RoTxn};
+use heed::{BytesDecode, BytesEncode, Database, RoTxn};
 use roaring::RoaringBitmap;
 
 use super::interner::Interned;
 use super::Word;
-use crate::heed_codec::{BytesDecodeOwned, StrBEU16Codec};
-use crate::update::{merge_cbo_roaring_bitmaps, MergeFn};
+use crate::heed_codec::StrBEU16Codec;
 use crate::{
     CboRoaringBitmapCodec, CboRoaringBitmapLenCodec, Result, RoaringBitmapCodec, SearchContext,
 };
@@ -23,104 +22,50 @@ use crate::{
 #[derive(Default)]
 pub struct DatabaseCache<'ctx> {
     pub word_pair_proximity_docids:
-        FxHashMap<(u8, Interned<String>, Interned<String>), Option<Cow<'ctx, [u8]>>>,
+        FxHashMap<(u8, Interned<String>, Interned<String>), Option<&'ctx [u8]>>,
     pub word_prefix_pair_proximity_docids:
-        FxHashMap<(u8, Interned<String>, Interned<String>), Option<Cow<'ctx, [u8]>>>,
+        FxHashMap<(u8, Interned<String>, Interned<String>), Option<&'ctx [u8]>>,
     pub prefix_word_pair_proximity_docids:
-        FxHashMap<(u8, Interned<String>, Interned<String>), Option<Cow<'ctx, [u8]>>>,
-    pub word_docids: FxHashMap<Interned<String>, Option<Cow<'ctx, [u8]>>>,
-    pub exact_word_docids: FxHashMap<Interned<String>, Option<Cow<'ctx, [u8]>>>,
-    pub word_prefix_docids: FxHashMap<Interned<String>, Option<Cow<'ctx, [u8]>>>,
-    pub exact_word_prefix_docids: FxHashMap<Interned<String>, Option<Cow<'ctx, [u8]>>>,
+        FxHashMap<(u8, Interned<String>, Interned<String>), Option<&'ctx [u8]>>,
+    pub word_docids: FxHashMap<Interned<String>, Option<&'ctx [u8]>>,
+    pub exact_word_docids: FxHashMap<Interned<String>, Option<&'ctx [u8]>>,
+    pub word_prefix_docids: FxHashMap<Interned<String>, Option<&'ctx [u8]>>,
+    pub exact_word_prefix_docids: FxHashMap<Interned<String>, Option<&'ctx [u8]>>,
 
     pub words_fst: Option<fst::Set<Cow<'ctx, [u8]>>>,
-    pub word_position_docids: FxHashMap<(Interned<String>, u16), Option<Cow<'ctx, [u8]>>>,
-    pub word_prefix_position_docids: FxHashMap<(Interned<String>, u16), Option<Cow<'ctx, [u8]>>>,
+    pub word_position_docids: FxHashMap<(Interned<String>, u16), Option<&'ctx [u8]>>,
+    pub word_prefix_position_docids: FxHashMap<(Interned<String>, u16), Option<&'ctx [u8]>>,
     pub word_positions: FxHashMap<Interned<String>, Vec<u16>>,
     pub word_prefix_positions: FxHashMap<Interned<String>, Vec<u16>>,
 
-    pub word_fid_docids: FxHashMap<(Interned<String>, u16), Option<Cow<'ctx, [u8]>>>,
-    pub word_prefix_fid_docids: FxHashMap<(Interned<String>, u16), Option<Cow<'ctx, [u8]>>>,
+    pub word_fid_docids: FxHashMap<(Interned<String>, u16), Option<&'ctx [u8]>>,
+    pub word_prefix_fid_docids: FxHashMap<(Interned<String>, u16), Option<&'ctx [u8]>>,
     pub word_fids: FxHashMap<Interned<String>, Vec<u16>>,
     pub word_prefix_fids: FxHashMap<Interned<String>, Vec<u16>>,
 }
 impl<'ctx> DatabaseCache<'ctx> {
-    fn get_value<'v, K1, KC, DC>(
+    fn get_value<'v, K1, KC>(
         txn: &'ctx RoTxn,
         cache_key: K1,
         db_key: &'v KC::EItem,
-        cache: &mut FxHashMap<K1, Option<Cow<'ctx, [u8]>>>,
+        cache: &mut FxHashMap<K1, Option<&'ctx [u8]>>,
         db: Database<KC, ByteSlice>,
-    ) -> Result<Option<DC::DItem>>
+    ) -> Result<Option<&'ctx [u8]>>
     where
         K1: Copy + Eq + Hash,
         KC: BytesEncode<'v>,
-        DC: BytesDecodeOwned,
     {
-        if let Entry::Vacant(entry) = cache.entry(cache_key) {
-            let bitmap_ptr = db.get(txn, db_key)?.map(Cow::Borrowed);
-            entry.insert(bitmap_ptr);
-        }
-
-        match cache.get(&cache_key).unwrap() {
-            Some(Cow::Borrowed(bytes)) => {
-                DC::bytes_decode_owned(bytes).ok_or(heed::Error::Decoding.into()).map(Some)
+        let bitmap_ptr = match cache.entry(cache_key) {
+            Entry::Occupied(bitmap_ptr) => *bitmap_ptr.get(),
+            Entry::Vacant(entry) => {
+                let bitmap_ptr = db.get(txn, db_key)?;
+                entry.insert(bitmap_ptr);
+                bitmap_ptr
             }
-            Some(Cow::Owned(bytes)) => {
-                DC::bytes_decode_owned(bytes).ok_or(heed::Error::Decoding.into()).map(Some)
-            }
-            None => Ok(None),
-        }
-    }
-
-    fn get_value_from_keys<'v, K1, KC, DC>(
-        txn: &'ctx RoTxn,
-        cache_key: K1,
-        db_keys: &'v [KC::EItem],
-        cache: &mut FxHashMap<K1, Option<Cow<'ctx, [u8]>>>,
-        db: Database<KC, ByteSlice>,
-        merger: MergeFn,
-    ) -> Result<Option<DC::DItem>>
-    where
-        K1: Copy + Eq + Hash,
-        KC: BytesEncode<'v>,
-        DC: BytesDecodeOwned,
-        KC::EItem: Sized,
-    {
-        if let Entry::Vacant(entry) = cache.entry(cache_key) {
-            let bitmap_ptr: Option<Cow<'ctx, [u8]>> = match db_keys {
-                [] => None,
-                [key] => db.get(txn, key)?.map(Cow::Borrowed),
-                keys => {
-                    let bitmaps = keys
-                        .iter()
-                        .filter_map(|key| db.get(txn, key).transpose())
-                        .map(|v| v.map(Cow::Borrowed))
-                        .collect::<std::result::Result<Vec<Cow<[u8]>>, _>>()?;
-
-                    if bitmaps.is_empty() {
-                        None
-                    } else {
-                        Some(merger(&[], &bitmaps[..])?)
-                    }
-                }
-            };
-
-            entry.insert(bitmap_ptr);
-        }
-
-        match cache.get(&cache_key).unwrap() {
-            Some(Cow::Borrowed(bytes)) => {
-                DC::bytes_decode_owned(bytes).ok_or(heed::Error::Decoding.into()).map(Some)
-            }
-            Some(Cow::Owned(bytes)) => {
-                DC::bytes_decode_owned(bytes).ok_or(heed::Error::Decoding.into()).map(Some)
-            }
-            None => Ok(None),
-        }
+        };
+        Ok(bitmap_ptr)
     }
 }
-
 impl<'ctx> SearchContext<'ctx> {
     pub fn get_words_fst(&mut self) -> Result<fst::Set<Cow<'ctx, [u8]>>> {
         if let Some(fst) = self.db_cache.words_fst.clone() {
@@ -154,41 +99,30 @@ impl<'ctx> SearchContext<'ctx> {
 
     /// Retrieve or insert the given value in the `word_docids` database.
     fn get_db_word_docids(&mut self, word: Interned<String>) -> Result<Option<RoaringBitmap>> {
-        match &self.restricted_fids {
-            Some(restricted_fids) => {
-                let interned = self.word_interner.get(word).as_str();
-                let keys: Vec<_> = restricted_fids.iter().map(|fid| (interned, *fid)).collect();
-
-                DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>(
-                    self.txn,
-                    word,
-                    &keys[..],
-                    &mut self.db_cache.word_docids,
-                    self.index.word_fid_docids.remap_data_type::<ByteSlice>(),
-                    merge_cbo_roaring_bitmaps,
-                )
-            }
-            None => DatabaseCache::get_value::<_, _, RoaringBitmapCodec>(
-                self.txn,
-                word,
-                self.word_interner.get(word).as_str(),
-                &mut self.db_cache.word_docids,
-                self.index.word_docids.remap_data_type::<ByteSlice>(),
-            ),
-        }
+        DatabaseCache::get_value(
+            self.txn,
+            word,
+            self.word_interner.get(word).as_str(),
+            &mut self.db_cache.word_docids,
+            self.index.word_docids.remap_data_type::<ByteSlice>(),
+        )?
+        .map(|bytes| RoaringBitmapCodec::bytes_decode(bytes).ok_or(heed::Error::Decoding.into()))
+        .transpose()
     }
 
     fn get_db_exact_word_docids(
         &mut self,
         word: Interned<String>,
     ) -> Result<Option<RoaringBitmap>> {
-        DatabaseCache::get_value::<_, _, RoaringBitmapCodec>(
+        DatabaseCache::get_value(
             self.txn,
             word,
             self.word_interner.get(word).as_str(),
             &mut self.db_cache.exact_word_docids,
             self.index.exact_word_docids.remap_data_type::<ByteSlice>(),
-        )
+        )?
+        .map(|bytes| RoaringBitmapCodec::bytes_decode(bytes).ok_or(heed::Error::Decoding.into()))
+        .transpose()
     }
 
     pub fn word_prefix_docids(&mut self, prefix: Word) -> Result<Option<RoaringBitmap>> {
@@ -216,41 +150,30 @@ impl<'ctx> SearchContext<'ctx> {
         &mut self,
         prefix: Interned<String>,
     ) -> Result<Option<RoaringBitmap>> {
-        match &self.restricted_fids {
-            Some(restricted_fids) => {
-                let interned = self.word_interner.get(prefix).as_str();
-                let keys: Vec<_> = restricted_fids.iter().map(|fid| (interned, *fid)).collect();
-
-                DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>(
-                    self.txn,
-                    prefix,
-                    &keys[..],
-                    &mut self.db_cache.word_prefix_docids,
-                    self.index.word_prefix_fid_docids.remap_data_type::<ByteSlice>(),
-                    merge_cbo_roaring_bitmaps,
-                )
-            }
-            None => DatabaseCache::get_value::<_, _, RoaringBitmapCodec>(
-                self.txn,
-                prefix,
-                self.word_interner.get(prefix).as_str(),
-                &mut self.db_cache.word_prefix_docids,
-                self.index.word_prefix_docids.remap_data_type::<ByteSlice>(),
-            ),
-        }
+        DatabaseCache::get_value(
+            self.txn,
+            prefix,
+            self.word_interner.get(prefix).as_str(),
+            &mut self.db_cache.word_prefix_docids,
+            self.index.word_prefix_docids.remap_data_type::<ByteSlice>(),
+        )?
+        .map(|bytes| RoaringBitmapCodec::bytes_decode(bytes).ok_or(heed::Error::Decoding.into()))
+        .transpose()
     }
 
     fn get_db_exact_word_prefix_docids(
         &mut self,
         prefix: Interned<String>,
     ) -> Result<Option<RoaringBitmap>> {
-        DatabaseCache::get_value::<_, _, RoaringBitmapCodec>(
+        DatabaseCache::get_value(
             self.txn,
             prefix,
             self.word_interner.get(prefix).as_str(),
             &mut self.db_cache.exact_word_prefix_docids,
             self.index.exact_word_prefix_docids.remap_data_type::<ByteSlice>(),
-        )
+        )?
+        .map(|bytes| RoaringBitmapCodec::bytes_decode(bytes).ok_or(heed::Error::Decoding.into()))
+        .transpose()
     }
 
     pub fn get_db_word_pair_proximity_docids(
@@ -259,7 +182,7 @@ impl<'ctx> SearchContext<'ctx> {
         word2: Interned<String>,
         proximity: u8,
     ) -> Result<Option<RoaringBitmap>> {
-        DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
+        DatabaseCache::get_value(
             self.txn,
             (proximity, word1, word2),
             &(
@@ -269,7 +192,9 @@ impl<'ctx> SearchContext<'ctx> {
             ),
             &mut self.db_cache.word_pair_proximity_docids,
             self.index.word_pair_proximity_docids.remap_data_type::<ByteSlice>(),
-        )
+        )?
+        .map(|bytes| CboRoaringBitmapCodec::bytes_decode(bytes).ok_or(heed::Error::Decoding.into()))
+        .transpose()
     }
 
     pub fn get_db_word_pair_proximity_docids_len(
@@ -278,7 +203,7 @@ impl<'ctx> SearchContext<'ctx> {
         word2: Interned<String>,
         proximity: u8,
     ) -> Result<Option<u64>> {
-        DatabaseCache::get_value::<_, _, CboRoaringBitmapLenCodec>(
+        DatabaseCache::get_value(
             self.txn,
             (proximity, word1, word2),
             &(
@@ -288,7 +213,11 @@ impl<'ctx> SearchContext<'ctx> {
             ),
             &mut self.db_cache.word_pair_proximity_docids,
             self.index.word_pair_proximity_docids.remap_data_type::<ByteSlice>(),
-        )
+        )?
+        .map(|bytes| {
+            CboRoaringBitmapLenCodec::bytes_decode(bytes).ok_or(heed::Error::Decoding.into())
+        })
+        .transpose()
     }
 
     pub fn get_db_word_prefix_pair_proximity_docids(
@@ -297,7 +226,7 @@ impl<'ctx> SearchContext<'ctx> {
         prefix2: Interned<String>,
         proximity: u8,
     ) -> Result<Option<RoaringBitmap>> {
-        DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
+        DatabaseCache::get_value(
             self.txn,
             (proximity, word1, prefix2),
             &(
@@ -307,7 +236,9 @@ impl<'ctx> SearchContext<'ctx> {
             ),
             &mut self.db_cache.word_prefix_pair_proximity_docids,
             self.index.word_prefix_pair_proximity_docids.remap_data_type::<ByteSlice>(),
-        )
+        )?
+        .map(|bytes| CboRoaringBitmapCodec::bytes_decode(bytes).ok_or(heed::Error::Decoding.into()))
+        .transpose()
     }
     pub fn get_db_prefix_word_pair_proximity_docids(
         &mut self,
@@ -315,7 +246,7 @@ impl<'ctx> SearchContext<'ctx> {
         right: Interned<String>,
         proximity: u8,
     ) -> Result<Option<RoaringBitmap>> {
-        DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
+        DatabaseCache::get_value(
             self.txn,
             (proximity, left_prefix, right),
             &(
@@ -325,7 +256,9 @@ impl<'ctx> SearchContext<'ctx> {
             ),
             &mut self.db_cache.prefix_word_pair_proximity_docids,
             self.index.prefix_word_pair_proximity_docids.remap_data_type::<ByteSlice>(),
-        )
+        )?
+        .map(|bytes| CboRoaringBitmapCodec::bytes_decode(bytes).ok_or(heed::Error::Decoding.into()))
+        .transpose()
     }
 
     pub fn get_db_word_fid_docids(
@@ -333,18 +266,15 @@ impl<'ctx> SearchContext<'ctx> {
         word: Interned<String>,
         fid: u16,
     ) -> Result<Option<RoaringBitmap>> {
-        // if the requested fid isn't in the restricted list, return None.
-        if self.restricted_fids.as_ref().map_or(false, |fids| !fids.contains(&fid)) {
-            return Ok(None);
-        }
-
-        DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
+        DatabaseCache::get_value(
             self.txn,
             (word, fid),
             &(self.word_interner.get(word).as_str(), fid),
             &mut self.db_cache.word_fid_docids,
             self.index.word_fid_docids.remap_data_type::<ByteSlice>(),
-        )
+        )?
+        .map(|bytes| CboRoaringBitmapCodec::bytes_decode(bytes).ok_or(heed::Error::Decoding.into()))
+        .transpose()
     }
 
     pub fn get_db_word_prefix_fid_docids(
@@ -352,18 +282,15 @@ impl<'ctx> SearchContext<'ctx> {
         word_prefix: Interned<String>,
         fid: u16,
     ) -> Result<Option<RoaringBitmap>> {
-        // if the requested fid isn't in the restricted list, return None.
-        if self.restricted_fids.as_ref().map_or(false, |fids| !fids.contains(&fid)) {
-            return Ok(None);
-        }
-
-        DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
+        DatabaseCache::get_value(
             self.txn,
             (word_prefix, fid),
             &(self.word_interner.get(word_prefix).as_str(), fid),
             &mut self.db_cache.word_prefix_fid_docids,
             self.index.word_prefix_fid_docids.remap_data_type::<ByteSlice>(),
-        )
+        )?
+        .map(|bytes| CboRoaringBitmapCodec::bytes_decode(bytes).ok_or(heed::Error::Decoding.into()))
+        .transpose()
     }
 
     pub fn get_db_word_fids(&mut self, word: Interned<String>) -> Result<Vec<u16>> {
@@ -382,7 +309,7 @@ impl<'ctx> SearchContext<'ctx> {
                 for result in remap_key_type {
                     let ((_, fid), value) = result?;
                     // filling other caches to avoid searching for them again
-                    self.db_cache.word_fid_docids.insert((word, fid), Some(Cow::Borrowed(value)));
+                    self.db_cache.word_fid_docids.insert((word, fid), Some(value));
                     fids.push(fid);
                 }
                 entry.insert(fids.clone());
@@ -408,9 +335,7 @@ impl<'ctx> SearchContext<'ctx> {
                 for result in remap_key_type {
                     let ((_, fid), value) = result?;
                     // filling other caches to avoid searching for them again
-                    self.db_cache
-                        .word_prefix_fid_docids
-                        .insert((word_prefix, fid), Some(Cow::Borrowed(value)));
+                    self.db_cache.word_prefix_fid_docids.insert((word_prefix, fid), Some(value));
                     fids.push(fid);
                 }
                 entry.insert(fids.clone());
@@ -425,13 +350,15 @@ impl<'ctx> SearchContext<'ctx> {
         word: Interned<String>,
         position: u16,
     ) -> Result<Option<RoaringBitmap>> {
-        DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
+        DatabaseCache::get_value(
             self.txn,
             (word, position),
             &(self.word_interner.get(word).as_str(), position),
             &mut self.db_cache.word_position_docids,
             self.index.word_position_docids.remap_data_type::<ByteSlice>(),
-        )
+        )?
+        .map(|bytes| CboRoaringBitmapCodec::bytes_decode(bytes).ok_or(heed::Error::Decoding.into()))
+        .transpose()
     }
 
     pub fn get_db_word_prefix_position_docids(
@@ -439,13 +366,15 @@ impl<'ctx> SearchContext<'ctx> {
         word_prefix: Interned<String>,
         position: u16,
     ) -> Result<Option<RoaringBitmap>> {
-        DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
+        DatabaseCache::get_value(
             self.txn,
             (word_prefix, position),
             &(self.word_interner.get(word_prefix).as_str(), position),
             &mut self.db_cache.word_prefix_position_docids,
             self.index.word_prefix_position_docids.remap_data_type::<ByteSlice>(),
-        )
+        )?
+        .map(|bytes| CboRoaringBitmapCodec::bytes_decode(bytes).ok_or(heed::Error::Decoding.into()))
+        .transpose()
     }
 
     pub fn get_db_word_positions(&mut self, word: Interned<String>) -> Result<Vec<u16>> {
@@ -464,9 +393,7 @@ impl<'ctx> SearchContext<'ctx> {
                 for result in remap_key_type {
                     let ((_, position), value) = result?;
                     // filling other caches to avoid searching for them again
-                    self.db_cache
-                        .word_position_docids
-                        .insert((word, position), Some(Cow::Borrowed(value)));
+                    self.db_cache.word_position_docids.insert((word, position), Some(value));
                     positions.push(position);
                 }
                 entry.insert(positions.clone());
@@ -497,7 +424,7 @@ impl<'ctx> SearchContext<'ctx> {
                     // filling other caches to avoid searching for them again
                     self.db_cache
                         .word_prefix_position_docids
-                        .insert((word_prefix, position), Some(Cow::Borrowed(value)));
+                        .insert((word_prefix, position), Some(value));
                     positions.push(position);
                 }
                 entry.insert(positions.clone());

milli/src/search/new/distinct.rs

@@ -26,6 +26,7 @@ pub fn apply_distinct_rule(
     ctx: &mut SearchContext,
     field_id: u16,
     candidates: &RoaringBitmap,
+    // TODO: add a universe here, such that the `excluded` are a subset of the universe?
 ) -> Result<DistinctOutput> {
     let mut excluded = RoaringBitmap::new();
     let mut remaining = RoaringBitmap::new();

milli/src/search/new/exact_attribute.rs

@@ -207,7 +207,7 @@ impl State {
             )?;
             intersection &= &candidates;
             if !intersection.is_empty() {
-                // Although not really worth it in terms of performance,
+                // TODO: although not really worth it in terms of performance,
                 // if would be good to put this in cache for the sake of consistency
                 let candidates_with_exact_word_count = if count_all_positions < u8::MAX as usize {
                     ctx.index
@@ -249,7 +249,7 @@ impl State {
                         query: query_graph,
                         candidates,
                         score: ScoreDetails::ExactAttribute(
-                            score_details::ExactAttribute::ExactMatch,
+                            score_details::ExactAttribute::MatchesFull,
                         ),
                     }),
                 )

milli/src/search/new/graph_based_ranking_rule.rs

@@ -134,20 +134,7 @@ impl<'ctx, G: RankingRuleGraphTrait> RankingRule<'ctx, QueryGraph> for GraphBase
         _universe: &RoaringBitmap,
         query_graph: &QueryGraph,
     ) -> Result<()> {
-        // the `next_max_cost` is the successor integer to the maximum cost of the paths in the graph.
-        //
-        // When there is a matching strategy, it also factors the additional costs of:
-        // 1. The words that are matched in phrases
-        // 2. Skipping words (by adding them to the paths with a cost)
-        let mut next_max_cost = 1;
         let removal_cost = if let Some(terms_matching_strategy) = self.terms_matching_strategy {
-            // add the cost of the phrase to the next_max_cost
-            next_max_cost += query_graph
-                .words_in_phrases_count(ctx)
-                // remove 1 from the words in phrases count, because when there is a phrase we can now have a document
-                // where only the phrase is matching, and none of the non-phrase words.
-                // With the `1` that `next_max_cost` is initialized with, this gets counted twice.
-                .saturating_sub(1) as u64;
             match terms_matching_strategy {
                 TermsMatchingStrategy::Last => {
                     let removal_order =
@@ -177,8 +164,8 @@ impl<'ctx, G: RankingRuleGraphTrait> RankingRule<'ctx, QueryGraph> for GraphBase
         // Then pre-compute the cost of all paths from each node to the end node
         let all_costs = graph.find_all_costs_to_end();
 
-        next_max_cost +=
-            all_costs.get(graph.query_graph.root_node).iter().copied().max().unwrap_or(0);
+        let next_max_cost =
+            all_costs.get(graph.query_graph.root_node).iter().copied().max().unwrap_or(0) + 1;
 
         let state = GraphBasedRankingRuleState {
             graph,
@@ -206,9 +193,11 @@ impl<'ctx, G: RankingRuleGraphTrait> RankingRule<'ctx, QueryGraph> for GraphBase
 
         let all_costs = state.all_costs.get(state.graph.query_graph.root_node);
         // Retrieve the cost of the paths to compute
-        let Some(&cost) = all_costs.iter().find(|c| **c >= state.cur_cost) else {
-            self.state = None;
-            return Ok(None);
+        let Some(&cost) = all_costs
+            .iter()
+            .find(|c| **c >= state.cur_cost) else {
+                self.state = None;
+                return Ok(None);
         };
         state.cur_cost = cost + 1;
 
@@ -312,6 +301,8 @@ impl<'ctx, G: RankingRuleGraphTrait> RankingRule<'ctx, QueryGraph> for GraphBase
 
         // We modify the next query graph so that it only contains the subgraph
         // that was used to compute this bucket
+        // But we only do it in case the bucket length is >1, because otherwise
+        // we know the child ranking rule won't be called anyway
 
         let paths: Vec<Vec<(Option<LocatedQueryTermSubset>, LocatedQueryTermSubset)>> = good_paths
             .into_iter()