Activate only the necessary features for Japanese

4108: Fix bug where search with distinct attribute and no ranking, returns offset+limit hits r=curquiza a=vivek-26

# Pull Request

## Related issue
Fixes #4078 

## What does this PR do?
This PR - 
- Fixes bug where search with distinct attribute and no ranking, returns offset+limit hits.
- Adds unit and integration tests.

## PR checklist
Please check if your PR fulfills the following requirements:
- [x] Does this PR fix an existing issue, or have you listed the changes applied in the PR description (and why they are needed)?
- [x] Have you read the contributing guidelines?
- [x] Have you made sure that the title is accurate and descriptive of the changes?

Thank you so much for contributing to Meilisearch!


Co-authored-by: Vivek Kumar <vivek.26@outlook.com>

.github/workflows/dependency-issue.yml

@@ -2,8 +2,8 @@ name: Create issue to upgrade dependencies
 
 on:
   schedule:
-    # Run the first of the month, every 6 month
-    - cron: '0 0 1 */6 *'
+    # Run the first of the month, every 3 month
+    - cron: '0 0 1 */3 *'
   workflow_dispatch:
 
 jobs:

.github/workflows/sdks-tests.yml

@@ -14,7 +14,6 @@ on:
 env:
   MEILI_MASTER_KEY: 'masterKey'
   MEILI_NO_ANALYTICS: 'true'
-  DISABLE_COVERAGE: 'true'
 
 jobs:
   define-docker-image:
@@ -31,117 +30,6 @@ jobs:
           if [[ $event == 'workflow_dispatch' ]]; then
             echo "docker-image=${{ github.event.inputs.docker_image }}" >> $GITHUB_OUTPUT
           fi
-      - name: Docker image is ${{ steps.define-image.outputs.docker-image }}
-        run: echo "Docker image is ${{ steps.define-image.outputs.docker-image }}"
-
-##########
-## SDKs ##
-##########
-
-  meilisearch-dotnet-tests:
-    needs: define-docker-image
-    name: .NET SDK tests
-    runs-on: ubuntu-latest
-    env:
-      MEILISEARCH_VERSION: ${{ needs.define-docker-image.outputs.docker-image }}
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-dotnet
-      - name: Setup .NET Core
-        uses: actions/setup-dotnet@v3
-        with:
-          dotnet-version: "6.0.x"
-      - name: Install dependencies
-        run: dotnet restore
-      - name: Build
-        run: dotnet build --configuration Release --no-restore
-      - name: Meilisearch (latest version) setup with Docker
-        run: docker compose up -d
-      - name: Run tests
-        run: dotnet test --no-restore --verbosity normal
-
-  meilisearch-dart-tests:
-    needs: define-docker-image
-    name: Dart SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-dart
-      - uses: dart-lang/setup-dart@v1
-        with:
-          sdk: 3.1.1
-      - name: Install dependencies
-        run: dart pub get
-      - name: Run integration tests
-        run: dart test --concurrency=4
-
-  meilisearch-go-tests:
-    needs: define-docker-image
-    name: Go SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: stable
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-go
-      - name: Get dependencies
-        run: |
-          go get -v -t -d ./...
-          if [ -f Gopkg.toml ]; then
-            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-            dep ensure
-          fi
-      - name: Run integration tests
-        run: go test -v ./...
-
-  meilisearch-java-tests:
-    needs: define-docker-image
-    name: Java SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-java
-      - name: Set up Java
-        uses: actions/setup-java@v3
-        with:
-          java-version: 8
-          distribution: 'zulu'
-          cache: gradle
-      - name: Grant execute permission for gradlew
-        run: chmod +x gradlew
-      - name: Build and run unit and integration tests
-        run: ./gradlew build integrationTest
 
   meilisearch-js-tests:
     needs: define-docker-image
@@ -178,6 +66,33 @@ jobs:
       - name: Run Browser env
         run: yarn test:env:browser
 
+  instant-meilisearch-tests:
+    needs: define-docker-image
+    name: instant-meilisearch tests
+    runs-on: ubuntu-latest
+    services:
+      meilisearch:
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        env:
+          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
+          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
+        ports:
+          - '7700:7700'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: meilisearch/instant-meilisearch
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          cache: yarn
+      - name: Install dependencies
+        run: yarn install
+      - name: Run tests
+        run: yarn test
+      - name: Build all the playgrounds and the packages
+        run: yarn build
+
   meilisearch-php-tests:
     needs: define-docker-image
     name: PHP SDK tests
@@ -196,6 +111,8 @@ jobs:
           repository: meilisearch/meilisearch-php
       - name: Install PHP
         uses: shivammathur/setup-php@v2
+        with:
+          coverage: none
       - name: Validate composer.json and composer.lock
         run: composer validate
       - name: Install dependencies
@@ -232,6 +149,36 @@ jobs:
       - name: Test with pytest
         run: pipenv run pytest
 
+  meilisearch-go-tests:
+    needs: define-docker-image
+    name: Go SDK tests
+    runs-on: ubuntu-latest
+    services:
+      meilisearch:
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        env:
+          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
+          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
+        ports:
+          - '7700:7700'
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: stable
+      - uses: actions/checkout@v3
+        with:
+          repository: meilisearch/meilisearch-go
+      - name: Get dependencies
+        run: |
+          go get -v -t -d ./...
+          if [ -f Gopkg.toml ]; then
+            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+            dep ensure
+          fi
+      - name: Run integration tests
+        run: go test -v ./...
+
   meilisearch-ruby-tests:
     needs: define-docker-image
     name: Ruby SDK tests
@@ -277,110 +224,3 @@ jobs:
         run: cargo build --verbose
       - name: Run tests
         run: cargo test --verbose
-
-  meilisearch-swift-tests:
-    needs: define-docker-image
-    name: Swift SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-swift
-      - name: Run tests
-        run: swift test
-
-########################
-## FRONT-END PLUGINS ##
-########################
-
-  meilisearch-js-plugins-tests:
-    needs: define-docker-image
-    name: meilisearch-js-plugins tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-js-plugins
-      - name: Setup node
-        uses: actions/setup-node@v3
-        with:
-          cache: yarn
-      - name: Install dependencies
-        run: yarn install
-      - name: Run tests
-        run: yarn test
-      - name: Build all the playgrounds and the packages
-        run: yarn build
-
-########################
-## BACK-END PLUGINS ###
-########################
-
-  meilisearch-rails-tests:
-    needs: define-docker-image
-    name: meilisearch-rails tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-rails
-      - name: Set up Ruby 3
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 3
-          bundler-cache: true
-      - name: Run tests
-        run: bundle exec rspec
-
-  meilisearch-symfony-tests:
-    needs: define-docker-image
-    name: meilisearch-symfony tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-symfony
-      - name: Install PHP
-        uses: shivammathur/setup-php@v2
-        with:
-          tools: composer:v2, flex
-      - name: Validate composer.json and composer.lock
-        run: composer validate
-      - name: Install dependencies
-        run: composer install --prefer-dist --no-progress --quiet
-      - name: Remove doctrine/annotations
-        run: composer remove --dev doctrine/annotations
-      - name: Run test suite
-        run: composer test:unit

.github/workflows/test-suite.yml

@@ -43,7 +43,7 @@ jobs:
           toolchain: nightly
           override: true
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
+        uses: Swatinem/rust-cache@v2.5.1
       - name: Run cargo check without any default features
         uses: actions-rs/cargo@v1
         with:
@@ -65,7 +65,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
+        uses: Swatinem/rust-cache@v2.5.1
       - name: Run cargo check without any default features
         uses: actions-rs/cargo@v1
         with:
@@ -123,10 +123,7 @@ jobs:
           override: true
       - name: Run cargo tree without default features and check lindera is not present
         run: |
-          if cargo tree -f '{p} {f}' -e normal --no-default-features | grep -vqz lindera; then
-            echo "lindera has been found in the sources and it shouldn't"
-            exit 1
-          fi
+          cargo tree -f '{p} {f}' -e normal --no-default-features | grep lindera -vqz
       - name: Run cargo tree with default features and check lindera is pressent
         run: |
           cargo tree -f '{p} {f}' -e normal | grep lindera -qz
@@ -149,7 +146,7 @@ jobs:
           toolchain: stable
           override: true
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
+        uses: Swatinem/rust-cache@v2.5.1
       - name: Run tests in debug
         uses: actions-rs/cargo@v1
         with:
@@ -168,7 +165,7 @@ jobs:
           override: true
           components: clippy
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
+        uses: Swatinem/rust-cache@v2.5.1
       - name: Run cargo clippy
         uses: actions-rs/cargo@v1
         with:
@@ -187,7 +184,7 @@ jobs:
           override: true
           components: rustfmt
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
+        uses: Swatinem/rust-cache@v2.5.1
       - name: Run cargo fmt
         # Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file.
         # Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate

Cargo.lock

@@ -468,7 +468,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
 
 [[package]]
 name = "benchmarks"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "anyhow",
  "bytes",
@@ -1206,7 +1206,7 @@ dependencies = [
 
 [[package]]
 name = "dump"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "anyhow",
  "big_s",
@@ -1417,7 +1417,7 @@ dependencies = [
 
 [[package]]
 name = "file-store"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "faux",
  "tempfile",
@@ -1439,7 +1439,7 @@ dependencies = [
 
 [[package]]
 name = "filter-parser"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "insta",
  "nom",
@@ -1459,7 +1459,7 @@ dependencies = [
 
 [[package]]
 name = "flatten-serde-json"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "criterion",
  "serde_json",
@@ -1577,7 +1577,7 @@ dependencies = [
 
 [[package]]
 name = "fuzzers"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "arbitrary",
  "clap",
@@ -1891,7 +1891,7 @@ dependencies = [
 
 [[package]]
 name = "index-scheduler"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "anyhow",
  "big_s",
@@ -2088,7 +2088,7 @@ dependencies = [
 
 [[package]]
 name = "json-depth-checker"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "criterion",
  "serde_json",
@@ -2500,7 +2500,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
 
 [[package]]
 name = "meili-snap"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "insta",
  "md5",
@@ -2509,7 +2509,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "actix-cors",
  "actix-http",
@@ -2600,7 +2600,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-auth"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "base64 0.21.2",
  "enum-iterator",
@@ -2619,7 +2619,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-types"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "actix-web",
  "anyhow",
@@ -2673,7 +2673,7 @@ dependencies = [
 
 [[package]]
 name = "milli"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "big_s",
  "bimap",
@@ -2995,7 +2995,7 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
 
 [[package]]
 name = "permissive-json-pointer"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "big_s",
  "serde_json",
@@ -3538,9 +3538,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-webpki"
-version = "0.100.2"
+version = "0.100.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e98ff011474fa39949b7e5c0428f9b4937eda7da7848bbb947786b7be0b27dab"
+checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b"
 dependencies = [
  "ring",
  "untrusted",
@@ -4248,7 +4248,7 @@ dependencies = [
  "log",
  "once_cell",
  "rustls 0.21.6",
- "rustls-webpki 0.100.2",
+ "rustls-webpki 0.100.1",
  "url",
  "webpki-roots 0.23.1",
 ]
@@ -4443,9 +4443,9 @@ dependencies = [
 
 [[package]]
 name = "webpki"
-version = "0.22.1"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0e74f82d49d545ad128049b7e88f6576df2da6b02e9ce565c6f533be576957e"
+checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd"
 dependencies = [
  "ring",
  "untrusted",
@@ -4466,7 +4466,7 @@ version = "0.23.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338"
 dependencies = [
- "rustls-webpki 0.100.2",
+ "rustls-webpki 0.100.1",
 ]
 
 [[package]]

Cargo.toml

@@ -18,7 +18,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.4.0"
+version = "1.4.1"
 authors = ["Quentin de Quelen <quentin@dequelen.me>", "Clément Renault <clement@meilisearch.com>"]
 description = "Meilisearch HTTP server"
 homepage = "https://meilisearch.com"

Dockerfile

@@ -17,7 +17,7 @@ RUN     set -eux; \
         if [ "$apkArch" = "aarch64" ]; then \
             export JEMALLOC_SYS_WITH_LG_PAGE=16; \
         fi && \
-        cargo build --release
+        cargo build --release --no-default-features --features "analytics mini-dashboard japanese"
 
 # Run
 FROM    alpine:3.16

meilisearch-auth/src/store.rs

@@ -129,9 +129,6 @@ impl HeedAuthStore {
                 Action::DumpsAll => {
                     actions.insert(Action::DumpsCreate);
                 }
-                Action::SnapshotsAll => {
-                    actions.insert(Action::SnapshotsCreate);
-                }
                 Action::TasksAll => {
                     actions.extend([Action::TasksGet, Action::TasksDelete, Action::TasksCancel]);
                 }

meilisearch-types/src/keys.rs

@@ -257,12 +257,6 @@ pub enum Action {
     #[serde(rename = "dumps.create")]
     #[deserr(rename = "dumps.create")]
     DumpsCreate,
-    #[serde(rename = "snapshots.*")]
-    #[deserr(rename = "snapshots.*")]
-    SnapshotsAll,
-    #[serde(rename = "snapshots.create")]
-    #[deserr(rename = "snapshots.create")]
-    SnapshotsCreate,
     #[serde(rename = "version")]
     #[deserr(rename = "version")]
     Version,
@@ -315,7 +309,6 @@ impl Action {
             METRICS_GET => Some(Self::MetricsGet),
             DUMPS_ALL => Some(Self::DumpsAll),
             DUMPS_CREATE => Some(Self::DumpsCreate),
-            SNAPSHOTS_CREATE => Some(Self::SnapshotsCreate),
             VERSION => Some(Self::Version),
             KEYS_CREATE => Some(Self::KeysAdd),
             KEYS_GET => Some(Self::KeysGet),
@@ -360,7 +353,6 @@ pub mod actions {
     pub const METRICS_GET: u8 = MetricsGet.repr();
     pub const DUMPS_ALL: u8 = DumpsAll.repr();
     pub const DUMPS_CREATE: u8 = DumpsCreate.repr();
-    pub const SNAPSHOTS_CREATE: u8 = SnapshotsCreate.repr();
     pub const VERSION: u8 = Version.repr();
     pub const KEYS_CREATE: u8 = KeysAdd.repr();
     pub const KEYS_GET: u8 = KeysGet.repr();

meilisearch/src/analytics/mod.rs

@@ -1,5 +1,6 @@
 mod mock_analytics;
-#[cfg(feature = "analytics")]
+// if we are in release mode and the feature analytics was enabled
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
 mod segment_analytics;
 
 use std::fs;
@@ -16,25 +17,26 @@ use serde_json::Value;
 use crate::routes::indexes::documents::UpdateDocumentsQuery;
 use crate::routes::tasks::TasksFilterQuery;
 
-// if the analytics feature is disabled
+// if we are in debug mode OR the analytics feature is disabled
 // the `SegmentAnalytics` point to the mock instead of the real analytics
-#[cfg(not(feature = "analytics"))]
+#[cfg(any(debug_assertions, not(feature = "analytics")))]
 pub type SegmentAnalytics = mock_analytics::MockAnalytics;
-#[cfg(not(feature = "analytics"))]
+#[cfg(any(debug_assertions, not(feature = "analytics")))]
 pub type SearchAggregator = mock_analytics::SearchAggregator;
-#[cfg(not(feature = "analytics"))]
+#[cfg(any(debug_assertions, not(feature = "analytics")))]
 pub type MultiSearchAggregator = mock_analytics::MultiSearchAggregator;
-#[cfg(not(feature = "analytics"))]
+#[cfg(any(debug_assertions, not(feature = "analytics")))]
 pub type FacetSearchAggregator = mock_analytics::FacetSearchAggregator;
 
-// if the feature analytics is enabled we use the real analytics
-#[cfg(feature = "analytics")]
+// if we are in release mode and the feature analytics was enabled
+// we use the real analytics
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
 pub type SegmentAnalytics = segment_analytics::SegmentAnalytics;
-#[cfg(feature = "analytics")]
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
 pub type SearchAggregator = segment_analytics::SearchAggregator;
-#[cfg(feature = "analytics")]
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
 pub type MultiSearchAggregator = segment_analytics::MultiSearchAggregator;
-#[cfg(feature = "analytics")]
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
 pub type FacetSearchAggregator = segment_analytics::FacetSearchAggregator;
 
 /// The Meilisearch config dir:

meilisearch/src/option.rs

@@ -28,7 +28,7 @@ const MEILI_DB_PATH: &str = "MEILI_DB_PATH";
 const MEILI_HTTP_ADDR: &str = "MEILI_HTTP_ADDR";
 const MEILI_MASTER_KEY: &str = "MEILI_MASTER_KEY";
 const MEILI_ENV: &str = "MEILI_ENV";
-#[cfg(feature = "analytics")]
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
 const MEILI_NO_ANALYTICS: &str = "MEILI_NO_ANALYTICS";
 const MEILI_HTTP_PAYLOAD_SIZE_LIMIT: &str = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT";
 const MEILI_SSL_CERT_PATH: &str = "MEILI_SSL_CERT_PATH";
@@ -159,7 +159,7 @@ pub struct Opt {
     /// Meilisearch automatically collects data from all instances that do not opt out using this flag.
     /// All gathered data is used solely for the purpose of improving Meilisearch, and can be deleted
     /// at any time.
-    #[cfg(feature = "analytics")]
+    #[cfg(all(not(debug_assertions), feature = "analytics"))]
     #[serde(default)] // we can't send true
     #[clap(long, env = MEILI_NO_ANALYTICS)]
     pub no_analytics: bool,
@@ -390,7 +390,7 @@ impl Opt {
             ignore_missing_dump: _,
             ignore_dump_if_db_exists: _,
             config_file_path: _,
-            #[cfg(feature = "analytics")]
+            #[cfg(all(not(debug_assertions), feature = "analytics"))]
             no_analytics,
             experimental_enable_metrics: enable_metrics_route,
             experimental_reduce_indexing_memory_usage: reduce_indexing_memory_usage,
@@ -401,7 +401,7 @@ impl Opt {
             export_to_env_if_not_present(MEILI_MASTER_KEY, master_key);
         }
         export_to_env_if_not_present(MEILI_ENV, env);
-        #[cfg(feature = "analytics")]
+        #[cfg(all(not(debug_assertions), feature = "analytics"))]
         {
             export_to_env_if_not_present(MEILI_NO_ANALYTICS, no_analytics.to_string());
         }

meilisearch/src/routes/mod.rs

@@ -24,7 +24,6 @@ pub mod features;
 pub mod indexes;
 mod metrics;
 mod multi_search;
-mod snapshot;
 mod swap_indexes;
 pub mod tasks;
 
@@ -33,7 +32,6 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
         .service(web::resource("/health").route(web::get().to(get_health)))
         .service(web::scope("/keys").configure(api_key::configure))
         .service(web::scope("/dumps").configure(dump::configure))
-        .service(web::scope("/snapshots").configure(snapshot::configure))
         .service(web::resource("/stats").route(web::get().to(get_stats)))
         .service(web::resource("/version").route(web::get().to(get_version)))
         .service(web::scope("/indexes").configure(indexes::configure))

meilisearch/src/routes/snapshot.rs

@@ -1,32 +0,0 @@
-use actix_web::web::Data;
-use actix_web::{web, HttpRequest, HttpResponse};
-use index_scheduler::IndexScheduler;
-use log::debug;
-use meilisearch_types::error::ResponseError;
-use meilisearch_types::tasks::KindWithContent;
-use serde_json::json;
-
-use crate::analytics::Analytics;
-use crate::extractors::authentication::policies::*;
-use crate::extractors::authentication::GuardedData;
-use crate::extractors::sequential_extractor::SeqHandler;
-use crate::routes::SummarizedTaskView;
-
-pub fn configure(cfg: &mut web::ServiceConfig) {
-    cfg.service(web::resource("").route(web::post().to(SeqHandler(create_snapshot))));
-}
-
-pub async fn create_snapshot(
-    index_scheduler: GuardedData<ActionPolicy<{ actions::SNAPSHOTS_CREATE }>, Data<IndexScheduler>>,
-    req: HttpRequest,
-    analytics: web::Data<dyn Analytics>,
-) -> Result<HttpResponse, ResponseError> {
-    analytics.publish("Snapshot Created".to_string(), json!({}), Some(&req));
-
-    let task = KindWithContent::SnapshotCreation;
-    let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
-
-    debug!("returns: {:?}", task);
-    Ok(HttpResponse::Accepted().json(task))
-}

meilisearch/tests/auth/api_keys.rs

@@ -1,7 +1,8 @@
 use std::{thread, time};
 
-use crate::common::{Server, Value};
-use crate::json;
+use serde_json::{json, Value};
+
+use crate::common::Server;
 
 #[actix_rt::test]
 async fn add_valid_api_key() {
@@ -161,7 +162,7 @@ async fn add_valid_api_key_null_description() {
     server.use_api_key("MASTER_KEY");
 
     let content = json!({
-        "description": json!(null),
+        "description": Value::Null,
         "indexes": ["products"],
         "actions": ["documents.add"],
         "expiresAt": "2050-11-13T00:00:00"
@@ -364,7 +365,7 @@ async fn error_add_api_key_invalid_index_uids() {
     server.use_api_key("MASTER_KEY");
 
     let content = json!({
-        "description": json!(null),
+        "description": Value::Null,
         "indexes": ["invalid index # / \\name with spaces"],
         "actions": [
             "documents.add"
@@ -421,7 +422,7 @@ async fn error_add_api_key_invalid_parameters_actions() {
     meili_snap::snapshot!(code, @"400 Bad Request");
     meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]" }), @r###"
     {
-      "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`",
+      "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`",
       "code": "invalid_api_key_actions",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions"
@@ -506,7 +507,7 @@ async fn error_add_api_key_invalid_parameters_uid() {
 async fn error_add_api_key_parameters_uid_already_exist() {
     let mut server = Server::new_auth().await;
     server.use_api_key("MASTER_KEY");
-    let content: Value = json!({
+    let content = json!({
         "uid": "4bc0887a-0e41-4f3b-935d-0c451dcee9c8",
         "indexes": ["products"],
         "actions": ["search"],
@@ -1145,7 +1146,7 @@ async fn patch_api_key_description() {
     meili_snap::snapshot!(code, @"200 OK");
 
     // Remove the description
-    let content = json!({ "description": null });
+    let content = json!({ "description": serde_json::Value::Null });
 
     let (response, code) = server.patch_api_key(&uid, content).await;
     meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]", ".uid" => "[ignored]", ".key" => "[ignored]" }), @r###"

meilisearch/tests/auth/authorization.rs

@@ -3,10 +3,10 @@ use std::collections::{HashMap, HashSet};
 use ::time::format_description::well_known::Rfc3339;
 use maplit::{hashmap, hashset};
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 use time::{Duration, OffsetDateTime};
 
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 pub static AUTHORIZATIONS: Lazy<HashMap<(&'static str, &'static str), HashSet<&'static str>>> =
     Lazy::new(|| {
@@ -54,7 +54,6 @@ pub static AUTHORIZATIONS: Lazy<HashMap<(&'static str, &'static str), HashSet<&'
             ("GET",     "/indexes/products/stats") =>                          hashset!{"stats.get", "stats.*", "*"},
             ("GET",     "/stats") =>                                           hashset!{"stats.get", "stats.*", "*"},
             ("POST",    "/dumps") =>                                           hashset!{"dumps.create", "dumps.*", "*"},
-            ("POST",    "/snapshots") =>                                       hashset!{"snapshots.create", "snapshots.*", "*"},
             ("GET",     "/version") =>                                         hashset!{"version", "*"},
             ("GET",     "/metrics") =>                                         hashset!{"metrics.get", "metrics.*", "*"},
             ("PATCH",   "/keys/mykey/") =>                                     hashset!{"keys.update", "*"},

meilisearch/tests/auth/errors.rs

@@ -1,8 +1,8 @@
 use meili_snap::*;
+use serde_json::json;
 use uuid::Uuid;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn create_api_key_bad_description() {
@@ -90,7 +90,7 @@ async fn create_api_key_bad_actions() {
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`",
+      "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`",
       "code": "invalid_api_key_actions",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions"

meilisearch/tests/auth/mod.rs

@@ -7,9 +7,9 @@ mod tenant_token;
 mod tenant_token_multi_search;
 
 use actix_web::http::StatusCode;
+use serde_json::{json, Value};
 
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 impl Server {
     pub fn use_api_key(&mut self, api_key: impl AsRef<str>) {

meilisearch/tests/auth/tenant_token.rs

@@ -3,11 +3,11 @@ use std::collections::HashMap;
 use ::time::format_description::well_known::Rfc3339;
 use maplit::hashmap;
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 use time::{Duration, OffsetDateTime};
 
 use super::authorization::{ALL_ACTIONS, AUTHORIZATIONS};
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 fn generate_tenant_token(
     parent_uid: impl AsRef<str>,
@@ -233,31 +233,31 @@ async fn search_authorized_simple_token() {
         },
         hashmap! {
             "searchRules" => json!({"*": {}}),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
-            "exp" => json!(null)
+            "searchRules" => json!({"*": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["*"]),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!({"sales": {}}),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
-            "exp" => json!(null)
+            "searchRules" => json!({"sales": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sales"]),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sa*"]),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
     ];
 
@@ -386,7 +386,7 @@ async fn error_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
+            "searchRules" => json!({"*": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -398,7 +398,7 @@ async fn error_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
+            "searchRules" => json!({"sales": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -428,15 +428,15 @@ async fn error_search_forbidden_token() {
         },
         hashmap! {
             "searchRules" => json!({"products": {}}),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"products": null}),
-            "exp" => json!(null)
+            "searchRules" => json!({"products": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["products"]),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
         // expired token
         hashmap! {
@@ -444,7 +444,7 @@ async fn error_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
+            "searchRules" => json!({"*": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -456,7 +456,7 @@ async fn error_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
+            "searchRules" => json!({"sales": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {

meilisearch/tests/auth/tenant_token_multi_search.rs

@@ -3,11 +3,11 @@ use std::collections::HashMap;
 use ::time::format_description::well_known::Rfc3339;
 use maplit::hashmap;
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 use time::{Duration, OffsetDateTime};
 
 use super::authorization::ALL_ACTIONS;
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 fn generate_tenant_token(
     parent_uid: impl AsRef<str>,
@@ -512,31 +512,31 @@ async fn single_search_authorized_simple_token() {
         },
         hashmap! {
             "searchRules" => json!({"*": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"*": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["*"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!({"sales": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"sales": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sales"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sa*"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
     ];
 
@@ -564,31 +564,31 @@ async fn multi_search_authorized_simple_token() {
         },
         hashmap! {
             "searchRules" => json!({"*": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"*": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["*"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!({"sales": {}, "products": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"sales": null, "products": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"sales": Value::Null, "products": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sales", "products"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sa*", "pro*"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
     ];
 
@@ -823,7 +823,7 @@ async fn error_single_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
+            "searchRules" => json!({"*": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -835,7 +835,7 @@ async fn error_single_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
+            "searchRules" => json!({"sales": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -864,7 +864,7 @@ async fn error_multi_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
+            "searchRules" => json!({"*": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -876,7 +876,7 @@ async fn error_multi_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": null, "products": null}),
+            "searchRules" => json!({"sales": Value::Null, "products": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -919,15 +919,15 @@ async fn error_single_search_forbidden_token() {
         },
         hashmap! {
             "searchRules" => json!({"products": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"products": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"products": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["products"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         // expired token
         hashmap! {
@@ -935,7 +935,7 @@ async fn error_single_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
+            "searchRules" => json!({"*": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -947,7 +947,7 @@ async fn error_single_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
+            "searchRules" => json!({"sales": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -978,15 +978,15 @@ async fn error_multi_search_forbidden_token() {
         },
         hashmap! {
             "searchRules" => json!({"products": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"products": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"products": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["products"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!({"sales": {}}),
@@ -998,15 +998,15 @@ async fn error_multi_search_forbidden_token() {
         },
         hashmap! {
             "searchRules" => json!({"sales": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"sales": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sales"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         // expired token
         hashmap! {
@@ -1014,7 +1014,7 @@ async fn error_multi_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
+            "searchRules" => json!({"*": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -1026,7 +1026,7 @@ async fn error_multi_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": null, "products": {}}),
+            "searchRules" => json!({"sales": Value::Null, "products": {}}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {

meilisearch/tests/common/index.rs

@@ -3,13 +3,12 @@ use std::panic::{catch_unwind, resume_unwind, UnwindSafe};
 use std::time::Duration;
 
 use actix_web::http::StatusCode;
+use serde_json::{json, Value};
 use tokio::time::sleep;
 use urlencoding::encode as urlencode;
 
 use super::encoder::Encoder;
 use super::service::Service;
-use super::Value;
-use crate::json;
 
 pub struct Index<'a> {
     pub uid: String,
@@ -243,9 +242,7 @@ impl Index<'_> {
 
     pub async fn delete_batch(&self, ids: Vec<u64>) -> (Value, StatusCode) {
         let url = format!("/indexes/{}/documents/delete-batch", urlencode(self.uid.as_ref()));
-        self.service
-            .post_encoded(url, serde_json::to_value(&ids).unwrap().into(), self.encoder)
-            .await
+        self.service.post_encoded(url, serde_json::to_value(&ids).unwrap(), self.encoder).await
     }
 
     pub async fn delete_batch_raw(&self, body: Value) -> (Value, StatusCode) {

meilisearch/tests/common/mod.rs

@@ -3,83 +3,9 @@ pub mod index;
 pub mod server;
 pub mod service;
 
-use std::fmt::{self, Display};
-
 pub use index::{GetAllDocumentsOptions, GetDocumentOptions};
-use meili_snap::json_string;
-use serde::{Deserialize, Serialize};
 pub use server::{default_settings, Server};
 
-#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
-pub struct Value(pub serde_json::Value);
-
-impl Value {
-    pub fn uid(&self) -> u64 {
-        if let Some(uid) = self["uid"].as_u64() {
-            uid
-        } else if let Some(uid) = self["taskUid"].as_u64() {
-            uid
-        } else {
-            panic!("Didn't find any task id in: {self}");
-        }
-    }
-}
-
-impl From<serde_json::Value> for Value {
-    fn from(value: serde_json::Value) -> Self {
-        Value(value)
-    }
-}
-
-impl std::ops::Deref for Value {
-    type Target = serde_json::Value;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl PartialEq<serde_json::Value> for Value {
-    fn eq(&self, other: &serde_json::Value) -> bool {
-        &self.0 == other
-    }
-}
-
-impl PartialEq<Value> for serde_json::Value {
-    fn eq(&self, other: &Value) -> bool {
-        self == &other.0
-    }
-}
-
-impl PartialEq<&str> for Value {
-    fn eq(&self, other: &&str) -> bool {
-        self.0.eq(other)
-    }
-}
-
-impl Display for Value {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(
-            f,
-            "{}",
-            json_string!(self, { ".enqueuedAt" => "[date]", ".processedAt" => "[date]", ".finishedAt" => "[date]", ".duration" => "[duration]" })
-        )
-    }
-}
-
-impl From<Vec<Value>> for Value {
-    fn from(value: Vec<Value>) -> Self {
-        Self(value.into_iter().map(|value| value.0).collect::<serde_json::Value>())
-    }
-}
-
-#[macro_export]
-macro_rules! json {
-    ($($json:tt)+) => {
-        $crate::common::Value(serde_json::json!($($json)+))
-    };
-}
-
 /// Performs a search test on both post and get routes
 #[macro_export]
 macro_rules! test_post_get_search {

meilisearch/tests/common/server.rs

@@ -11,14 +11,13 @@ use clap::Parser;
 use meilisearch::option::{IndexerOpts, MaxMemory, Opt};
 use meilisearch::{analytics, create_app, setup_meilisearch};
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 use tempfile::TempDir;
 use tokio::time::sleep;
 
 use super::index::Index;
 use super::service::Service;
 use crate::common::encoder::Encoder;
-use crate::common::Value;
-use crate::json;
 
 pub struct Server {
     pub service: Service,
@@ -157,10 +156,6 @@ impl Server {
         self.service.post("/dumps", json!(null)).await
     }
 
-    pub async fn create_snapshot(&self) -> (Value, StatusCode) {
-        self.service.post("/snapshots", json!(null)).await
-    }
-
     pub async fn index_swap(&self, value: Value) -> (Value, StatusCode) {
         self.service.post("/swap-indexes", value).await
     }
@@ -209,7 +204,7 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
         db_path: dir.as_ref().join("db"),
         dump_dir: dir.as_ref().join("dumps"),
         env: "development".to_owned(),
-        #[cfg(feature = "analytics")]
+        #[cfg(all(not(debug_assertions), feature = "analytics"))]
         no_analytics: true,
         max_index_size: Byte::from_unit(100.0, ByteUnit::MiB).unwrap(),
         max_task_db_size: Byte::from_unit(1.0, ByteUnit::GiB).unwrap(),

meilisearch/tests/common/service.rs

@@ -7,9 +7,9 @@ use actix_web::test::TestRequest;
 use index_scheduler::IndexScheduler;
 use meilisearch::{analytics, create_app, Opt};
 use meilisearch_auth::AuthController;
+use serde_json::Value;
 
 use crate::common::encoder::Encoder;
-use crate::common::Value;
 
 pub struct Service {
     pub index_scheduler: Arc<IndexScheduler>,

meilisearch/tests/content_type.rs

@@ -3,8 +3,9 @@
 mod common;
 
 use actix_web::test;
+use serde_json::{json, Value};
 
-use crate::common::{Server, Value};
+use crate::common::Server;
 
 enum HttpVerb {
     Put,

meilisearch/tests/documents/add_documents.rs

@@ -1,11 +1,11 @@
 use actix_web::test;
 use meili_snap::{json_string, snapshot};
+use serde_json::{json, Value};
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
 use crate::common::encoder::Encoder;
-use crate::common::{GetAllDocumentsOptions, Server, Value};
-use crate::json;
+use crate::common::{GetAllDocumentsOptions, Server};
 
 /// This is the basic usage of our API and every other tests uses the content-type application/json
 #[actix_rt::test]

meilisearch/tests/documents/delete_documents.rs

@@ -1,7 +1,7 @@
 use meili_snap::{json_string, snapshot};
+use serde_json::json;
 
 use crate::common::{GetAllDocumentsOptions, Server};
-use crate::json;
 
 #[actix_rt::test]
 async fn delete_one_document_unexisting_index() {

meilisearch/tests/documents/errors.rs

@@ -1,8 +1,8 @@
 use meili_snap::*;
+use serde_json::json;
 use urlencoding::encode;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn get_all_documents_bad_offset() {

meilisearch/tests/documents/get_documents.rs

@@ -1,11 +1,11 @@
 use actix_web::test;
 use http::header::ACCEPT_ENCODING;
 use meili_snap::*;
+use serde_json::{json, Value};
 use urlencoding::encode as urlencode;
 
 use crate::common::encoder::Encoder;
-use crate::common::{GetAllDocumentsOptions, GetDocumentOptions, Server, Value};
-use crate::json;
+use crate::common::{GetAllDocumentsOptions, GetDocumentOptions, Server};
 
 // TODO: partial test since we are testing error, amd error is not yet fully implemented in
 // transplant
@@ -40,7 +40,7 @@ async fn get_document() {
     let server = Server::new().await;
     let index = server.index("test");
     index.create(None).await;
-    let documents = json!([
+    let documents = serde_json::json!([
         {
             "id": 0,
             "nested": { "content": "foobar" },
@@ -53,7 +53,7 @@ async fn get_document() {
     assert_eq!(code, 200);
     assert_eq!(
         response,
-        json!({
+        serde_json::json!({
             "id": 0,
             "nested": { "content": "foobar" },
         })
@@ -64,7 +64,7 @@ async fn get_document() {
     assert_eq!(code, 200);
     assert_eq!(
         response,
-        json!({
+        serde_json::json!({
             "id": 0,
         })
     );
@@ -75,7 +75,7 @@ async fn get_document() {
     assert_eq!(code, 200);
     assert_eq!(
         response,
-        json!({
+        serde_json::json!({
             "nested": { "content": "foobar" },
         })
     );
@@ -122,7 +122,7 @@ async fn get_all_documents_no_options() {
     assert_eq!(code, 200);
     let arr = response["results"].as_array().unwrap();
     assert_eq!(arr.len(), 20);
-    let first = json!({
+    let first = serde_json::json!({
         "id":0,
         "isActive":false,
         "balance":"$2,668.55",

meilisearch/tests/documents/update_documents.rs

@@ -1,8 +1,7 @@
-use meili_snap::snapshot;
+use serde_json::json;
 
 use crate::common::encoder::Encoder;
 use crate::common::{GetAllDocumentsOptions, Server};
-use crate::json;
 
 #[actix_rt::test]
 async fn error_document_update_create_index_bad_uid() {
@@ -85,13 +84,7 @@ async fn update_document() {
 
     let (response, code) = index.get_document(1, None).await;
     assert_eq!(code, 200);
-    snapshot!(response, @r###"
-    {
-      "doc_id": 1,
-      "content": "foo",
-      "other": "bar"
-    }
-    "###);
+    assert_eq!(response.to_string(), r##"{"doc_id":1,"content":"foo","other":"bar"}"##);
 }
 
 #[actix_rt::test]
@@ -129,13 +122,7 @@ async fn update_document_gzip_encoded() {
 
     let (response, code) = index.get_document(1, None).await;
     assert_eq!(code, 200);
-    snapshot!(response, @r###"
-    {
-      "doc_id": 1,
-      "content": "foo",
-      "other": "bar"
-    }
-    "###);
+    assert_eq!(response.to_string(), r##"{"doc_id":1,"content":"foo","other":"bar"}"##);
 }
 
 #[actix_rt::test]

meilisearch/tests/dumps/mod.rs

@@ -2,10 +2,10 @@ mod data;
 
 use meili_snap::{json_string, snapshot};
 use meilisearch::Opt;
+use serde_json::json;
 
 use self::data::GetDump;
 use crate::common::{default_settings, GetAllDocumentsOptions, Server};
-use crate::json;
 
 // all the following test are ignored on windows. See #2364
 #[actix_rt::test]

meilisearch/tests/features/mod.rs

@@ -1,5 +1,6 @@
+use serde_json::json;
+
 use crate::common::Server;
-use crate::json;
 
 /// Feature name to test against.
 /// This will have to be changed by a different one when that feature is stabilized.

meilisearch/tests/index/create_index.rs

@@ -2,10 +2,10 @@ use actix_web::http::header::ContentType;
 use actix_web::test;
 use http::header::ACCEPT_ENCODING;
 use meili_snap::{json_string, snapshot};
+use serde_json::{json, Value};
 
 use crate::common::encoder::Encoder;
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 #[actix_rt::test]
 async fn create_index_no_primary_key() {
@@ -21,7 +21,7 @@ async fn create_index_no_primary_key() {
 
     assert_eq!(response["status"], "succeeded");
     assert_eq!(response["type"], "indexCreation");
-    assert_eq!(response["details"]["primaryKey"], json!(null));
+    assert_eq!(response["details"]["primaryKey"], Value::Null);
 }
 
 #[actix_rt::test]
@@ -38,7 +38,7 @@ async fn create_index_with_gzip_encoded_request() {
 
     assert_eq!(response["status"], "succeeded");
     assert_eq!(response["type"], "indexCreation");
-    assert_eq!(response["details"]["primaryKey"], json!(null));
+    assert_eq!(response["details"]["primaryKey"], Value::Null);
 }
 
 #[actix_rt::test]
@@ -86,7 +86,7 @@ async fn create_index_with_zlib_encoded_request() {
 
     assert_eq!(response["status"], "succeeded");
     assert_eq!(response["type"], "indexCreation");
-    assert_eq!(response["details"]["primaryKey"], json!(null));
+    assert_eq!(response["details"]["primaryKey"], Value::Null);
 }
 
 #[actix_rt::test]
@@ -103,7 +103,7 @@ async fn create_index_with_brotli_encoded_request() {
 
     assert_eq!(response["status"], "succeeded");
     assert_eq!(response["type"], "indexCreation");
-    assert_eq!(response["details"]["primaryKey"], json!(null));
+    assert_eq!(response["details"]["primaryKey"], Value::Null);
 }
 
 #[actix_rt::test]
@@ -136,7 +136,7 @@ async fn create_index_with_invalid_primary_key() {
 
     let (response, code) = index.get().await;
     assert_eq!(code, 200);
-    assert_eq!(response["primaryKey"], json!(null));
+    assert_eq!(response["primaryKey"], Value::Null);
 }
 
 #[actix_rt::test]

meilisearch/tests/index/delete_index.rs

@@ -1,5 +1,6 @@
+use serde_json::json;
+
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn create_and_delete_index() {

meilisearch/tests/index/errors.rs

@@ -1,7 +1,7 @@
 use meili_snap::*;
+use serde_json::json;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn get_indexes_bad_offset() {

meilisearch/tests/index/stats.rs

@@ -1,5 +1,6 @@
+use serde_json::json;
+
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn stats() {

meilisearch/tests/index/update_index.rs

@@ -1,9 +1,9 @@
+use serde_json::json;
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
 use crate::common::encoder::Encoder;
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn update_primary_key() {

meilisearch/tests/search/distinct.rs

@@ -0,0 +1,63 @@
+use meili_snap::snapshot;
+use once_cell::sync::Lazy;
+use serde_json::{json, Value};
+
+use crate::common::Server;
+
+pub(self) static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
+    json!([
+        {"productId": 1, "shopId": 1},
+        {"productId": 2, "shopId": 1},
+        {"productId": 3, "shopId": 2},
+        {"productId": 4, "shopId": 2},
+        {"productId": 5, "shopId": 3},
+        {"productId": 6, "shopId": 3},
+        {"productId": 7, "shopId": 4},
+        {"productId": 8, "shopId": 4},
+        {"productId": 9, "shopId": 5},
+        {"productId": 10, "shopId": 5}
+    ])
+});
+
+pub(self) static DOCUMENT_PRIMARY_KEY: &str = "productId";
+pub(self) static DOCUMENT_DISTINCT_KEY: &str = "shopId";
+
+/// testing: https://github.com/meilisearch/meilisearch/issues/4078
+#[actix_rt::test]
+async fn distinct_search_with_offset_no_ranking() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let documents = DOCUMENTS.clone();
+    index.add_documents(documents, Some(DOCUMENT_PRIMARY_KEY)).await;
+    index.update_distinct_attribute(json!(DOCUMENT_DISTINCT_KEY)).await;
+    index.wait_task(1).await;
+
+    fn get_hits(response: Value) -> Vec<i64> {
+        let hits_array = response["hits"].as_array().unwrap();
+        hits_array.iter().map(|h| h[DOCUMENT_DISTINCT_KEY].as_i64().unwrap()).collect::<Vec<_>>()
+    }
+
+    let (response, code) = index.search_post(json!({"limit": 2, "offset": 0})).await;
+    let hits = get_hits(response);
+    snapshot!(code, @"200 OK");
+    snapshot!(hits.len(), @"2");
+    snapshot!(format!("{:?}", hits), @"[1, 2]");
+
+    let (response, code) = index.search_post(json!({"limit": 2, "offset": 2})).await;
+    let hits = get_hits(response);
+    snapshot!(code, @"200 OK");
+    snapshot!(hits.len(), @"2");
+    snapshot!(format!("{:?}", hits), @"[3, 4]");
+
+    let (response, code) = index.search_post(json!({"limit": 10, "offset": 4})).await;
+    let hits = get_hits(response);
+    snapshot!(code, @"200 OK");
+    snapshot!(hits.len(), @"1");
+    snapshot!(format!("{:?}", hits), @"[5]");
+
+    let (response, code) = index.search_post(json!({"limit": 10, "offset": 5})).await;
+    let hits = get_hits(response);
+    snapshot!(code, @"200 OK");
+    snapshot!(hits.len(), @"0");
+}

meilisearch/tests/search/errors.rs

@@ -1,8 +1,8 @@
 use meili_snap::*;
+use serde_json::json;
 
 use super::DOCUMENTS;
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn search_unexisting_index() {

meilisearch/tests/search/facet_search.rs

@@ -1,8 +1,8 @@
 use meili_snap::snapshot;
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 pub(self) static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
     json!([

meilisearch/tests/search/formatted.rs

@@ -1,8 +1,8 @@
 use insta::{allow_duplicates, assert_json_snapshot};
+use serde_json::json;
 
 use super::*;
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn formatted_contain_wildcard() {

meilisearch/tests/search/geo.rs

@@ -1,8 +1,8 @@
 use meili_snap::{json_string, snapshot};
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 pub(self) static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
     json!([

meilisearch/tests/search/mod.rs

@@ -1,6 +1,7 @@
 // This modules contains all the test concerning search. Each particular feature of the search
 // should be tested in its own module to isolate tests and keep the tests readable.
 
+mod distinct;
 mod errors;
 mod facet_search;
 mod formatted;
@@ -10,9 +11,9 @@ mod pagination;
 mod restrict_searchable;
 
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 pub(self) static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
     json!([

meilisearch/tests/search/multi.rs

@@ -1,8 +1,8 @@
 use meili_snap::{json_string, snapshot};
+use serde_json::json;
 
 use super::{DOCUMENTS, NESTED_DOCUMENTS};
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn search_empty_list() {

meilisearch/tests/search/pagination.rs

@@ -1,5 +1,6 @@
+use serde_json::json;
+
 use crate::common::Server;
-use crate::json;
 use crate::search::DOCUMENTS;
 
 #[actix_rt::test]

meilisearch/tests/search/restrict_searchable.rs

@@ -1,9 +1,9 @@
 use meili_snap::{json_string, snapshot};
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 
 use crate::common::index::Index;
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 async fn index_with_documents<'a>(server: &'a Server, documents: &Value) -> Index<'a> {
     let index = server.index("test");

meilisearch/tests/settings/distinct.rs

@@ -1,5 +1,6 @@
+use serde_json::json;
+
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn set_and_reset_distinct_attribute() {

meilisearch/tests/settings/errors.rs

@@ -1,7 +1,7 @@
 use meili_snap::*;
+use serde_json::json;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn settings_bad_displayed_attributes() {

meilisearch/tests/settings/get_settings.rs

@@ -1,16 +1,16 @@
 use std::collections::HashMap;
 
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 static DEFAULT_SETTINGS_VALUES: Lazy<HashMap<&'static str, Value>> = Lazy::new(|| {
     let mut map = HashMap::new();
     map.insert("displayed_attributes", json!(["*"]));
     map.insert("searchable_attributes", json!(["*"]));
     map.insert("filterable_attributes", json!([]));
-    map.insert("distinct_attribute", json!(null));
+    map.insert("distinct_attribute", json!(Value::Null));
     map.insert(
         "ranking_rules",
         json!(["words", "typo", "proximity", "attribute", "sort", "exactness"]),
@@ -229,7 +229,7 @@ macro_rules! test_setting_routes {
                         .chars()
                         .map(|c| if c == '_' { '-' } else { c })
                         .collect::<String>());
-                    let (response, code) = server.service.$write_method(url, serde_json::Value::Null.into()).await;
+                    let (response, code) = server.service.$write_method(url, serde_json::Value::Null).await;
                     assert_eq!(code, 202, "{}", response);
                     server.index("").wait_task(0).await;
                     let (response, code) = server.index("test").get().await;

meilisearch/tests/settings/tokenizer_customization.rs

@@ -1,7 +1,7 @@
 use meili_snap::{json_string, snapshot};
+use serde_json::json;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn set_and_reset() {

meilisearch/tests/snapshot/mod.rs

@@ -1,13 +1,11 @@
 use std::time::Duration;
 
 use actix_rt::time::sleep;
-use meili_snap::{json_string, snapshot};
 use meilisearch::option::ScheduleSnapshot;
 use meilisearch::Opt;
 
 use crate::common::server::default_settings;
 use crate::common::{GetAllDocumentsOptions, Server};
-use crate::json;
 
 macro_rules! verify_snapshot {
     (
@@ -46,7 +44,7 @@ async fn perform_snapshot() {
 
     let index = server.index("test");
     index
-        .update_settings(json! ({
+        .update_settings(serde_json::json! ({
         "searchableAttributes": [],
         }))
         .await;
@@ -92,95 +90,3 @@ async fn perform_snapshot() {
         server.index("test1").settings(),
     );
 }
-
-#[actix_rt::test]
-async fn perform_on_demand_snapshot() {
-    let temp = tempfile::tempdir().unwrap();
-    let snapshot_dir = tempfile::tempdir().unwrap();
-
-    let options =
-        Opt { snapshot_dir: snapshot_dir.path().to_owned(), ..default_settings(temp.path()) };
-
-    let server = Server::new_with_options(options).await.unwrap();
-
-    let index = server.index("catto");
-    index
-        .update_settings(json! ({
-        "searchableAttributes": [],
-        }))
-        .await;
-
-    index.load_test_set().await;
-
-    server.index("doggo").create(Some("bone")).await;
-    index.wait_task(2).await;
-
-    server.index("doggo").create(Some("bone")).await;
-    index.wait_task(2).await;
-
-    let (task, code) = server.create_snapshot().await;
-    snapshot!(code, @"202 Accepted");
-    snapshot!(json_string!(task, { ".enqueuedAt" => "[date]" }), @r###"
-    {
-      "taskUid": 4,
-      "indexUid": null,
-      "status": "enqueued",
-      "type": "snapshotCreation",
-      "enqueuedAt": "[date]"
-    }
-    "###);
-    let task = index.wait_task(task.uid()).await;
-    snapshot!(json_string!(task, { ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".duration" => "[duration]" }), @r###"
-    {
-      "uid": 4,
-      "indexUid": null,
-      "status": "succeeded",
-      "type": "snapshotCreation",
-      "canceledBy": null,
-      "error": null,
-      "duration": "[duration]",
-      "enqueuedAt": "[date]",
-      "startedAt": "[date]",
-      "finishedAt": "[date]"
-    }
-    "###);
-
-    let temp = tempfile::tempdir().unwrap();
-
-    let snapshots: Vec<String> = std::fs::read_dir(&snapshot_dir)
-        .unwrap()
-        .map(|entry| entry.unwrap().path().file_name().unwrap().to_str().unwrap().to_string())
-        .collect();
-    meili_snap::snapshot!(format!("{snapshots:?}"), @r###"["db.snapshot"]"###);
-
-    let snapshot_path = snapshot_dir.path().to_owned().join("db.snapshot");
-    #[cfg_attr(windows, allow(unused))]
-    let snapshot_meta = std::fs::metadata(&snapshot_path).unwrap();
-
-    #[cfg(unix)]
-    {
-        use std::os::unix::fs::PermissionsExt;
-        let mode = snapshot_meta.permissions().mode();
-        //                                                    rwxrwxrwx
-        meili_snap::snapshot!(format!("{:b}", mode), @"1000000100100100");
-    }
-
-    let options = Opt { import_snapshot: Some(snapshot_path), ..default_settings(temp.path()) };
-
-    let snapshot_server = Server::new_with_options(options).await.unwrap();
-
-    verify_snapshot!(server, snapshot_server, |server| =>
-        server.list_indexes(None, None),
-        // for some reason the db sizes differ. this may be due to the compaction options we have
-        // set when performing the snapshot
-        //server.stats(),
-
-        // The original instance contains the snapshotCreation task, while the snapshotted-instance does not. For this reason we need to compare the task queue **after** the task 4
-        server.tasks_filter("?from=2"),
-
-        server.index("catto").get_all_documents(GetAllDocumentsOptions::default()),
-        server.index("catto").settings(),
-        server.index("doggo").get_all_documents(GetAllDocumentsOptions::default()),
-        server.index("doggo").settings(),
-    );
-}

meilisearch/tests/stats/mod.rs

@@ -1,8 +1,8 @@
+use serde_json::json;
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn get_settings_unexisting_index() {

meilisearch/tests/swap_indexes/errors.rs

@@ -1,7 +1,7 @@
 use meili_snap::*;
+use serde_json::json;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn swap_indexes_bad_format() {

meilisearch/tests/swap_indexes/mod.rs

@@ -1,9 +1,9 @@
 mod errors;
 
 use meili_snap::{json_string, snapshot};
+use serde_json::json;
 
 use crate::common::{GetAllDocumentsOptions, Server};
-use crate::json;
 
 #[actix_rt::test]
 async fn swap_indexes() {

meilisearch/tests/tasks/mod.rs

@@ -1,11 +1,11 @@
 mod errors;
 
 use meili_snap::insta::assert_json_snapshot;
+use serde_json::json;
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn error_get_unexisting_task_status() {
@@ -33,7 +33,7 @@ async fn get_task_status() {
     index.create(None).await;
     index
         .add_documents(
-            json!([{
+            serde_json::json!([{
                 "id": 1,
                 "content": "foobar",
             }]),

milli/src/documents/enriched.rs

@@ -1,4 +1,5 @@
 use std::fs::File;
+use std::io::BufReader;
 use std::{io, str};
 
 use obkv::KvReader;
@@ -19,14 +20,14 @@ use crate::FieldId;
 pub struct EnrichedDocumentsBatchReader<R> {
     documents: DocumentsBatchReader<R>,
     primary_key: String,
-    external_ids: grenad::ReaderCursor<File>,
+    external_ids: grenad::ReaderCursor<BufReader<File>>,
 }
 
 impl<R: io::Read + io::Seek> EnrichedDocumentsBatchReader<R> {
     pub fn new(
         documents: DocumentsBatchReader<R>,
         primary_key: String,
-        external_ids: grenad::Reader<File>,
+        external_ids: grenad::Reader<BufReader<File>>,
     ) -> Result<Self, Error> {
         if documents.documents_count() as u64 == external_ids.len() {
             Ok(EnrichedDocumentsBatchReader {
@@ -75,7 +76,7 @@ pub struct EnrichedDocument<'a> {
 pub struct EnrichedDocumentsBatchCursor<R> {
     documents: DocumentsBatchCursor<R>,
     primary_key: String,
-    external_ids: grenad::ReaderCursor<File>,
+    external_ids: grenad::ReaderCursor<BufReader<File>>,
 }
 
 impl<R> EnrichedDocumentsBatchCursor<R> {

milli/src/search/new/bucket_sort.rs

@@ -46,18 +46,27 @@ pub fn bucket_sort<'ctx, Q: RankingRuleQueryTrait>(
         if let Some(distinct_fid) = distinct_fid {
             let mut excluded = RoaringBitmap::new();
             let mut results = vec![];
+            let mut skip = 0;
             for docid in universe.iter() {
-                if results.len() >= from + length {
+                if results.len() >= length {
                     break;
                 }
                 if excluded.contains(docid) {
                     continue;
                 }
+
                 distinct_single_docid(ctx.index, ctx.txn, distinct_fid, docid, &mut excluded)?;
+                skip += 1;
+                if skip <= from {
+                    continue;
+                }
+
                 results.push(docid);
             }
+
             let mut all_candidates = universe - excluded;
             all_candidates.extend(results.iter().copied());
+
             return Ok(BucketSortOutput {
                 scores: vec![Default::default(); results.len()],
                 docids: results,

milli/src/search/new/db_cache.rs

@@ -192,7 +192,6 @@ impl<'ctx> SearchContext<'ctx> {
     }
 
     pub fn word_prefix_docids(&mut self, prefix: Word) -> Result<Option<RoaringBitmap>> {
-        unreachable!();
         match prefix {
             Word::Original(prefix) => {
                 let exact = self.get_db_exact_word_prefix_docids(prefix)?;
@@ -217,7 +216,6 @@ impl<'ctx> SearchContext<'ctx> {
         &mut self,
         prefix: Interned<String>,
     ) -> Result<Option<RoaringBitmap>> {
-        unreachable!();
         match &self.restricted_fids {
             Some(restricted_fids) => {
                 let interned = self.word_interner.get(prefix).as_str();
@@ -246,7 +244,6 @@ impl<'ctx> SearchContext<'ctx> {
         &mut self,
         prefix: Interned<String>,
     ) -> Result<Option<RoaringBitmap>> {
-        unreachable!();
         DatabaseCache::get_value::<_, _, RoaringBitmapCodec>(
             self.txn,
             prefix,
@@ -300,7 +297,6 @@ impl<'ctx> SearchContext<'ctx> {
         prefix2: Interned<String>,
         proximity: u8,
     ) -> Result<Option<RoaringBitmap>> {
-        unreachable!();
         DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
             self.txn,
             (proximity, word1, prefix2),
@@ -319,7 +315,6 @@ impl<'ctx> SearchContext<'ctx> {
         right: Interned<String>,
         proximity: u8,
     ) -> Result<Option<RoaringBitmap>> {
-        unreachable!();
         DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
             self.txn,
             (proximity, left_prefix, right),
@@ -357,7 +352,6 @@ impl<'ctx> SearchContext<'ctx> {
         word_prefix: Interned<String>,
         fid: u16,
     ) -> Result<Option<RoaringBitmap>> {
-        unreachable!();
         // if the requested fid isn't in the restricted list, return None.
         if self.restricted_fids.as_ref().map_or(false, |fids| !fids.contains(&fid)) {
             return Ok(None);
@@ -399,7 +393,6 @@ impl<'ctx> SearchContext<'ctx> {
     }
 
     pub fn get_db_word_prefix_fids(&mut self, word_prefix: Interned<String>) -> Result<Vec<u16>> {
-        unreachable!();
         let fids = match self.db_cache.word_prefix_fids.entry(word_prefix) {
             Entry::Occupied(fids) => fids.get().clone(),
             Entry::Vacant(entry) => {
@@ -446,7 +439,6 @@ impl<'ctx> SearchContext<'ctx> {
         word_prefix: Interned<String>,
         position: u16,
     ) -> Result<Option<RoaringBitmap>> {
-        unreachable!();
         DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
             self.txn,
             (word_prefix, position),
@@ -488,7 +480,6 @@ impl<'ctx> SearchContext<'ctx> {
         &mut self,
         word_prefix: Interned<String>,
     ) -> Result<Vec<u16>> {
-        unreachable!();
         let positions = match self.db_cache.word_prefix_positions.entry(word_prefix) {
             Entry::Occupied(positions) => positions.get().clone(),
             Entry::Vacant(entry) => {

milli/src/update/facet/bulk.rs

@@ -1,5 +1,6 @@
 use std::borrow::Cow;
 use std::fs::File;
+use std::io::BufReader;
 
 use grenad::CompressionType;
 use heed::types::ByteSlice;
@@ -30,7 +31,7 @@ pub struct FacetsUpdateBulk<'i> {
     facet_type: FacetType,
     field_ids: Vec<FieldId>,
     // None if level 0 does not need to be updated
-    new_data: Option<grenad::Reader<File>>,
+    new_data: Option<grenad::Reader<BufReader<File>>>,
 }
 
 impl<'i> FacetsUpdateBulk<'i> {
@@ -38,7 +39,7 @@ impl<'i> FacetsUpdateBulk<'i> {
         index: &'i Index,
         field_ids: Vec<FieldId>,
         facet_type: FacetType,
-        new_data: grenad::Reader<File>,
+        new_data: grenad::Reader<BufReader<File>>,
         group_size: u8,
         min_level_size: u8,
     ) -> FacetsUpdateBulk<'i> {
@@ -187,7 +188,7 @@ impl<R: std::io::Read + std::io::Seek> FacetsUpdateBulkInner<R> {
         &self,
         field_id: FieldId,
         txn: &RoTxn,
-    ) -> Result<(Vec<grenad::Reader<File>>, RoaringBitmap)> {
+    ) -> Result<(Vec<grenad::Reader<BufReader<File>>>, RoaringBitmap)> {
         let mut all_docids = RoaringBitmap::new();
         let subwriters = self.compute_higher_levels(txn, field_id, 32, &mut |bitmaps, _| {
             for bitmap in bitmaps {
@@ -259,7 +260,7 @@ impl<R: std::io::Read + std::io::Seek> FacetsUpdateBulkInner<R> {
         field_id: u16,
         level: u8,
         handle_group: &mut dyn FnMut(&[RoaringBitmap], &'t [u8]) -> Result<()>,
-    ) -> Result<Vec<grenad::Reader<File>>> {
+    ) -> Result<Vec<grenad::Reader<BufReader<File>>>> {
         if level == 0 {
             self.read_level_0(rtxn, field_id, handle_group)?;
             // Level 0 is already in the database

milli/src/update/facet/incremental.rs

@@ -1,5 +1,6 @@
 use std::collections::HashMap;
 use std::fs::File;
+use std::io::BufReader;
 
 use heed::types::{ByteSlice, DecodeIgnore};
 use heed::{BytesDecode, Error, RoTxn, RwTxn};
@@ -34,14 +35,14 @@ pub struct FacetsUpdateIncremental<'i> {
     index: &'i Index,
     inner: FacetsUpdateIncrementalInner,
     facet_type: FacetType,
-    new_data: grenad::Reader<File>,
+    new_data: grenad::Reader<BufReader<File>>,
 }
 
 impl<'i> FacetsUpdateIncremental<'i> {
     pub fn new(
         index: &'i Index,
         facet_type: FacetType,
-        new_data: grenad::Reader<File>,
+        new_data: grenad::Reader<BufReader<File>>,
         group_size: u8,
         min_level_size: u8,
         max_group_size: u8,

milli/src/update/facet/mod.rs

@@ -78,6 +78,7 @@ pub const FACET_MIN_LEVEL_SIZE: u8 = 5;
 
 use std::collections::BTreeSet;
 use std::fs::File;
+use std::io::BufReader;
 use std::iter::FromIterator;
 
 use charabia::normalizer::{Normalize, NormalizerOption};
@@ -108,13 +109,17 @@ pub struct FacetsUpdate<'i> {
     index: &'i Index,
     database: heed::Database<FacetGroupKeyCodec<ByteSliceRefCodec>, FacetGroupValueCodec>,
     facet_type: FacetType,
-    new_data: grenad::Reader<File>,
+    new_data: grenad::Reader<BufReader<File>>,
     group_size: u8,
     max_group_size: u8,
     min_level_size: u8,
 }
 impl<'i> FacetsUpdate<'i> {
-    pub fn new(index: &'i Index, facet_type: FacetType, new_data: grenad::Reader<File>) -> Self {
+    pub fn new(
+        index: &'i Index,
+        facet_type: FacetType,
+        new_data: grenad::Reader<BufReader<File>>,
+    ) -> Self {
         let database = match facet_type {
             FacetType::String => index
                 .facet_id_string_docids

milli/src/update/index_documents/enrich.rs

@@ -1,4 +1,4 @@
-use std::io::{Read, Seek};
+use std::io::{BufWriter, Read, Seek};
 use std::result::Result as StdResult;
 use std::{fmt, iter};
 
@@ -35,7 +35,7 @@ pub fn enrich_documents_batch<R: Read + Seek>(
 
     let (mut cursor, mut documents_batch_index) = reader.into_cursor_and_fields_index();
 
-    let mut external_ids = tempfile::tempfile().map(grenad::Writer::new)?;
+    let mut external_ids = tempfile::tempfile().map(BufWriter::new).map(grenad::Writer::new)?;
     let mut uuid_buffer = [0; uuid::fmt::Hyphenated::LENGTH];
 
     // The primary key *field id* that has already been set for this index or the one

milli/src/update/index_documents/extract/extract_docid_word_positions.rs

@@ -1,6 +1,7 @@
 use std::collections::{HashMap, HashSet};
 use std::convert::TryInto;
 use std::fs::File;
+use std::io::BufReader;
 use std::{io, mem, str};
 
 use charabia::{Language, Script, SeparatorKind, Token, TokenKind, Tokenizer, TokenizerBuilder};
@@ -31,7 +32,7 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
     allowed_separators: Option<&[&str]>,
     dictionary: Option<&[&str]>,
     max_positions_per_attributes: Option<u32>,
-) -> Result<(RoaringBitmap, grenad::Reader<File>, ScriptLanguageDocidsMap)> {
+) -> Result<(RoaringBitmap, grenad::Reader<BufReader<File>>, ScriptLanguageDocidsMap)> {
     puffin::profile_function!();
 
     let max_positions_per_attributes = max_positions_per_attributes

milli/src/update/index_documents/extract/extract_facet_number_docids.rs

@@ -1,5 +1,5 @@
 use std::fs::File;
-use std::io;
+use std::io::{self, BufReader};
 
 use heed::{BytesDecode, BytesEncode};
 
@@ -19,7 +19,7 @@ use crate::Result;
 pub fn extract_facet_number_docids<R: io::Read + io::Seek>(
     docid_fid_facet_number: grenad::Reader<R>,
     indexer: GrenadParameters,
-) -> Result<grenad::Reader<File>> {
+) -> Result<grenad::Reader<BufReader<File>>> {
     puffin::profile_function!();
 
     let max_memory = indexer.max_memory_by_thread();

milli/src/update/index_documents/extract/extract_facet_string_docids.rs

@@ -1,5 +1,5 @@
 use std::fs::File;
-use std::io;
+use std::io::{self, BufReader};
 
 use heed::BytesEncode;
 
@@ -17,7 +17,7 @@ use crate::{FieldId, Result, MAX_FACET_VALUE_LENGTH};
 pub fn extract_facet_string_docids<R: io::Read + io::Seek>(
     docid_fid_facet_string: grenad::Reader<R>,
     indexer: GrenadParameters,
-) -> Result<grenad::Reader<File>> {
+) -> Result<grenad::Reader<BufReader<File>>> {
     puffin::profile_function!();
 
     let max_memory = indexer.max_memory_by_thread();

milli/src/update/index_documents/extract/extract_fid_docid_facet_values.rs

@@ -1,7 +1,7 @@
 use std::collections::{BTreeMap, HashSet};
 use std::convert::TryInto;
 use std::fs::File;
-use std::io;
+use std::io::{self, BufReader};
 use std::mem::size_of;
 
 use heed::zerocopy::AsBytes;
@@ -17,11 +17,11 @@ use crate::{CboRoaringBitmapCodec, DocumentId, FieldId, Result, BEU32, MAX_FACET
 
 /// The extracted facet values stored in grenad files by type.
 pub struct ExtractedFacetValues {
-    pub docid_fid_facet_numbers_chunk: grenad::Reader<File>,
-    pub docid_fid_facet_strings_chunk: grenad::Reader<File>,
-    pub fid_facet_is_null_docids_chunk: grenad::Reader<File>,
-    pub fid_facet_is_empty_docids_chunk: grenad::Reader<File>,
-    pub fid_facet_exists_docids_chunk: grenad::Reader<File>,
+    pub docid_fid_facet_numbers_chunk: grenad::Reader<BufReader<File>>,
+    pub docid_fid_facet_strings_chunk: grenad::Reader<BufReader<File>>,
+    pub fid_facet_is_null_docids_chunk: grenad::Reader<BufReader<File>>,
+    pub fid_facet_is_empty_docids_chunk: grenad::Reader<BufReader<File>>,
+    pub fid_facet_exists_docids_chunk: grenad::Reader<BufReader<File>>,
 }
 
 /// Extracts the facet values of each faceted field of each document.

milli/src/update/index_documents/extract/extract_fid_word_count_docids.rs

@@ -1,6 +1,6 @@
 use std::collections::HashMap;
 use std::fs::File;
-use std::io;
+use std::io::{self, BufReader};
 
 use grenad::Sorter;
 
@@ -21,7 +21,7 @@ use crate::{relative_from_absolute_position, DocumentId, FieldId, Result};
 pub fn extract_fid_word_count_docids<R: io::Read + io::Seek>(
     docid_word_positions: grenad::Reader<R>,
     indexer: GrenadParameters,
-) -> Result<grenad::Reader<File>> {
+) -> Result<grenad::Reader<BufReader<File>>> {
     puffin::profile_function!();
 
     let max_memory = indexer.max_memory_by_thread();

milli/src/update/index_documents/extract/extract_geo_points.rs

@@ -1,5 +1,5 @@
 use std::fs::File;
-use std::io;
+use std::io::{self, BufReader};
 
 use concat_arrays::concat_arrays;
 use serde_json::Value;
@@ -18,7 +18,7 @@ pub fn extract_geo_points<R: io::Read + io::Seek>(
     indexer: GrenadParameters,
     primary_key_id: FieldId,
     (lat_fid, lng_fid): (FieldId, FieldId),
-) -> Result<grenad::Reader<File>> {
+) -> Result<grenad::Reader<BufReader<File>>> {
     puffin::profile_function!();
 
     let mut writer = create_writer(

milli/src/update/index_documents/extract/extract_vector_points.rs

@@ -1,6 +1,6 @@
 use std::convert::TryFrom;
 use std::fs::File;
-use std::io;
+use std::io::{self, BufReader};
 
 use bytemuck::cast_slice;
 use serde_json::{from_slice, Value};
@@ -18,7 +18,7 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
     indexer: GrenadParameters,
     primary_key_id: FieldId,
     vectors_fid: FieldId,
-) -> Result<grenad::Reader<File>> {
+) -> Result<grenad::Reader<BufReader<File>>> {
     puffin::profile_function!();
 
     let mut writer = create_writer(

milli/src/update/index_documents/extract/extract_word_docids.rs

@@ -1,6 +1,6 @@
 use std::collections::HashSet;
 use std::fs::File;
-use std::io;
+use std::io::{self, BufReader};
 use std::iter::FromIterator;
 
 use roaring::RoaringBitmap;
@@ -26,7 +26,7 @@ pub fn extract_word_docids<R: io::Read + io::Seek>(
     docid_word_positions: grenad::Reader<R>,
     indexer: GrenadParameters,
     exact_attributes: &HashSet<FieldId>,
-) -> Result<(grenad::Reader<File>, grenad::Reader<File>)> {
+) -> Result<(grenad::Reader<BufReader<File>>, grenad::Reader<BufReader<File>>)> {
     puffin::profile_function!();
 
     let max_memory = indexer.max_memory_by_thread();

milli/src/update/index_documents/extract/extract_word_fid_docids.rs

@@ -1,5 +1,5 @@
 use std::fs::File;
-use std::io;
+use std::io::{self, BufReader};
 
 use super::helpers::{
     create_sorter, merge_cbo_roaring_bitmaps, read_u32_ne_bytes, sorter_into_reader,
@@ -14,7 +14,7 @@ use crate::{relative_from_absolute_position, DocumentId, Result};
 pub fn extract_word_fid_docids<R: io::Read + io::Seek>(
     docid_word_positions: grenad::Reader<R>,
     indexer: GrenadParameters,
-) -> Result<grenad::Reader<File>> {
+) -> Result<grenad::Reader<BufReader<File>>> {
     puffin::profile_function!();
 
     let max_memory = indexer.max_memory_by_thread();

milli/src/update/index_documents/extract/extract_word_pair_proximity_docids.rs

@@ -1,6 +1,7 @@
 use std::cmp::Ordering;
 use std::collections::{BinaryHeap, HashMap};
 use std::fs::File;
+use std::io::BufReader;
 use std::{cmp, io, mem, str, vec};
 
 use super::helpers::{
@@ -20,7 +21,7 @@ use crate::{DocumentId, Result};
 pub fn extract_word_pair_proximity_docids<R: io::Read + io::Seek>(
     docid_word_positions: grenad::Reader<R>,
     indexer: GrenadParameters,
-) -> Result<grenad::Reader<File>> {
+) -> Result<grenad::Reader<BufReader<File>>> {
     puffin::profile_function!();
 
     let max_memory = indexer.max_memory_by_thread();

milli/src/update/index_documents/extract/extract_word_position_docids.rs

@@ -1,5 +1,5 @@
 use std::fs::File;
-use std::io;
+use std::io::{self, BufReader};
 
 use super::helpers::{
     create_sorter, merge_cbo_roaring_bitmaps, read_u32_ne_bytes, sorter_into_reader,
@@ -17,7 +17,7 @@ use crate::{bucketed_position, relative_from_absolute_position, DocumentId, Resu
 pub fn extract_word_position_docids<R: io::Read + io::Seek>(
     docid_word_positions: grenad::Reader<R>,
     indexer: GrenadParameters,
-) -> Result<grenad::Reader<File>> {
+) -> Result<grenad::Reader<BufReader<File>>> {
     puffin::profile_function!();
 
     let max_memory = indexer.max_memory_by_thread();

milli/src/update/index_documents/extract/mod.rs

@@ -12,6 +12,7 @@ mod extract_word_position_docids;
 
 use std::collections::HashSet;
 use std::fs::File;
+use std::io::BufReader;
 
 use crossbeam_channel::Sender;
 use log::debug;
@@ -39,8 +40,8 @@ use crate::{FieldId, Result};
 /// Send data in grenad file over provided Sender.
 #[allow(clippy::too_many_arguments)]
 pub(crate) fn data_from_obkv_documents(
-    original_obkv_chunks: impl Iterator<Item = Result<grenad::Reader<File>>> + Send,
-    flattened_obkv_chunks: impl Iterator<Item = Result<grenad::Reader<File>>> + Send,
+    original_obkv_chunks: impl Iterator<Item = Result<grenad::Reader<BufReader<File>>>> + Send,
+    flattened_obkv_chunks: impl Iterator<Item = Result<grenad::Reader<BufReader<File>>>> + Send,
     indexer: GrenadParameters,
     lmdb_writer_sx: Sender<Result<TypedChunk>>,
     searchable_fields: Option<HashSet<FieldId>>,
@@ -152,7 +153,7 @@ pub(crate) fn data_from_obkv_documents(
         });
     }
 
-    spawn_extraction_task::<_, _, Vec<grenad::Reader<File>>>(
+    spawn_extraction_task::<_, _, Vec<grenad::Reader<BufReader<File>>>>(
         docid_word_positions_chunks.clone(),
         indexer,
         lmdb_writer_sx.clone(),
@@ -162,7 +163,7 @@ pub(crate) fn data_from_obkv_documents(
         "word-pair-proximity-docids",
     );
 
-    spawn_extraction_task::<_, _, Vec<grenad::Reader<File>>>(
+    spawn_extraction_task::<_, _, Vec<grenad::Reader<BufReader<File>>>>(
         docid_word_positions_chunks.clone(),
         indexer,
         lmdb_writer_sx.clone(),
@@ -172,7 +173,11 @@ pub(crate) fn data_from_obkv_documents(
         "field-id-wordcount-docids",
     );
 
-    spawn_extraction_task::<_, _, Vec<(grenad::Reader<File>, grenad::Reader<File>)>>(
+    spawn_extraction_task::<
+        _,
+        _,
+        Vec<(grenad::Reader<BufReader<File>>, grenad::Reader<BufReader<File>>)>,
+    >(
         docid_word_positions_chunks.clone(),
         indexer,
         lmdb_writer_sx.clone(),
@@ -185,7 +190,7 @@ pub(crate) fn data_from_obkv_documents(
         "word-docids",
     );
 
-    spawn_extraction_task::<_, _, Vec<grenad::Reader<File>>>(
+    spawn_extraction_task::<_, _, Vec<grenad::Reader<BufReader<File>>>>(
         docid_word_positions_chunks.clone(),
         indexer,
         lmdb_writer_sx.clone(),
@@ -194,7 +199,7 @@ pub(crate) fn data_from_obkv_documents(
         TypedChunk::WordPositionDocids,
         "word-position-docids",
     );
-    spawn_extraction_task::<_, _, Vec<grenad::Reader<File>>>(
+    spawn_extraction_task::<_, _, Vec<grenad::Reader<BufReader<File>>>>(
         docid_word_positions_chunks,
         indexer,
         lmdb_writer_sx.clone(),
@@ -204,7 +209,7 @@ pub(crate) fn data_from_obkv_documents(
         "word-fid-docids",
     );
 
-    spawn_extraction_task::<_, _, Vec<grenad::Reader<File>>>(
+    spawn_extraction_task::<_, _, Vec<grenad::Reader<BufReader<File>>>>(
         docid_fid_facet_strings_chunks,
         indexer,
         lmdb_writer_sx.clone(),
@@ -214,7 +219,7 @@ pub(crate) fn data_from_obkv_documents(
         "field-id-facet-string-docids",
     );
 
-    spawn_extraction_task::<_, _, Vec<grenad::Reader<File>>>(
+    spawn_extraction_task::<_, _, Vec<grenad::Reader<BufReader<File>>>>(
         docid_fid_facet_numbers_chunks,
         indexer,
         lmdb_writer_sx,
@@ -269,7 +274,7 @@ fn spawn_extraction_task<FE, FS, M>(
 /// Extract chunked data and send it into lmdb_writer_sx sender:
 /// - documents
 fn send_original_documents_data(
-    original_documents_chunk: Result<grenad::Reader<File>>,
+    original_documents_chunk: Result<grenad::Reader<BufReader<File>>>,
     indexer: GrenadParameters,
     lmdb_writer_sx: Sender<Result<TypedChunk>>,
     vectors_field_id: Option<FieldId>,
@@ -311,7 +316,7 @@ fn send_original_documents_data(
 #[allow(clippy::too_many_arguments)]
 #[allow(clippy::type_complexity)]
 fn send_and_extract_flattened_documents_data(
-    flattened_documents_chunk: Result<grenad::Reader<File>>,
+    flattened_documents_chunk: Result<grenad::Reader<BufReader<File>>>,
     indexer: GrenadParameters,
     lmdb_writer_sx: Sender<Result<TypedChunk>>,
     searchable_fields: &Option<HashSet<FieldId>>,
@@ -328,7 +333,10 @@ fn send_and_extract_flattened_documents_data(
         grenad::Reader<CursorClonableMmap>,
         (
             grenad::Reader<CursorClonableMmap>,
-            (grenad::Reader<File>, (grenad::Reader<File>, grenad::Reader<File>)),
+            (
+                grenad::Reader<BufReader<File>>,
+                (grenad::Reader<BufReader<File>>, grenad::Reader<BufReader<File>>),
+            ),
         ),
     ),
 )> {

milli/src/update/index_documents/helpers/grenad_helpers.rs

@@ -1,6 +1,6 @@
 use std::borrow::Cow;
 use std::fs::File;
-use std::io::{self, Seek};
+use std::io::{self, BufReader, BufWriter, Seek};
 use std::time::Instant;
 
 use grenad::{CompressionType, Sorter};
@@ -17,13 +17,13 @@ pub fn create_writer<R: io::Write>(
     typ: grenad::CompressionType,
     level: Option<u32>,
     file: R,
-) -> grenad::Writer<R> {
+) -> grenad::Writer<BufWriter<R>> {
     let mut builder = grenad::Writer::builder();
     builder.compression_type(typ);
     if let Some(level) = level {
         builder.compression_level(level);
     }
-    builder.build(file)
+    builder.build(BufWriter::new(file))
 }
 
 pub fn create_sorter(
@@ -53,7 +53,7 @@ pub fn create_sorter(
 pub fn sorter_into_reader(
     sorter: grenad::Sorter<MergeFn>,
     indexer: GrenadParameters,
-) -> Result<grenad::Reader<File>> {
+) -> Result<grenad::Reader<BufReader<File>>> {
     let mut writer = create_writer(
         indexer.chunk_compression_type,
         indexer.chunk_compression_level,
@@ -64,16 +64,18 @@ pub fn sorter_into_reader(
     writer_into_reader(writer)
 }
 
-pub fn writer_into_reader(writer: grenad::Writer<File>) -> Result<grenad::Reader<File>> {
-    let mut file = writer.into_inner()?;
+pub fn writer_into_reader(
+    writer: grenad::Writer<BufWriter<File>>,
+) -> Result<grenad::Reader<BufReader<File>>> {
+    let mut file = writer.into_inner()?.into_inner().map_err(|err| err.into_error())?;
     file.rewind()?;
-    grenad::Reader::new(file).map_err(Into::into)
+    grenad::Reader::new(BufReader::new(file)).map_err(Into::into)
 }
 
 pub unsafe fn as_cloneable_grenad(
-    reader: &grenad::Reader<File>,
+    reader: &grenad::Reader<BufReader<File>>,
 ) -> Result<grenad::Reader<CursorClonableMmap>> {
-    let file = reader.get_ref();
+    let file = reader.get_ref().get_ref();
     let mmap = memmap2::Mmap::map(file)?;
     let cursor = io::Cursor::new(ClonableMmap::from(mmap));
     let reader = grenad::Reader::new(cursor)?;
@@ -89,8 +91,8 @@ where
     fn merge(self, merge_fn: MergeFn, indexer: &GrenadParameters) -> Result<Self::Output>;
 }
 
-impl MergeableReader for Vec<grenad::Reader<File>> {
-    type Output = grenad::Reader<File>;
+impl MergeableReader for Vec<grenad::Reader<BufReader<File>>> {
+    type Output = grenad::Reader<BufReader<File>>;
 
     fn merge(self, merge_fn: MergeFn, params: &GrenadParameters) -> Result<Self::Output> {
         let mut merger = MergerBuilder::new(merge_fn);
@@ -99,8 +101,8 @@ impl MergeableReader for Vec<grenad::Reader<File>> {
     }
 }
 
-impl MergeableReader for Vec<(grenad::Reader<File>, grenad::Reader<File>)> {
-    type Output = (grenad::Reader<File>, grenad::Reader<File>);
+impl MergeableReader for Vec<(grenad::Reader<BufReader<File>>, grenad::Reader<BufReader<File>>)> {
+    type Output = (grenad::Reader<BufReader<File>>, grenad::Reader<BufReader<File>>);
 
     fn merge(self, merge_fn: MergeFn, params: &GrenadParameters) -> Result<Self::Output> {
         let mut m1 = MergerBuilder::new(merge_fn);
@@ -125,7 +127,7 @@ impl<R: io::Read + io::Seek> MergerBuilder<R> {
         Ok(())
     }
 
-    fn finish(self, params: &GrenadParameters) -> Result<grenad::Reader<File>> {
+    fn finish(self, params: &GrenadParameters) -> Result<grenad::Reader<BufReader<File>>> {
         let merger = self.0.build();
         let mut writer = create_writer(
             params.chunk_compression_type,
@@ -176,7 +178,7 @@ pub fn grenad_obkv_into_chunks<R: io::Read + io::Seek>(
     reader: grenad::Reader<R>,
     indexer: GrenadParameters,
     documents_chunk_size: usize,
-) -> Result<impl Iterator<Item = Result<grenad::Reader<File>>>> {
+) -> Result<impl Iterator<Item = Result<grenad::Reader<BufReader<File>>>>> {
     let mut continue_reading = true;
     let mut cursor = reader.into_cursor()?;
 

milli/src/update/index_documents/mod.rs

@@ -470,13 +470,13 @@ where
         let all_documents_ids = index_documents_ids | new_documents_ids;
         self.index.put_documents_ids(self.wtxn, &all_documents_ids)?;
 
-        // self.execute_prefix_databases(
-        //     word_docids,
-        //     exact_word_docids,
-        //     word_pair_proximity_docids,
-        //     word_position_docids,
-        //     word_fid_docids,
-        // )?;
+        self.execute_prefix_databases(
+            word_docids,
+            exact_word_docids,
+            word_pair_proximity_docids,
+            word_position_docids,
+            word_fid_docids,
+        )?;
 
         Ok(all_documents_ids.len())
     }

milli/src/update/index_documents/transform.rs

@@ -659,8 +659,10 @@ impl<'a, 'i> Transform<'a, 'i> {
             new_documents_ids: self.new_documents_ids,
             replaced_documents_ids: self.replaced_documents_ids,
             documents_count: self.documents_count,
-            original_documents,
-            flattened_documents,
+            original_documents: original_documents.into_inner().map_err(|err| err.into_error())?,
+            flattened_documents: flattened_documents
+                .into_inner()
+                .map_err(|err| err.into_error())?,
         })
     }
 
@@ -779,8 +781,10 @@ impl<'a, 'i> Transform<'a, 'i> {
             new_documents_ids: documents_ids,
             replaced_documents_ids: RoaringBitmap::default(),
             documents_count,
-            original_documents,
-            flattened_documents,
+            original_documents: original_documents.into_inner().map_err(|err| err.into_error())?,
+            flattened_documents: flattened_documents
+                .into_inner()
+                .map_err(|err| err.into_error())?,
         };
 
         let new_facets = output.compute_real_facets(wtxn, self.index)?;

milli/src/update/index_documents/typed_chunk.rs

@@ -2,7 +2,7 @@ use std::borrow::Cow;
 use std::collections::HashMap;
 use std::convert::TryInto;
 use std::fs::File;
-use std::io;
+use std::io::{self, BufReader};
 
 use bytemuck::allocation::pod_collect_to_vec;
 use charabia::{Language, Script};
@@ -27,22 +27,22 @@ pub(crate) enum TypedChunk {
     FieldIdDocidFacetStrings(grenad::Reader<CursorClonableMmap>),
     FieldIdDocidFacetNumbers(grenad::Reader<CursorClonableMmap>),
     Documents(grenad::Reader<CursorClonableMmap>),
-    FieldIdWordcountDocids(grenad::Reader<File>),
+    FieldIdWordcountDocids(grenad::Reader<BufReader<File>>),
     NewDocumentsIds(RoaringBitmap),
     WordDocids {
-        word_docids_reader: grenad::Reader<File>,
-        exact_word_docids_reader: grenad::Reader<File>,
+        word_docids_reader: grenad::Reader<BufReader<File>>,
+        exact_word_docids_reader: grenad::Reader<BufReader<File>>,
     },
-    WordPositionDocids(grenad::Reader<File>),
-    WordFidDocids(grenad::Reader<File>),
-    WordPairProximityDocids(grenad::Reader<File>),
-    FieldIdFacetStringDocids(grenad::Reader<File>),
-    FieldIdFacetNumberDocids(grenad::Reader<File>),
-    FieldIdFacetExistsDocids(grenad::Reader<File>),
-    FieldIdFacetIsNullDocids(grenad::Reader<File>),
-    FieldIdFacetIsEmptyDocids(grenad::Reader<File>),
-    GeoPoints(grenad::Reader<File>),
-    VectorPoints(grenad::Reader<File>),
+    WordPositionDocids(grenad::Reader<BufReader<File>>),
+    WordFidDocids(grenad::Reader<BufReader<File>>),
+    WordPairProximityDocids(grenad::Reader<BufReader<File>>),
+    FieldIdFacetStringDocids(grenad::Reader<BufReader<File>>),
+    FieldIdFacetNumberDocids(grenad::Reader<BufReader<File>>),
+    FieldIdFacetExistsDocids(grenad::Reader<BufReader<File>>),
+    FieldIdFacetIsNullDocids(grenad::Reader<BufReader<File>>),
+    FieldIdFacetIsEmptyDocids(grenad::Reader<BufReader<File>>),
+    GeoPoints(grenad::Reader<BufReader<File>>),
+    VectorPoints(grenad::Reader<BufReader<File>>),
     ScriptLanguageDocids(HashMap<(Script, Language), RoaringBitmap>),
 }
 

milli/src/update/prefix_word_pairs/mod.rs

@@ -1,6 +1,6 @@
 use std::borrow::Cow;
 use std::collections::HashSet;
-use std::io::BufReader;
+use std::io::{BufReader, BufWriter};
 
 use grenad::CompressionType;
 use heed::types::ByteSlice;
@@ -119,9 +119,9 @@ pub fn insert_into_database(
 pub fn write_into_lmdb_database_without_merging(
     wtxn: &mut heed::RwTxn,
     database: heed::PolyDatabase,
-    writer: grenad::Writer<std::fs::File>,
+    writer: grenad::Writer<BufWriter<std::fs::File>>,
 ) -> Result<()> {
-    let file = writer.into_inner()?;
+    let file = writer.into_inner()?.into_inner().map_err(|err| err.into_error())?;
     let reader = grenad::Reader::new(BufReader::new(file))?;
     if database.is_empty(wtxn)? {
         let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?;

milli/tests/search/distinct.rs

@@ -8,7 +8,7 @@ use Criterion::*;
 use crate::search::{self, EXTERNAL_DOCUMENTS_IDS};
 
 macro_rules! test_distinct {
-    ($func:ident, $distinct:ident, $exhaustive:ident, $limit:expr, $criteria:expr, $n_res:expr) => {
+    ($func:ident, $distinct:ident, $exhaustive:ident, $limit:expr, $offset:expr, $criteria:expr, $n_res:expr) => {
         #[test]
         fn $func() {
             let criteria = $criteria;
@@ -27,6 +27,7 @@ macro_rules! test_distinct {
             let mut search = Search::new(&rtxn, &index);
             search.query(search::TEST_QUERY);
             search.limit($limit);
+            search.offset($offset);
             search.exhaustive_number_hits($exhaustive);
 
             search.terms_matching_strategy(TermsMatchingStrategy::default());
@@ -47,6 +48,7 @@ macro_rules! test_distinct {
                             Some(d.id)
                         }
                     })
+                    .skip($offset)
                     .take($limit)
                     .collect();
 
@@ -61,6 +63,7 @@ test_distinct!(
     tag,
     true,
     1,
+    0,
     vec![Words, Typo, Proximity, Attribute, Exactness],
     3
 );
@@ -69,6 +72,7 @@ test_distinct!(
     asc_desc_rank,
     true,
     1,
+    0,
     vec![Words, Typo, Proximity, Attribute, Exactness],
     7
 );
@@ -77,6 +81,7 @@ test_distinct!(
     asc_desc_rank,
     true,
     0,
+    0,
     vec![Desc(S("attribute_rank")), Desc(S("exactness_rank")), Exactness, Typo],
     7
 );
@@ -86,6 +91,7 @@ test_distinct!(
     tag,
     false,
     EXTERNAL_DOCUMENTS_IDS.len(),
+    0,
     vec![Words, Typo, Proximity, Attribute, Exactness],
     3
 );
@@ -94,6 +100,7 @@ test_distinct!(
     asc_desc_rank,
     false,
     EXTERNAL_DOCUMENTS_IDS.len(),
+    0,
     vec![Words, Typo, Proximity, Attribute, Exactness],
     7
 );
@@ -102,6 +109,7 @@ test_distinct!(
     tag,
     false,
     EXTERNAL_DOCUMENTS_IDS.len(),
+    0,
     vec![Words],
     3
 );
@@ -110,6 +118,7 @@ test_distinct!(
     asc_desc_rank,
     false,
     EXTERNAL_DOCUMENTS_IDS.len(),
+    0,
     vec![Words],
     7
 );
@@ -118,6 +127,7 @@ test_distinct!(
     tag,
     false,
     EXTERNAL_DOCUMENTS_IDS.len(),
+    0,
     vec![Words, Typo],
     3
 );
@@ -126,6 +136,7 @@ test_distinct!(
     asc_desc_rank,
     false,
     EXTERNAL_DOCUMENTS_IDS.len(),
+    0,
     vec![Words, Typo],
     7
 );
@@ -134,6 +145,7 @@ test_distinct!(
     tag,
     false,
     EXTERNAL_DOCUMENTS_IDS.len(),
+    0,
     vec![Words, Proximity],
     3
 );
@@ -142,6 +154,7 @@ test_distinct!(
     asc_desc_rank,
     false,
     EXTERNAL_DOCUMENTS_IDS.len(),
+    0,
     vec![Words, Proximity],
     7
 );
@@ -150,6 +163,7 @@ test_distinct!(
     tag,
     false,
     EXTERNAL_DOCUMENTS_IDS.len(),
+    0,
     vec![Words, Attribute],
     3
 );
@@ -158,6 +172,7 @@ test_distinct!(
     asc_desc_rank,
     false,
     EXTERNAL_DOCUMENTS_IDS.len(),
+    0,
     vec![Words, Attribute],
     7
 );
@@ -166,6 +181,7 @@ test_distinct!(
     tag,
     false,
     EXTERNAL_DOCUMENTS_IDS.len(),
+    0,
     vec![Words, Exactness],
     3
 );
@@ -174,6 +190,47 @@ test_distinct!(
     asc_desc_rank,
     false,
     EXTERNAL_DOCUMENTS_IDS.len(),
+    0,
     vec![Words, Exactness],
     7
 );
+test_distinct!(
+    // testing: https://github.com/meilisearch/meilisearch/issues/4078
+    distinct_string_limit_and_offset,
+    tag,
+    false,
+    EXTERNAL_DOCUMENTS_IDS.len(),
+    1,
+    vec![],
+    2
+);
+test_distinct!(
+    // testing: https://github.com/meilisearch/meilisearch/issues/4078
+    exhaustive_distinct_string_limit_and_offset,
+    tag,
+    true,
+    1,
+    2,
+    vec![],
+    1
+);
+test_distinct!(
+    // testing: https://github.com/meilisearch/meilisearch/issues/4078
+    distinct_number_limit_and_offset,
+    asc_desc_rank,
+    false,
+    EXTERNAL_DOCUMENTS_IDS.len(),
+    2,
+    vec![],
+    5
+);
+test_distinct!(
+    // testing: https://github.com/meilisearch/meilisearch/issues/4078
+    exhaustive_distinct_number_limit_and_offset,
+    asc_desc_rank,
+    true,
+    2,
+    4,
+    vec![],
+    3
+);