make the task id u64

.github/workflows/benchmarks-manual.yml

@@ -74,4 +74,4 @@ jobs:
           echo "${{ steps.file.outputs.basename }}.json has just been pushed."
           echo 'How to compare this benchmark with another one?'
           echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
-          echo "  - Run the following command: ./benchmaks/scripts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"
+          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/dependency-issue.yml

@@ -2,8 +2,8 @@ name: Create issue to upgrade dependencies
 
 on:
   schedule:
-    # Run the first of the month, every 6 month
-    - cron: '0 0 1 */6 *'
+    # Run the first of the month, every 3 month
+    - cron: '0 0 1 */3 *'
   workflow_dispatch:
 
 jobs:

.github/workflows/publish-apt-brew-pkg.yml

@@ -53,6 +53,5 @@ jobs:
         uses: mislav/bump-homebrew-formula-action@v2
         with:
           formula-name: meilisearch
-          formula-path: Formula/m/meilisearch.rb
         env:
           COMMITTER_TOKEN: ${{ secrets.HOMEBREW_COMMITTER_TOKEN }}

.github/workflows/publish-docker-images.yml

@@ -57,10 +57,10 @@ jobs:
           echo "date=$commit_date" >> $GITHUB_OUTPUT
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to Docker Hub
         uses: docker/login-action@v2
@@ -70,7 +70,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v4
         with:
           images: getmeili/meilisearch
           # Prevent `latest` to be updated for each new tag pushed.
@@ -83,7 +83,7 @@ jobs:
             type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable == 'true' && steps.check-tag-format.outputs.latest == 'true' }}
 
       - name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v4
         with:
           push: true
           platforms: linux/amd64,linux/arm64

.github/workflows/sdks-tests.yml

@@ -14,7 +14,6 @@ on:
 env:
   MEILI_MASTER_KEY: 'masterKey'
   MEILI_NO_ANALYTICS: 'true'
-  DISABLE_COVERAGE: 'true'
 
 jobs:
   define-docker-image:
@@ -31,117 +30,6 @@ jobs:
           if [[ $event == 'workflow_dispatch' ]]; then
             echo "docker-image=${{ github.event.inputs.docker_image }}" >> $GITHUB_OUTPUT
           fi
-      - name: Docker image is ${{ steps.define-image.outputs.docker-image }}
-        run: echo "Docker image is ${{ steps.define-image.outputs.docker-image }}"
-
-##########
-## SDKs ##
-##########
-
-  meilisearch-dotnet-tests:
-    needs: define-docker-image
-    name: .NET SDK tests
-    runs-on: ubuntu-latest
-    env:
-      MEILISEARCH_VERSION: ${{ needs.define-docker-image.outputs.docker-image }}
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-dotnet
-      - name: Setup .NET Core
-        uses: actions/setup-dotnet@v3
-        with:
-          dotnet-version: "6.0.x"
-      - name: Install dependencies
-        run: dotnet restore
-      - name: Build
-        run: dotnet build --configuration Release --no-restore
-      - name: Meilisearch (latest version) setup with Docker
-        run: docker compose up -d
-      - name: Run tests
-        run: dotnet test --no-restore --verbosity normal
-
-  meilisearch-dart-tests:
-    needs: define-docker-image
-    name: Dart SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-dart
-      - uses: dart-lang/setup-dart@v1
-        with:
-          sdk: 3.1.1
-      - name: Install dependencies
-        run: dart pub get
-      - name: Run integration tests
-        run: dart test --concurrency=4
-
-  meilisearch-go-tests:
-    needs: define-docker-image
-    name: Go SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: stable
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-go
-      - name: Get dependencies
-        run: |
-          go get -v -t -d ./...
-          if [ -f Gopkg.toml ]; then
-            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-            dep ensure
-          fi
-      - name: Run integration tests
-        run: go test -v ./...
-
-  meilisearch-java-tests:
-    needs: define-docker-image
-    name: Java SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-java
-      - name: Set up Java
-        uses: actions/setup-java@v3
-        with:
-          java-version: 8
-          distribution: 'zulu'
-          cache: gradle
-      - name: Grant execute permission for gradlew
-        run: chmod +x gradlew
-      - name: Build and run unit and integration tests
-        run: ./gradlew build integrationTest
 
   meilisearch-js-tests:
     needs: define-docker-image
@@ -178,6 +66,33 @@ jobs:
       - name: Run Browser env
         run: yarn test:env:browser
 
+  instant-meilisearch-tests:
+    needs: define-docker-image
+    name: instant-meilisearch tests
+    runs-on: ubuntu-latest
+    services:
+      meilisearch:
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        env:
+          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
+          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
+        ports:
+          - '7700:7700'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: meilisearch/instant-meilisearch
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          cache: yarn
+      - name: Install dependencies
+        run: yarn install
+      - name: Run tests
+        run: yarn test
+      - name: Build all the playgrounds and the packages
+        run: yarn build
+
   meilisearch-php-tests:
     needs: define-docker-image
     name: PHP SDK tests
@@ -196,6 +111,8 @@ jobs:
           repository: meilisearch/meilisearch-php
       - name: Install PHP
         uses: shivammathur/setup-php@v2
+        with:
+          coverage: none
       - name: Validate composer.json and composer.lock
         run: composer validate
       - name: Install dependencies
@@ -232,6 +149,36 @@ jobs:
       - name: Test with pytest
         run: pipenv run pytest
 
+  meilisearch-go-tests:
+    needs: define-docker-image
+    name: Go SDK tests
+    runs-on: ubuntu-latest
+    services:
+      meilisearch:
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        env:
+          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
+          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
+        ports:
+          - '7700:7700'
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: stable
+      - uses: actions/checkout@v3
+        with:
+          repository: meilisearch/meilisearch-go
+      - name: Get dependencies
+        run: |
+          go get -v -t -d ./...
+          if [ -f Gopkg.toml ]; then
+            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+            dep ensure
+          fi
+      - name: Run integration tests
+        run: go test -v ./...
+
   meilisearch-ruby-tests:
     needs: define-docker-image
     name: Ruby SDK tests
@@ -277,110 +224,3 @@ jobs:
         run: cargo build --verbose
       - name: Run tests
         run: cargo test --verbose
-
-  meilisearch-swift-tests:
-    needs: define-docker-image
-    name: Swift SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-swift
-      - name: Run tests
-        run: swift test
-
-########################
-## FRONT-END PLUGINS ##
-########################
-
-  meilisearch-js-plugins-tests:
-    needs: define-docker-image
-    name: meilisearch-js-plugins tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-js-plugins
-      - name: Setup node
-        uses: actions/setup-node@v3
-        with:
-          cache: yarn
-      - name: Install dependencies
-        run: yarn install
-      - name: Run tests
-        run: yarn test
-      - name: Build all the playgrounds and the packages
-        run: yarn build
-
-########################
-## BACK-END PLUGINS ###
-########################
-
-  meilisearch-rails-tests:
-    needs: define-docker-image
-    name: meilisearch-rails tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-rails
-      - name: Set up Ruby 3
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 3
-          bundler-cache: true
-      - name: Run tests
-        run: bundle exec rspec
-
-  meilisearch-symfony-tests:
-    needs: define-docker-image
-    name: meilisearch-symfony tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-symfony
-      - name: Install PHP
-        uses: shivammathur/setup-php@v2
-        with:
-          tools: composer:v2, flex
-      - name: Validate composer.json and composer.lock
-        run: composer validate
-      - name: Install dependencies
-        run: composer install --prefer-dist --no-progress --quiet
-      - name: Remove doctrine/annotations
-        run: composer remove --dev doctrine/annotations
-      - name: Run test suite
-        run: composer test:unit

.github/workflows/test-suite.yml

@@ -123,10 +123,7 @@ jobs:
           override: true
       - name: Run cargo tree without default features and check lindera is not present
         run: |
-          if cargo tree -f '{p} {f}' -e normal --no-default-features | grep -vqz lindera; then
-            echo "lindera has been found in the sources and it shouldn't"
-            exit 1
-          fi
+          cargo tree -f '{p} {f}' -e normal --no-default-features | grep lindera -vqz
       - name: Run cargo tree with default features and check lindera is pressent
         run: |
           cargo tree -f '{p} {f}' -e normal | grep lindera -qz

.github/workflows/trigger-benchmarks-on-message.yml

@@ -1,84 +0,0 @@
-name: Benchmarks (PR)
-on: issue_comment
-permissions:
-  issues: write
-
-env:
-  GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
-
-jobs:
-  run-benchmarks-on-comment:
-    name: Run and upload benchmarks
-    runs-on: benchmarks
-    timeout-minutes: 4320 # 72h
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      - name: Check for Command
-        id: command
-        uses: xt0rted/slash-command-action@v2
-        with:
-          command: benchmark
-          reaction-type: "eyes"
-          repo-token: ${{ env.GH_TOKEN }}
-
-      # Set variables
-      - name: Set current branch name
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
-        id: current_branch
-      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" >> $GITHUB_OUTPUT
-        id: normalized_current_branch
-      - name: Set shorter commit SHA
-        shell: bash
-        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
-        id: commit_sha
-      - name: Set file basename with format "dataset_branch_commitSHA"
-        shell: bash
-        run: echo "basename=$(echo ${{ steps.command.outputs.command-arguments }}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
-        id: file
-
-      # Run benchmarks
-      - name: Run benchmarks - Dataset ${{ steps.command.outputs.command-arguments }} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
-        run: |
-          cd benchmarks
-          cargo bench --bench ${{ steps.command.outputs.command-arguments }} -- --save-baseline ${{ steps.file.outputs.basename }}
-
-      # Generate critcmp files
-      - name: Install critcmp
-        uses: taiki-e/install-action@v2
-        with:
-          tool: critcmp
-      - name: Export cripcmp file
-        run: |
-          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
-
-      # Upload benchmarks
-      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
-        uses: BetaHuhn/do-spaces-action@v2
-        with:
-          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
-          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
-          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
-          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
-          source: ${{ steps.file.outputs.basename }}.json
-          out_dir: critcmp_results
-
-      # Compute the diff of the benchmarks and send a message on the GitHub PR
-      - name: Compute and send a message in the PR
-        env:
-          GITHUB_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
-        run: |
-          export base=$(git log --pretty=%p -n 1)
-          echo 'Here are your benchmarks diff 👊' >> body.txt
-          echo '```' >> body.txt
-          ./benchmarks/scripts/compare.sh $base ${{ steps.file.outputs.basename }}.json >> body.txt
-          echo '```' >> body.txt
-          gh pr comment ${GITHUB_REF#refs/heads/} --body-file body.txt

Cargo.lock

@@ -1444,7 +1444,6 @@ dependencies = [
  "insta",
  "nom",
  "nom_locate",
- "unescaper",
 ]
 
 [[package]]
@@ -2704,7 +2703,6 @@ dependencies = [
  "logging_timer",
  "maplit",
  "md5",
- "meili-snap",
  "memmap2",
  "mimalloc",
  "obkv",
@@ -3539,9 +3537,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-webpki"
-version = "0.100.2"
+version = "0.100.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e98ff011474fa39949b7e5c0428f9b4937eda7da7848bbb947786b7be0b27dab"
+checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b"
 dependencies = [
  "ring",
  "untrusted",
@@ -4182,15 +4180,6 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9"
 
-[[package]]
-name = "unescaper"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a96a44ae11e25afb520af4534fd7b0bd8cd613e35a78def813b8cf41631fa3c8"
-dependencies = [
- "thiserror",
-]
-
 [[package]]
 name = "unicase"
 version = "2.6.0"
@@ -4249,7 +4238,7 @@ dependencies = [
  "log",
  "once_cell",
  "rustls 0.21.6",
- "rustls-webpki 0.100.2",
+ "rustls-webpki 0.100.1",
  "url",
  "webpki-roots 0.23.1",
 ]
@@ -4444,9 +4433,9 @@ dependencies = [
 
 [[package]]
 name = "webpki"
-version = "0.22.1"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0e74f82d49d545ad128049b7e88f6576df2da6b02e9ce565c6f533be576957e"
+checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd"
 dependencies = [
  "ring",
  "untrusted",
@@ -4467,7 +4456,7 @@ version = "0.23.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338"
 dependencies = [
- "rustls-webpki 0.100.2",
+ "rustls-webpki 0.100.1",
 ]
 
 [[package]]

dump/src/lib.rs

@@ -7,7 +7,7 @@ use meilisearch_types::milli::update::IndexDocumentsMethod;
 use meilisearch_types::settings::Unchecked;
 use meilisearch_types::tasks::{Details, IndexSwap, KindWithContent, Status, Task, TaskId};
 use meilisearch_types::InstanceUid;
-use roaring::RoaringBitmap;
+use roaring::RoaringTreemap;
 use serde::{Deserialize, Serialize};
 use time::OffsetDateTime;
 
@@ -121,11 +121,11 @@ pub enum KindDump {
     },
     TaskCancelation {
         query: String,
-        tasks: RoaringBitmap,
+        tasks: RoaringTreemap,
     },
     TasksDeletion {
         query: String,
-        tasks: RoaringBitmap,
+        tasks: RoaringTreemap,
     },
     DumpCreation {
         keys: Vec<Key>,

dump/src/reader/compat/v5_to_v6.rs

@@ -69,7 +69,7 @@ impl CompatV5ToV6 {
                 }
 
                 let task = v6::Task {
-                    uid: task_view.uid,
+                    uid: task_view.uid as u64,
                     index_uid: task_view.index_uid,
                     status: match task_view.status {
                         v5::Status::Enqueued => v6::Status::Enqueued,

filter-parser/Cargo.toml

@@ -14,7 +14,6 @@ license.workspace = true
 [dependencies]
 nom = "7.1.3"
 nom_locate = "4.1.0"
-unescaper = "0.1.2"
 
 [dev-dependencies]
 insta = "1.29.0"

filter-parser/src/error.rs

@@ -62,7 +62,6 @@ pub enum ErrorKind<'a> {
     MisusedGeoRadius,
     MisusedGeoBoundingBox,
     InvalidPrimary,
-    InvalidEscapedNumber,
     ExpectedEof,
     ExpectedValue(ExpectedValueKind),
     MalformedValue,
@@ -148,9 +147,6 @@ impl<'a> Display for Error<'a> {
                 let text = if input.trim().is_empty() { "but instead got nothing.".to_string() } else { format!("at `{}`.", escaped_input) };
                 writeln!(f, "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` {}", text)?
             }
-            ErrorKind::InvalidEscapedNumber => {
-                writeln!(f, "Found an invalid escaped sequence number: `{}`.", escaped_input)?
-            }
             ErrorKind::ExpectedEof => {
                 writeln!(f, "Found unexpected characters at the end of the filter: `{}`. You probably forgot an `OR` or an `AND` rule.", escaped_input)?
             }

filter-parser/src/lib.rs

@@ -545,8 +545,6 @@ impl<'a> std::fmt::Display for Token<'a> {
 
 #[cfg(test)]
 pub mod tests {
-    use FilterCondition as Fc;
-
     use super::*;
 
     /// Create a raw [Token]. You must specify the string that appear BEFORE your element followed by your element
@@ -558,22 +556,14 @@ pub mod tests {
         unsafe { Span::new_from_raw_offset(offset, lines as u32, value, "") }.into()
     }
 
-    fn p(s: &str) -> impl std::fmt::Display + '_ {
-        Fc::parse(s).unwrap().unwrap()
-    }
-
-    #[test]
-    fn parse_escaped() {
-        insta::assert_display_snapshot!(p(r#"title = 'foo\\'"#), @r#"{title} = {foo\}"#);
-        insta::assert_display_snapshot!(p(r#"title = 'foo\\\\'"#), @r#"{title} = {foo\\}"#);
-        insta::assert_display_snapshot!(p(r#"title = 'foo\\\\\\'"#), @r#"{title} = {foo\\\}"#);
-        insta::assert_display_snapshot!(p(r#"title = 'foo\\\\\\\\'"#), @r#"{title} = {foo\\\\}"#);
-        // but it also works with other sequencies
-        insta::assert_display_snapshot!(p(r#"title = 'foo\x20\n\t\"\'"'"#), @"{title} = {foo \n\t\"\'\"}");
-    }
-
     #[test]
     fn parse() {
+        use FilterCondition as Fc;
+
+        fn p(s: &str) -> impl std::fmt::Display + '_ {
+            Fc::parse(s).unwrap().unwrap()
+        }
+
         // Test equal
         insta::assert_display_snapshot!(p("channel = Ponce"), @"{channel} = {Ponce}");
         insta::assert_display_snapshot!(p("subscribers = 12"), @"{subscribers} = {12}");

filter-parser/src/value.rs

@@ -171,24 +171,7 @@ pub fn parse_value(input: Span) -> IResult<Token> {
         })
     })?;
 
-    match unescaper::unescape(value.value()) {
-        Ok(content) => {
-            if content.len() != value.value().len() {
-                Ok((input, Token::new(value.original_span(), Some(content))))
-            } else {
-                Ok((input, value))
-            }
-        }
-        Err(unescaper::Error::IncompleteStr(_)) => Err(nom::Err::Incomplete(nom::Needed::Unknown)),
-        Err(unescaper::Error::ParseIntError { .. }) => Err(nom::Err::Error(Error::new_from_kind(
-            value.original_span(),
-            ErrorKind::InvalidEscapedNumber,
-        ))),
-        Err(unescaper::Error::InvalidChar { .. }) => Err(nom::Err::Error(Error::new_from_kind(
-            value.original_span(),
-            ErrorKind::MalformedValue,
-        ))),
-    }
+    Ok((input, value))
 }
 
 fn is_value_component(c: char) -> bool {
@@ -335,17 +318,17 @@ pub mod test {
             ("\"cha'nnel\"", "cha'nnel", false),
             ("I'm tamo", "I", false),
             // escaped thing but not quote
-            (r#""\\""#, r#"\"#, true),
-            (r#""\\\\\\""#, r#"\\\"#, true),
-            (r#""aa\\aa""#, r#"aa\aa"#, true),
+            (r#""\\""#, r#"\\"#, false),
+            (r#""\\\\\\""#, r#"\\\\\\"#, false),
+            (r#""aa\\aa""#, r#"aa\\aa"#, false),
             // with double quote
             (r#""Hello \"world\"""#, r#"Hello "world""#, true),
-            (r#""Hello \\\"world\\\"""#, r#"Hello \"world\""#, true),
+            (r#""Hello \\\"world\\\"""#, r#"Hello \\"world\\""#, true),
             (r#""I'm \"super\" tamo""#, r#"I'm "super" tamo"#, true),
             (r#""\"\"""#, r#""""#, true),
             // with simple quote
             (r#"'Hello \'world\''"#, r#"Hello 'world'"#, true),
-            (r#"'Hello \\\'world\\\''"#, r#"Hello \'world\'"#, true),
+            (r#"'Hello \\\'world\\\''"#, r#"Hello \\'world\\'"#, true),
             (r#"'I\'m "super" tamo'"#, r#"I'm "super" tamo"#, true),
             (r#"'\'\''"#, r#"''"#, true),
         ];
@@ -367,14 +350,7 @@ pub mod test {
                 "Filter `{}` was not supposed to be escaped",
                 input
             );
-            assert_eq!(
-                token.value(),
-                expected,
-                "Filter `{}` failed by giving `{}` instead of `{}`.",
-                input,
-                token.value(),
-                expected
-            );
+            assert_eq!(token.value(), expected, "Filter `{}` failed.", input);
         }
     }
 

index-scheduler/src/batch.rs

@@ -32,11 +32,11 @@ use meilisearch_types::milli::update::{
     DeleteDocuments, DocumentDeletionResult, IndexDocumentsConfig, IndexDocumentsMethod,
     Settings as MilliSettings,
 };
-use meilisearch_types::milli::{self, Filter, BEU32};
+use meilisearch_types::milli::{self, Filter, BEU64};
 use meilisearch_types::settings::{apply_settings_to_builder, Settings, Unchecked};
 use meilisearch_types::tasks::{Details, IndexSwap, Kind, KindWithContent, Status, Task};
 use meilisearch_types::{compression, Index, VERSION_FILE_NAME};
-use roaring::RoaringBitmap;
+use roaring::RoaringTreemap;
 use time::macros::format_description;
 use time::OffsetDateTime;
 use uuid::Uuid;
@@ -58,7 +58,7 @@ pub(crate) enum Batch {
         /// The date and time at which the previously processing tasks started.
         previous_started_at: OffsetDateTime,
         /// The list of tasks that were processing when this task cancelation appeared.
-        previous_processing_tasks: RoaringBitmap,
+        previous_processing_tasks: RoaringTreemap,
     },
     TaskDeletion(Task),
     SnapshotCreation(Vec<Task>),
@@ -67,6 +67,10 @@ pub(crate) enum Batch {
         op: IndexOperation,
         must_create_index: bool,
     },
+    IndexDocumentDeletionByFilter {
+        index_uid: String,
+        task: Task,
+    },
     IndexCreation {
         index_uid: String,
         primary_key: Option<String>,
@@ -110,10 +114,6 @@ pub(crate) enum IndexOperation {
         documents: Vec<Vec<String>>,
         tasks: Vec<Task>,
     },
-    IndexDocumentDeletionByFilter {
-        index_uid: String,
-        task: Task,
-    },
     DocumentClear {
         index_uid: String,
         tasks: Vec<Task>,
@@ -155,6 +155,7 @@ impl Batch {
             | Batch::TaskDeletion(task)
             | Batch::Dump(task)
             | Batch::IndexCreation { task, .. }
+            | Batch::IndexDocumentDeletionByFilter { task, .. }
             | Batch::IndexUpdate { task, .. } => vec![task.uid],
             Batch::SnapshotCreation(tasks) | Batch::IndexDeletion { tasks, .. } => {
                 tasks.iter().map(|task| task.uid).collect()
@@ -166,7 +167,6 @@ impl Batch {
                 | IndexOperation::DocumentClear { tasks, .. } => {
                     tasks.iter().map(|task| task.uid).collect()
                 }
-                IndexOperation::IndexDocumentDeletionByFilter { task, .. } => vec![task.uid],
                 IndexOperation::SettingsAndDocumentOperation {
                     document_import_tasks: tasks,
                     settings_tasks: other,
@@ -194,7 +194,8 @@ impl Batch {
             IndexOperation { op, .. } => Some(op.index_uid()),
             IndexCreation { index_uid, .. }
             | IndexUpdate { index_uid, .. }
-            | IndexDeletion { index_uid, .. } => Some(index_uid),
+            | IndexDeletion { index_uid, .. }
+            | IndexDocumentDeletionByFilter { index_uid, .. } => Some(index_uid),
         }
     }
 }
@@ -204,7 +205,6 @@ impl IndexOperation {
         match self {
             IndexOperation::DocumentOperation { index_uid, .. }
             | IndexOperation::DocumentDeletion { index_uid, .. }
-            | IndexOperation::IndexDocumentDeletionByFilter { index_uid, .. }
             | IndexOperation::DocumentClear { index_uid, .. }
             | IndexOperation::Settings { index_uid, .. }
             | IndexOperation::DocumentClearAndSetting { index_uid, .. }
@@ -239,12 +239,9 @@ impl IndexScheduler {
                 let task = self.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?;
                 match &task.kind {
                     KindWithContent::DocumentDeletionByFilter { index_uid, .. } => {
-                        Ok(Some(Batch::IndexOperation {
-                            op: IndexOperation::IndexDocumentDeletionByFilter {
-                                index_uid: index_uid.clone(),
-                                task,
-                            },
-                            must_create_index: false,
+                        Ok(Some(Batch::IndexDocumentDeletionByFilter {
+                            index_uid: index_uid.clone(),
+                            task,
                         }))
                     }
                     _ => unreachable!(),
@@ -899,6 +896,51 @@ impl IndexScheduler {
 
                 Ok(tasks)
             }
+            Batch::IndexDocumentDeletionByFilter { mut task, index_uid: _ } => {
+                let (index_uid, filter) =
+                    if let KindWithContent::DocumentDeletionByFilter { index_uid, filter_expr } =
+                        &task.kind
+                    {
+                        (index_uid, filter_expr)
+                    } else {
+                        unreachable!()
+                    };
+                let index = {
+                    let rtxn = self.env.read_txn()?;
+                    self.index_mapper.index(&rtxn, index_uid)?
+                };
+                let deleted_documents = delete_document_by_filter(filter, index);
+                let original_filter = if let Some(Details::DocumentDeletionByFilter {
+                    original_filter,
+                    deleted_documents: _,
+                }) = task.details
+                {
+                    original_filter
+                } else {
+                    // In the case of a `documentDeleteByFilter` the details MUST be set
+                    unreachable!();
+                };
+
+                match deleted_documents {
+                    Ok(deleted_documents) => {
+                        task.status = Status::Succeeded;
+                        task.details = Some(Details::DocumentDeletionByFilter {
+                            original_filter,
+                            deleted_documents: Some(deleted_documents),
+                        });
+                    }
+                    Err(e) => {
+                        task.status = Status::Failed;
+                        task.details = Some(Details::DocumentDeletionByFilter {
+                            original_filter,
+                            deleted_documents: Some(0),
+                        });
+                        task.error = Some(e.into());
+                    }
+                }
+
+                Ok(vec![task])
+            }
             Batch::IndexCreation { index_uid, primary_key, task } => {
                 let wtxn = self.env.write_txn()?;
                 if self.index_mapper.exists(&wtxn, &index_uid)? {
@@ -1023,7 +1065,13 @@ impl IndexScheduler {
     }
 
     /// Swap the index `lhs` with the index `rhs`.
-    fn apply_index_swap(&self, wtxn: &mut RwTxn, task_id: u32, lhs: &str, rhs: &str) -> Result<()> {
+    fn apply_index_swap(
+        &self,
+        wtxn: &mut RwTxn,
+        task_id: TaskId,
+        lhs: &str,
+        rhs: &str,
+    ) -> Result<()> {
         // 1. Verify that both lhs and rhs are existing indexes
         let index_lhs_exists = self.index_mapper.index_exists(wtxn, lhs)?;
         if !index_lhs_exists {
@@ -1044,7 +1092,7 @@ impl IndexScheduler {
         for task_id in &index_lhs_task_ids | &index_rhs_task_ids {
             let mut task = self.get_task(wtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?;
             swap_index_uid_in_task(&mut task, (lhs, rhs));
-            self.all_tasks.put(wtxn, &BEU32::new(task_id), &task)?;
+            self.all_tasks.put(wtxn, &BEU64::new(task_id), &task)?;
         }
 
         // 4. remove the task from indexuid = before_name
@@ -1257,47 +1305,6 @@ impl IndexScheduler {
 
                 Ok(tasks)
             }
-            IndexOperation::IndexDocumentDeletionByFilter { mut task, index_uid: _ } => {
-                let filter =
-                    if let KindWithContent::DocumentDeletionByFilter { filter_expr, .. } =
-                        &task.kind
-                    {
-                        filter_expr
-                    } else {
-                        unreachable!()
-                    };
-                let deleted_documents = delete_document_by_filter(index_wtxn, filter, index);
-                let original_filter = if let Some(Details::DocumentDeletionByFilter {
-                    original_filter,
-                    deleted_documents: _,
-                }) = task.details
-                {
-                    original_filter
-                } else {
-                    // In the case of a `documentDeleteByFilter` the details MUST be set
-                    unreachable!();
-                };
-
-                match deleted_documents {
-                    Ok(deleted_documents) => {
-                        task.status = Status::Succeeded;
-                        task.details = Some(Details::DocumentDeletionByFilter {
-                            original_filter,
-                            deleted_documents: Some(deleted_documents),
-                        });
-                    }
-                    Err(e) => {
-                        task.status = Status::Failed;
-                        task.details = Some(Details::DocumentDeletionByFilter {
-                            original_filter,
-                            deleted_documents: Some(0),
-                        });
-                        task.error = Some(e.into());
-                    }
-                }
-
-                Ok(vec![task])
-            }
             IndexOperation::Settings { index_uid: _, settings, mut tasks } => {
                 let indexer_config = self.index_mapper.indexer_config();
                 let mut builder = milli::update::Settings::new(index_wtxn, index, indexer_config);
@@ -1388,7 +1395,11 @@ impl IndexScheduler {
     /// Delete each given task from all the databases (if it is deleteable).
     ///
     /// Return the number of tasks that were actually deleted.
-    fn delete_matched_tasks(&self, wtxn: &mut RwTxn, matched_tasks: &RoaringBitmap) -> Result<u64> {
+    fn delete_matched_tasks(
+        &self,
+        wtxn: &mut RwTxn,
+        matched_tasks: &RoaringTreemap,
+    ) -> Result<u64> {
         // 1. Remove from this list the tasks that we are not allowed to delete
         let enqueued_tasks = self.get_status(wtxn, Status::Enqueued)?;
         let processing_tasks = &self.processing_tasks.read().unwrap().processing.clone();
@@ -1403,7 +1414,7 @@ impl IndexScheduler {
         let mut affected_indexes = HashSet::new();
         let mut affected_statuses = HashSet::new();
         let mut affected_kinds = HashSet::new();
-        let mut affected_canceled_by = RoaringBitmap::new();
+        let mut affected_canceled_by = RoaringTreemap::new();
 
         for task_id in to_delete_tasks.iter() {
             let task = self.get_task(wtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?;
@@ -1440,10 +1451,10 @@ impl IndexScheduler {
         }
 
         for task in to_delete_tasks.iter() {
-            self.all_tasks.delete(wtxn, &BEU32::new(task))?;
+            self.all_tasks.delete(wtxn, &BEU64::new(task))?;
         }
         for canceled_by in affected_canceled_by {
-            let canceled_by = BEU32::new(canceled_by);
+            let canceled_by = BEU64::new(canceled_by);
             if let Some(mut tasks) = self.canceled_by.get(wtxn, &canceled_by)? {
                 tasks -= &to_delete_tasks;
                 if tasks.is_empty() {
@@ -1464,9 +1475,9 @@ impl IndexScheduler {
         &self,
         wtxn: &mut RwTxn,
         cancel_task_id: TaskId,
-        matched_tasks: &RoaringBitmap,
+        matched_tasks: &RoaringTreemap,
         previous_started_at: OffsetDateTime,
-        previous_processing_tasks: &RoaringBitmap,
+        previous_processing_tasks: &RoaringTreemap,
     ) -> Result<Vec<Uuid>> {
         let now = OffsetDateTime::now_utc();
 
@@ -1491,28 +1502,29 @@ impl IndexScheduler {
             task.details = task.details.map(|d| d.to_failed());
             self.update_task(wtxn, &task)?;
         }
-        self.canceled_by.put(wtxn, &BEU32::new(cancel_task_id), &tasks_to_cancel)?;
+        self.canceled_by.put(wtxn, &BEU64::new(cancel_task_id), &tasks_to_cancel)?;
 
         Ok(content_files_to_delete)
     }
 }
 
-fn delete_document_by_filter<'a>(
-    wtxn: &mut RwTxn<'a, '_>,
-    filter: &serde_json::Value,
-    index: &'a Index,
-) -> Result<u64> {
+fn delete_document_by_filter(filter: &serde_json::Value, index: Index) -> Result<u64> {
     let filter = Filter::from_json(filter)?;
     Ok(if let Some(filter) = filter {
-        let candidates = filter.evaluate(wtxn, index).map_err(|err| match err {
+        let mut wtxn = index.write_txn()?;
+
+        let candidates = filter.evaluate(&wtxn, &index).map_err(|err| match err {
             milli::Error::UserError(milli::UserError::InvalidFilter(_)) => {
                 Error::from(err).with_custom_error_code(Code::InvalidDocumentFilter)
             }
             e => e.into(),
         })?;
-        let mut delete_operation = DeleteDocuments::new(wtxn, index)?;
+        let mut delete_operation = DeleteDocuments::new(&mut wtxn, &index)?;
         delete_operation.delete_documents(&candidates);
-        delete_operation.execute().map(|result| result.deleted_documents)?
+        let deleted_documents =
+            delete_operation.execute().map(|result| result.deleted_documents)?;
+        wtxn.commit()?;
+        deleted_documents
     } else {
         0
     })

index-scheduler/src/error.rs

@@ -48,6 +48,8 @@ impl From<DateField> for Code {
 pub enum Error {
     #[error("{1}")]
     WithCustomErrorCode(Code, Box<Self>),
+    #[error("Received bad task id: {received} should be >= to {expected}.")]
+    BadTaskId { received: TaskId, expected: TaskId },
     #[error("Index `{0}` not found.")]
     IndexNotFound(String),
     #[error("Index `{0}` already exists.")]
@@ -159,6 +161,7 @@ impl Error {
         match self {
             Error::IndexNotFound(_)
             | Error::WithCustomErrorCode(_, _)
+            | Error::BadTaskId { .. }
             | Error::IndexAlreadyExists(_)
             | Error::SwapDuplicateIndexFound(_)
             | Error::SwapDuplicateIndexesFound(_)
@@ -202,6 +205,7 @@ impl ErrorCode for Error {
     fn error_code(&self) -> Code {
         match self {
             Error::WithCustomErrorCode(code, _) => *code,
+            Error::BadTaskId { .. } => Code::BadRequest,
             Error::IndexNotFound(_) => Code::IndexNotFound,
             Error::IndexAlreadyExists(_) => Code::IndexAlreadyExists,
             Error::SwapDuplicateIndexesFound(_) => Code::InvalidSwapDuplicateIndexFound,

index-scheduler/src/insta_snapshot.rs

@@ -3,9 +3,10 @@ use std::fmt::Write;
 
 use meilisearch_types::heed::types::{OwnedType, SerdeBincode, SerdeJson, Str};
 use meilisearch_types::heed::{Database, RoTxn};
-use meilisearch_types::milli::{CboRoaringBitmapCodec, RoaringBitmapCodec, BEU32};
+use meilisearch_types::milli::heed_codec::{CboRoaringTreemapCodec, RoaringTreemapCodec};
+use meilisearch_types::milli::BEU64;
 use meilisearch_types::tasks::{Details, Task};
-use roaring::RoaringBitmap;
+use roaring::RoaringTreemap;
 
 use crate::index_mapper::IndexMapper;
 use crate::{IndexScheduler, Kind, Status, BEI128};
@@ -47,7 +48,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
     let processing_tasks = processing_tasks.read().unwrap().processing.clone();
     snap.push_str(&format!("### Autobatching Enabled = {autobatching_enabled}\n"));
     snap.push_str("### Processing Tasks:\n");
-    snap.push_str(&snapshot_bitmap(&processing_tasks));
+    snap.push_str(&snapshot_treemap(&processing_tasks));
     snap.push_str("\n----------------------------------------------------------------------\n");
 
     snap.push_str("### All Tasks:\n");
@@ -103,7 +104,7 @@ pub fn snapshot_file_store(file_store: &file_store::FileStore) -> String {
     snap
 }
 
-pub fn snapshot_bitmap(r: &RoaringBitmap) -> String {
+pub fn snapshot_treemap(r: &RoaringTreemap) -> String {
     let mut snap = String::new();
     snap.push('[');
     for x in r {
@@ -113,7 +114,7 @@ pub fn snapshot_bitmap(r: &RoaringBitmap) -> String {
     snap
 }
 
-pub fn snapshot_all_tasks(rtxn: &RoTxn, db: Database<OwnedType<BEU32>, SerdeJson<Task>>) -> String {
+pub fn snapshot_all_tasks(rtxn: &RoTxn, db: Database<OwnedType<BEU64>, SerdeJson<Task>>) -> String {
     let mut snap = String::new();
     let iter = db.iter(rtxn).unwrap();
     for next in iter {
@@ -125,13 +126,13 @@ pub fn snapshot_all_tasks(rtxn: &RoTxn, db: Database<OwnedType<BEU32>, SerdeJson
 
 pub fn snapshot_date_db(
     rtxn: &RoTxn,
-    db: Database<OwnedType<BEI128>, CboRoaringBitmapCodec>,
+    db: Database<OwnedType<BEI128>, CboRoaringTreemapCodec>,
 ) -> String {
     let mut snap = String::new();
     let iter = db.iter(rtxn).unwrap();
     for next in iter {
         let (_timestamp, task_ids) = next.unwrap();
-        snap.push_str(&format!("[timestamp] {}\n", snapshot_bitmap(&task_ids)));
+        snap.push_str(&format!("[timestamp] {}\n", snapshot_treemap(&task_ids)));
     }
     snap
 }
@@ -216,45 +217,48 @@ fn snapshot_details(d: &Details) -> String {
 
 pub fn snapshot_status(
     rtxn: &RoTxn,
-    db: Database<SerdeBincode<Status>, RoaringBitmapCodec>,
+    db: Database<SerdeBincode<Status>, RoaringTreemapCodec>,
 ) -> String {
     let mut snap = String::new();
     let iter = db.iter(rtxn).unwrap();
     for next in iter {
         let (status, task_ids) = next.unwrap();
-        writeln!(snap, "{status} {}", snapshot_bitmap(&task_ids)).unwrap();
+        writeln!(snap, "{status} {}", snapshot_treemap(&task_ids)).unwrap();
     }
     snap
 }
-pub fn snapshot_kind(rtxn: &RoTxn, db: Database<SerdeBincode<Kind>, RoaringBitmapCodec>) -> String {
-    let mut snap = String::new();
-    let iter = db.iter(rtxn).unwrap();
-    for next in iter {
-        let (kind, task_ids) = next.unwrap();
-        let kind = serde_json::to_string(&kind).unwrap();
-        writeln!(snap, "{kind} {}", snapshot_bitmap(&task_ids)).unwrap();
-    }
-    snap
-}
-
-pub fn snapshot_index_tasks(rtxn: &RoTxn, db: Database<Str, RoaringBitmapCodec>) -> String {
-    let mut snap = String::new();
-    let iter = db.iter(rtxn).unwrap();
-    for next in iter {
-        let (index, task_ids) = next.unwrap();
-        writeln!(snap, "{index} {}", snapshot_bitmap(&task_ids)).unwrap();
-    }
-    snap
-}
-pub fn snapshot_canceled_by(
+pub fn snapshot_kind(
     rtxn: &RoTxn,
-    db: Database<OwnedType<BEU32>, RoaringBitmapCodec>,
+    db: Database<SerdeBincode<Kind>, RoaringTreemapCodec>,
 ) -> String {
     let mut snap = String::new();
     let iter = db.iter(rtxn).unwrap();
     for next in iter {
         let (kind, task_ids) = next.unwrap();
-        writeln!(snap, "{kind} {}", snapshot_bitmap(&task_ids)).unwrap();
+        let kind = serde_json::to_string(&kind).unwrap();
+        writeln!(snap, "{kind} {}", snapshot_treemap(&task_ids)).unwrap();
+    }
+    snap
+}
+
+pub fn snapshot_index_tasks(rtxn: &RoTxn, db: Database<Str, RoaringTreemapCodec>) -> String {
+    let mut snap = String::new();
+    let iter = db.iter(rtxn).unwrap();
+    for next in iter {
+        let (index, task_ids) = next.unwrap();
+        writeln!(snap, "{index} {}", snapshot_treemap(&task_ids)).unwrap();
+    }
+    snap
+}
+pub fn snapshot_canceled_by(
+    rtxn: &RoTxn,
+    db: Database<OwnedType<BEU64>, RoaringTreemapCodec>,
+) -> String {
+    let mut snap = String::new();
+    let iter = db.iter(rtxn).unwrap();
+    for next in iter {
+        let (kind, task_ids) = next.unwrap();
+        writeln!(snap, "{kind} {}", snapshot_treemap(&task_ids)).unwrap();
     }
     snap
 }

index-scheduler/src/snapshots/lib.rs/cancel_enqueued_task/cancel_processed.snap

@@ -7,7 +7,7 @@ source: index-scheduler/src/lib.rs
 ----------------------------------------------------------------------
 ### All Tasks:
 0 {uid: 0, status: canceled, canceled_by: 1, details: { received_documents: 1, indexed_documents: Some(0) }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
-1 {uid: 1, status: succeeded, details: { matched_tasks: 1, canceled_tasks: Some(1), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0]> }}
+1 {uid: 1, status: succeeded, details: { matched_tasks: 1, canceled_tasks: Some(1), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringTreemap<[0]> }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []

index-scheduler/src/snapshots/lib.rs/cancel_enqueued_task/initial_tasks_enqueued.snap

@@ -7,7 +7,7 @@ source: index-scheduler/src/lib.rs
 ----------------------------------------------------------------------
 ### All Tasks:
 0 {uid: 0, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
-1 {uid: 1, status: enqueued, details: { matched_tasks: 1, canceled_tasks: None, original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0]> }}
+1 {uid: 1, status: enqueued, details: { matched_tasks: 1, canceled_tasks: None, original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringTreemap<[0]> }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,1,]

index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/aborted_indexation.snap

@@ -9,7 +9,7 @@ source: index-scheduler/src/lib.rs
 0 {uid: 0, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "beavero", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "wolfo", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: true }}
-3 {uid: 3, status: enqueued, details: { matched_tasks: 3, canceled_tasks: None, original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0, 1, 2]> }}
+3 {uid: 3, status: enqueued, details: { matched_tasks: 3, canceled_tasks: None, original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringTreemap<[0, 1, 2]> }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [1,2,3,]

index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/cancel_processed.snap

@@ -9,7 +9,7 @@ source: index-scheduler/src/lib.rs
 0 {uid: 0, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 1 {uid: 1, status: canceled, canceled_by: 3, details: { received_documents: 1, indexed_documents: Some(0) }, kind: DocumentAdditionOrUpdate { index_uid: "beavero", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
 2 {uid: 2, status: canceled, canceled_by: 3, details: { received_documents: 1, indexed_documents: Some(0) }, kind: DocumentAdditionOrUpdate { index_uid: "wolfo", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: true }}
-3 {uid: 3, status: succeeded, details: { matched_tasks: 3, canceled_tasks: Some(2), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0, 1, 2]> }}
+3 {uid: 3, status: succeeded, details: { matched_tasks: 3, canceled_tasks: Some(2), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringTreemap<[0, 1, 2]> }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []

index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/processing_second_task_cancel_enqueued.snap

@@ -9,7 +9,7 @@ source: index-scheduler/src/lib.rs
 0 {uid: 0, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "beavero", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "wolfo", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: true }}
-3 {uid: 3, status: enqueued, details: { matched_tasks: 3, canceled_tasks: None, original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0, 1, 2]> }}
+3 {uid: 3, status: enqueued, details: { matched_tasks: 3, canceled_tasks: None, original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringTreemap<[0, 1, 2]> }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [1,2,3,]

index-scheduler/src/snapshots/lib.rs/cancel_processing_task/aborted_indexation.snap

@@ -7,7 +7,7 @@ source: index-scheduler/src/lib.rs
 ----------------------------------------------------------------------
 ### All Tasks:
 0 {uid: 0, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
-1 {uid: 1, status: enqueued, details: { matched_tasks: 1, canceled_tasks: None, original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0]> }}
+1 {uid: 1, status: enqueued, details: { matched_tasks: 1, canceled_tasks: None, original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringTreemap<[0]> }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,1,]

index-scheduler/src/snapshots/lib.rs/cancel_processing_task/cancel_processed.snap

@@ -7,7 +7,7 @@ source: index-scheduler/src/lib.rs
 ----------------------------------------------------------------------
 ### All Tasks:
 0 {uid: 0, status: canceled, canceled_by: 1, details: { received_documents: 1, indexed_documents: Some(0) }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
-1 {uid: 1, status: succeeded, details: { matched_tasks: 1, canceled_tasks: Some(1), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0]> }}
+1 {uid: 1, status: succeeded, details: { matched_tasks: 1, canceled_tasks: Some(1), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringTreemap<[0]> }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []

index-scheduler/src/snapshots/lib.rs/cancel_processing_task/cancel_task_registered.snap

@@ -7,7 +7,7 @@ source: index-scheduler/src/lib.rs
 ----------------------------------------------------------------------
 ### All Tasks:
 0 {uid: 0, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
-1 {uid: 1, status: enqueued, details: { matched_tasks: 1, canceled_tasks: None, original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0]> }}
+1 {uid: 1, status: enqueued, details: { matched_tasks: 1, canceled_tasks: None, original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringTreemap<[0]> }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,1,]

index-scheduler/src/snapshots/lib.rs/cancel_succeeded_task/cancel_processed.snap

@@ -7,7 +7,7 @@ source: index-scheduler/src/lib.rs
 ----------------------------------------------------------------------
 ### All Tasks:
 0 {uid: 0, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
-1 {uid: 1, status: succeeded, details: { matched_tasks: 1, canceled_tasks: Some(0), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0]> }}
+1 {uid: 1, status: succeeded, details: { matched_tasks: 1, canceled_tasks: Some(0), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringTreemap<[0]> }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []

index-scheduler/src/snapshots/lib.rs/query_tasks_canceled_by/start.snap

@@ -9,7 +9,7 @@ source: index-scheduler/src/lib.rs
 0 {uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 1 {uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
 2 {uid: 2, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }}
-3 {uid: 3, status: succeeded, details: { matched_tasks: 3, canceled_tasks: Some(0), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0, 1, 2]> }}
+3 {uid: 3, status: succeeded, details: { matched_tasks: 3, canceled_tasks: Some(0), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringTreemap<[0, 1, 2]> }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []

index-scheduler/src/snapshots/lib.rs/task_deletion_delete_same_task_twice/task_deletion_processed.snap

@@ -7,8 +7,8 @@ source: index-scheduler/src/lib.rs
 ----------------------------------------------------------------------
 ### All Tasks:
 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
-2 {uid: 2, status: succeeded, details: { matched_tasks: 1, deleted_tasks: Some(1), original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0]> }}
-3 {uid: 3, status: succeeded, details: { matched_tasks: 1, deleted_tasks: Some(0), original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0]> }}
+2 {uid: 2, status: succeeded, details: { matched_tasks: 1, deleted_tasks: Some(1), original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringTreemap<[0]> }}
+3 {uid: 3, status: succeeded, details: { matched_tasks: 1, deleted_tasks: Some(0), original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringTreemap<[0]> }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [1,]

index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/after_registering_the_task_deletion.snap

@@ -8,7 +8,7 @@ source: index-scheduler/src/lib.rs
 ### All Tasks:
 0 {uid: 0, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
-2 {uid: 2, status: enqueued, details: { matched_tasks: 1, deleted_tasks: None, original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0]> }}
+2 {uid: 2, status: enqueued, details: { matched_tasks: 1, deleted_tasks: None, original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringTreemap<[0]> }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [1,2,]

index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/task_deletion_processed.snap

@@ -7,7 +7,7 @@ source: index-scheduler/src/lib.rs
 ----------------------------------------------------------------------
 ### All Tasks:
 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
-2 {uid: 2, status: succeeded, details: { matched_tasks: 1, deleted_tasks: Some(1), original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0]> }}
+2 {uid: 2, status: succeeded, details: { matched_tasks: 1, deleted_tasks: Some(1), original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringTreemap<[0]> }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [1,]

index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_done.snap

@@ -9,7 +9,7 @@ source: index-scheduler/src/lib.rs
 0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
-3 {uid: 3, status: succeeded, details: { matched_tasks: 2, deleted_tasks: Some(0), original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }}
+3 {uid: 3, status: succeeded, details: { matched_tasks: 2, deleted_tasks: Some(0), original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringTreemap<[0, 1]> }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,1,2,]

index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_enqueued.snap

@@ -9,7 +9,7 @@ source: index-scheduler/src/lib.rs
 0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
-3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }}
+3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringTreemap<[0, 1]> }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,1,2,3,]

index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_processing.snap

@@ -9,7 +9,7 @@ source: index-scheduler/src/lib.rs
 0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
-3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }}
+3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringTreemap<[0, 1]> }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,1,2,3,]

index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap

@@ -41,6 +41,18 @@ source: index-scheduler/src/lib.rs
       "taskDeletion": {
         "query": "[query]",
         "tasks": [
+          1,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
           58,
           48,
           0,

index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/everything_has_been_processed.snap

@@ -21,6 +21,18 @@ source: index-scheduler/src/lib.rs
       "taskDeletion": {
         "query": "[query]",
         "tasks": [
+          1,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
           58,
           48,
           0,

index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap

@@ -106,6 +106,18 @@ source: index-scheduler/src/lib.rs
       "taskDeletion": {
         "query": "[query]",
         "tasks": [
+          1,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
           58,
           48,
           0,

index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap

@@ -61,6 +61,18 @@ source: index-scheduler/src/lib.rs
       "taskDeletion": {
         "query": "[query]",
         "tasks": [
+          1,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
+          0,
           58,
           48,
           0,

index-scheduler/src/utils.rs

@@ -5,15 +5,16 @@ use std::ops::Bound;
 
 use meilisearch_types::heed::types::{DecodeIgnore, OwnedType};
 use meilisearch_types::heed::{Database, RoTxn, RwTxn};
-use meilisearch_types::milli::{CboRoaringBitmapCodec, BEU32};
+use meilisearch_types::milli::heed_codec::CboRoaringTreemapCodec;
+use meilisearch_types::milli::BEU64;
 use meilisearch_types::tasks::{Details, IndexSwap, Kind, KindWithContent, Status};
-use roaring::{MultiOps, RoaringBitmap};
+use roaring::{MultiOps, RoaringTreemap};
 use time::OffsetDateTime;
 
 use crate::{Error, IndexScheduler, Result, Task, TaskId, BEI128};
 
 impl IndexScheduler {
-    pub(crate) fn all_task_ids(&self, rtxn: &RoTxn) -> Result<RoaringBitmap> {
+    pub(crate) fn all_task_ids(&self, rtxn: &RoTxn) -> Result<RoaringTreemap> {
         enum_iterator::all().map(|s| self.get_status(rtxn, s)).union()
     }
 
@@ -26,7 +27,7 @@ impl IndexScheduler {
     }
 
     pub(crate) fn get_task(&self, rtxn: &RoTxn, task_id: TaskId) -> Result<Option<Task>> {
-        Ok(self.all_tasks.get(rtxn, &BEU32::new(task_id))?)
+        Ok(self.all_tasks.get(rtxn, &BEU64::new(task_id))?)
     }
 
     /// Convert an iterator to a `Vec` of tasks. The tasks MUST exist or a
@@ -88,12 +89,12 @@ impl IndexScheduler {
             }
         }
 
-        self.all_tasks.put(wtxn, &BEU32::new(task.uid), task)?;
+        self.all_tasks.put(wtxn, &BEU64::new(task.uid), task)?;
         Ok(())
     }
 
     /// Returns the whole set of tasks that belongs to this index.
-    pub(crate) fn index_tasks(&self, rtxn: &RoTxn, index: &str) -> Result<RoaringBitmap> {
+    pub(crate) fn index_tasks(&self, rtxn: &RoTxn, index: &str) -> Result<RoaringTreemap> {
         Ok(self.index_tasks.get(rtxn, index)?.unwrap_or_default())
     }
 
@@ -101,7 +102,7 @@ impl IndexScheduler {
         &self,
         wtxn: &mut RwTxn,
         index: &str,
-        f: impl Fn(&mut RoaringBitmap),
+        f: impl Fn(&mut RoaringTreemap),
     ) -> Result<()> {
         let mut tasks = self.index_tasks(wtxn, index)?;
         f(&mut tasks);
@@ -114,7 +115,7 @@ impl IndexScheduler {
         Ok(())
     }
 
-    pub(crate) fn get_status(&self, rtxn: &RoTxn, status: Status) -> Result<RoaringBitmap> {
+    pub(crate) fn get_status(&self, rtxn: &RoTxn, status: Status) -> Result<RoaringTreemap> {
         Ok(self.status.get(rtxn, &status)?.unwrap_or_default())
     }
 
@@ -122,7 +123,7 @@ impl IndexScheduler {
         &self,
         wtxn: &mut RwTxn,
         status: Status,
-        bitmap: &RoaringBitmap,
+        bitmap: &RoaringTreemap,
     ) -> Result<()> {
         Ok(self.status.put(wtxn, &status, bitmap)?)
     }
@@ -131,7 +132,7 @@ impl IndexScheduler {
         &self,
         wtxn: &mut RwTxn,
         status: Status,
-        f: impl Fn(&mut RoaringBitmap),
+        f: impl Fn(&mut RoaringTreemap),
     ) -> Result<()> {
         let mut tasks = self.get_status(wtxn, status)?;
         f(&mut tasks);
@@ -140,7 +141,7 @@ impl IndexScheduler {
         Ok(())
     }
 
-    pub(crate) fn get_kind(&self, rtxn: &RoTxn, kind: Kind) -> Result<RoaringBitmap> {
+    pub(crate) fn get_kind(&self, rtxn: &RoTxn, kind: Kind) -> Result<RoaringTreemap> {
         Ok(self.kind.get(rtxn, &kind)?.unwrap_or_default())
     }
 
@@ -148,7 +149,7 @@ impl IndexScheduler {
         &self,
         wtxn: &mut RwTxn,
         kind: Kind,
-        bitmap: &RoaringBitmap,
+        bitmap: &RoaringTreemap,
     ) -> Result<()> {
         Ok(self.kind.put(wtxn, &kind, bitmap)?)
     }
@@ -157,7 +158,7 @@ impl IndexScheduler {
         &self,
         wtxn: &mut RwTxn,
         kind: Kind,
-        f: impl Fn(&mut RoaringBitmap),
+        f: impl Fn(&mut RoaringTreemap),
     ) -> Result<()> {
         let mut tasks = self.get_kind(wtxn, kind)?;
         f(&mut tasks);
@@ -169,20 +170,20 @@ impl IndexScheduler {
 
 pub(crate) fn insert_task_datetime(
     wtxn: &mut RwTxn,
-    database: Database<OwnedType<BEI128>, CboRoaringBitmapCodec>,
+    database: Database<OwnedType<BEI128>, CboRoaringTreemapCodec>,
     time: OffsetDateTime,
     task_id: TaskId,
 ) -> Result<()> {
     let timestamp = BEI128::new(time.unix_timestamp_nanos());
     let mut task_ids = database.get(wtxn, &timestamp)?.unwrap_or_default();
     task_ids.insert(task_id);
-    database.put(wtxn, &timestamp, &RoaringBitmap::from_iter(task_ids))?;
+    database.put(wtxn, &timestamp, &RoaringTreemap::from_iter(task_ids))?;
     Ok(())
 }
 
 pub(crate) fn remove_task_datetime(
     wtxn: &mut RwTxn,
-    database: Database<OwnedType<BEI128>, CboRoaringBitmapCodec>,
+    database: Database<OwnedType<BEI128>, CboRoaringTreemapCodec>,
     time: OffsetDateTime,
     task_id: TaskId,
 ) -> Result<()> {
@@ -192,7 +193,7 @@ pub(crate) fn remove_task_datetime(
         if existing.is_empty() {
             database.delete(wtxn, &timestamp)?;
         } else {
-            database.put(wtxn, &timestamp, &RoaringBitmap::from_iter(existing))?;
+            database.put(wtxn, &timestamp, &RoaringTreemap::from_iter(existing))?;
         }
     }
 
@@ -201,8 +202,8 @@ pub(crate) fn remove_task_datetime(
 
 pub(crate) fn keep_tasks_within_datetimes(
     rtxn: &RoTxn,
-    tasks: &mut RoaringBitmap,
-    database: Database<OwnedType<BEI128>, CboRoaringBitmapCodec>,
+    tasks: &mut RoaringTreemap,
+    database: Database<OwnedType<BEI128>, CboRoaringTreemapCodec>,
     after: Option<OffsetDateTime>,
     before: Option<OffsetDateTime>,
 ) -> Result<()> {
@@ -212,7 +213,7 @@ pub(crate) fn keep_tasks_within_datetimes(
         (Some(after), None) => (Bound::Excluded(*after), Bound::Unbounded),
         (Some(after), Some(before)) => (Bound::Excluded(*after), Bound::Excluded(*before)),
     };
-    let mut collected_task_ids = RoaringBitmap::new();
+    let mut collected_task_ids = RoaringTreemap::new();
     let start = map_bound(start, |b| BEI128::new(b.unix_timestamp_nanos()));
     let end = map_bound(end, |b| BEI128::new(b.unix_timestamp_nanos()));
     let iter = database.range(rtxn, &(start, end))?;

meilisearch-auth/src/store.rs

@@ -129,9 +129,6 @@ impl HeedAuthStore {
                 Action::DumpsAll => {
                     actions.insert(Action::DumpsCreate);
                 }
-                Action::SnapshotsAll => {
-                    actions.insert(Action::SnapshotsCreate);
-                }
                 Action::TasksAll => {
                     actions.extend([Action::TasksGet, Action::TasksDelete, Action::TasksCancel]);
                 }

meilisearch-types/src/deserr/query_params.rs

@@ -104,6 +104,7 @@ macro_rules! impl_from_query_param_wrap_original_value_in_error {
 }
 impl_from_query_param_wrap_original_value_in_error!(usize, DeserrParseIntError);
 impl_from_query_param_wrap_original_value_in_error!(u32, DeserrParseIntError);
+impl_from_query_param_wrap_original_value_in_error!(u64, DeserrParseIntError);
 impl_from_query_param_wrap_original_value_in_error!(bool, DeserrParseBoolError);
 
 impl FromQueryParameter for String {

meilisearch-types/src/keys.rs

@@ -257,12 +257,6 @@ pub enum Action {
     #[serde(rename = "dumps.create")]
     #[deserr(rename = "dumps.create")]
     DumpsCreate,
-    #[serde(rename = "snapshots.*")]
-    #[deserr(rename = "snapshots.*")]
-    SnapshotsAll,
-    #[serde(rename = "snapshots.create")]
-    #[deserr(rename = "snapshots.create")]
-    SnapshotsCreate,
     #[serde(rename = "version")]
     #[deserr(rename = "version")]
     Version,
@@ -315,7 +309,6 @@ impl Action {
             METRICS_GET => Some(Self::MetricsGet),
             DUMPS_ALL => Some(Self::DumpsAll),
             DUMPS_CREATE => Some(Self::DumpsCreate),
-            SNAPSHOTS_CREATE => Some(Self::SnapshotsCreate),
             VERSION => Some(Self::Version),
             KEYS_CREATE => Some(Self::KeysAdd),
             KEYS_GET => Some(Self::KeysGet),
@@ -360,7 +353,6 @@ pub mod actions {
     pub const METRICS_GET: u8 = MetricsGet.repr();
     pub const DUMPS_ALL: u8 = DumpsAll.repr();
     pub const DUMPS_CREATE: u8 = DumpsCreate.repr();
-    pub const SNAPSHOTS_CREATE: u8 = SnapshotsCreate.repr();
     pub const VERSION: u8 = Version.repr();
     pub const KEYS_CREATE: u8 = KeysAdd.repr();
     pub const KEYS_GET: u8 = KeysGet.repr();

meilisearch-types/src/tasks.rs

@@ -5,7 +5,7 @@ use std::str::FromStr;
 
 use enum_iterator::Sequence;
 use milli::update::IndexDocumentsMethod;
-use roaring::RoaringBitmap;
+use roaring::RoaringTreemap;
 use serde::{Deserialize, Serialize, Serializer};
 use time::{Duration, OffsetDateTime};
 use uuid::Uuid;
@@ -15,7 +15,7 @@ use crate::keys::Key;
 use crate::settings::{Settings, Unchecked};
 use crate::InstanceUid;
 
-pub type TaskId = u32;
+pub type TaskId = u64;
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -127,11 +127,11 @@ pub enum KindWithContent {
     },
     TaskCancelation {
         query: String,
-        tasks: RoaringBitmap,
+        tasks: RoaringTreemap,
     },
     TaskDeletion {
         query: String,
-        tasks: RoaringBitmap,
+        tasks: RoaringTreemap,
     },
     DumpCreation {
         keys: Vec<Key>,

meilisearch/src/analytics/mod.rs

@@ -1,5 +1,6 @@
 mod mock_analytics;
-#[cfg(feature = "analytics")]
+// if we are in release mode and the feature analytics was enabled
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
 mod segment_analytics;
 
 use std::fs;
@@ -16,25 +17,26 @@ use serde_json::Value;
 use crate::routes::indexes::documents::UpdateDocumentsQuery;
 use crate::routes::tasks::TasksFilterQuery;
 
-// if the analytics feature is disabled
+// if we are in debug mode OR the analytics feature is disabled
 // the `SegmentAnalytics` point to the mock instead of the real analytics
-#[cfg(not(feature = "analytics"))]
+#[cfg(any(debug_assertions, not(feature = "analytics")))]
 pub type SegmentAnalytics = mock_analytics::MockAnalytics;
-#[cfg(not(feature = "analytics"))]
+#[cfg(any(debug_assertions, not(feature = "analytics")))]
 pub type SearchAggregator = mock_analytics::SearchAggregator;
-#[cfg(not(feature = "analytics"))]
+#[cfg(any(debug_assertions, not(feature = "analytics")))]
 pub type MultiSearchAggregator = mock_analytics::MultiSearchAggregator;
-#[cfg(not(feature = "analytics"))]
+#[cfg(any(debug_assertions, not(feature = "analytics")))]
 pub type FacetSearchAggregator = mock_analytics::FacetSearchAggregator;
 
-// if the feature analytics is enabled we use the real analytics
-#[cfg(feature = "analytics")]
+// if we are in release mode and the feature analytics was enabled
+// we use the real analytics
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
 pub type SegmentAnalytics = segment_analytics::SegmentAnalytics;
-#[cfg(feature = "analytics")]
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
 pub type SearchAggregator = segment_analytics::SearchAggregator;
-#[cfg(feature = "analytics")]
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
 pub type MultiSearchAggregator = segment_analytics::MultiSearchAggregator;
-#[cfg(feature = "analytics")]
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
 pub type FacetSearchAggregator = segment_analytics::FacetSearchAggregator;
 
 /// The Meilisearch config dir:

meilisearch/src/lib.rs

@@ -203,7 +203,7 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc<IndexScheduler>, Arc<
             .name(String::from("register-snapshot-tasks"))
             .spawn(move || loop {
                 thread::sleep(snapshot_delay);
-                if let Err(e) = index_scheduler.register(KindWithContent::SnapshotCreation) {
+                if let Err(e) = index_scheduler.register(KindWithContent::SnapshotCreation, None) {
                     error!("Error while registering snapshot: {}", e);
                 }
             })

meilisearch/src/option.rs

@@ -28,7 +28,7 @@ const MEILI_DB_PATH: &str = "MEILI_DB_PATH";
 const MEILI_HTTP_ADDR: &str = "MEILI_HTTP_ADDR";
 const MEILI_MASTER_KEY: &str = "MEILI_MASTER_KEY";
 const MEILI_ENV: &str = "MEILI_ENV";
-#[cfg(feature = "analytics")]
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
 const MEILI_NO_ANALYTICS: &str = "MEILI_NO_ANALYTICS";
 const MEILI_HTTP_PAYLOAD_SIZE_LIMIT: &str = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT";
 const MEILI_SSL_CERT_PATH: &str = "MEILI_SSL_CERT_PATH";
@@ -159,7 +159,7 @@ pub struct Opt {
     /// Meilisearch automatically collects data from all instances that do not opt out using this flag.
     /// All gathered data is used solely for the purpose of improving Meilisearch, and can be deleted
     /// at any time.
-    #[cfg(feature = "analytics")]
+    #[cfg(all(not(debug_assertions), feature = "analytics"))]
     #[serde(default)] // we can't send true
     #[clap(long, env = MEILI_NO_ANALYTICS)]
     pub no_analytics: bool,
@@ -390,7 +390,7 @@ impl Opt {
             ignore_missing_dump: _,
             ignore_dump_if_db_exists: _,
             config_file_path: _,
-            #[cfg(feature = "analytics")]
+            #[cfg(all(not(debug_assertions), feature = "analytics"))]
             no_analytics,
             experimental_enable_metrics: enable_metrics_route,
             experimental_reduce_indexing_memory_usage: reduce_indexing_memory_usage,
@@ -401,7 +401,7 @@ impl Opt {
             export_to_env_if_not_present(MEILI_MASTER_KEY, master_key);
         }
         export_to_env_if_not_present(MEILI_ENV, env);
-        #[cfg(feature = "analytics")]
+        #[cfg(all(not(debug_assertions), feature = "analytics"))]
         {
             export_to_env_if_not_present(MEILI_NO_ANALYTICS, no_analytics.to_string());
         }

meilisearch/src/routes/dump.rs

@@ -11,7 +11,7 @@ use crate::analytics::Analytics;
 use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::GuardedData;
 use crate::extractors::sequential_extractor::SeqHandler;
-use crate::routes::SummarizedTaskView;
+use crate::routes::{get_task_id, SummarizedTaskView};
 
 pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(web::resource("").route(web::post().to(SeqHandler(create_dump))));
@@ -29,8 +29,9 @@ pub async fn create_dump(
         keys: auth_controller.list_keys()?,
         instance_uid: analytics.instance_uid().cloned(),
     };
+    let uid = get_task_id(&req)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid)).await??.into();
 
     debug!("returns: {:?}", task);
     Ok(HttpResponse::Accepted().json(task))

meilisearch/src/routes/indexes/documents.rs

@@ -7,7 +7,7 @@ use bstr::ByteSlice;
 use deserr::actix_web::{AwebJson, AwebQueryParameter};
 use deserr::Deserr;
 use futures::StreamExt;
-use index_scheduler::IndexScheduler;
+use index_scheduler::{IndexScheduler, TaskId};
 use log::debug;
 use meilisearch_types::deserr::query_params::Param;
 use meilisearch_types::deserr::{DeserrJsonError, DeserrQueryParamError};
@@ -36,7 +36,7 @@ use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::GuardedData;
 use crate::extractors::payload::Payload;
 use crate::extractors::sequential_extractor::SeqHandler;
-use crate::routes::{PaginationView, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT};
+use crate::routes::{get_task_id, PaginationView, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT};
 use crate::search::parse_filter;
 
 static ACCEPTED_CONTENT_TYPE: Lazy<Vec<String>> = Lazy::new(|| {
@@ -129,8 +129,9 @@ pub async fn delete_document(
         index_uid: index_uid.to_string(),
         documents_ids: vec![document_id],
     };
+    let uid = get_task_id(&req)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid)).await??.into();
     debug!("returns: {:?}", task);
     Ok(HttpResponse::Accepted().json(task))
 }
@@ -277,6 +278,7 @@ pub async fn replace_documents(
     analytics.add_documents(&params, index_scheduler.index(&index_uid).is_err(), &req);
 
     let allow_index_creation = index_scheduler.filters().allow_index_creation(&index_uid);
+    let uid = get_task_id(&req)?;
     let task = document_addition(
         extract_mime_type(&req)?,
         index_scheduler,
@@ -285,6 +287,7 @@ pub async fn replace_documents(
         params.csv_delimiter,
         body,
         IndexDocumentsMethod::ReplaceDocuments,
+        uid,
         allow_index_creation,
     )
     .await?;
@@ -308,6 +311,7 @@ pub async fn update_documents(
     analytics.update_documents(&params, index_scheduler.index(&index_uid).is_err(), &req);
 
     let allow_index_creation = index_scheduler.filters().allow_index_creation(&index_uid);
+    let uid = get_task_id(&req)?;
     let task = document_addition(
         extract_mime_type(&req)?,
         index_scheduler,
@@ -316,6 +320,7 @@ pub async fn update_documents(
         params.csv_delimiter,
         body,
         IndexDocumentsMethod::UpdateDocuments,
+        uid,
         allow_index_creation,
     )
     .await?;
@@ -332,6 +337,7 @@ async fn document_addition(
     csv_delimiter: Option<u8>,
     mut body: Payload,
     method: IndexDocumentsMethod,
+    task_id: Option<TaskId>,
     allow_index_creation: bool,
 ) -> Result<SummarizedTaskView, MeilisearchHttpError> {
     let format = match (
@@ -445,7 +451,7 @@ async fn document_addition(
     };
 
     let scheduler = index_scheduler.clone();
-    let task = match tokio::task::spawn_blocking(move || scheduler.register(task)).await? {
+    let task = match tokio::task::spawn_blocking(move || scheduler.register(task, task_id)).await? {
         Ok(task) => task,
         Err(e) => {
             index_scheduler.delete_update_file(uuid)?;
@@ -476,8 +482,9 @@ pub async fn delete_documents_batch(
 
     let task =
         KindWithContent::DocumentDeletion { index_uid: index_uid.to_string(), documents_ids: ids };
+    let uid = get_task_id(&req)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid)).await??.into();
 
     debug!("returns: {:?}", task);
     Ok(HttpResponse::Accepted().json(task))
@@ -512,8 +519,9 @@ pub async fn delete_documents_by_filter(
     .map_err(|err| ResponseError::from_msg(err.message, Code::InvalidDocumentFilter))?;
     let task = KindWithContent::DocumentDeletionByFilter { index_uid, filter_expr: filter };
 
+    let uid = get_task_id(&req)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid)).await??.into();
 
     debug!("returns: {:?}", task);
     Ok(HttpResponse::Accepted().json(task))
@@ -529,8 +537,9 @@ pub async fn clear_all_documents(
     analytics.delete_documents(DocumentDeletionKind::ClearAll, &req);
 
     let task = KindWithContent::DocumentClear { index_uid: index_uid.to_string() };
+    let uid = get_task_id(&req)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid)).await??.into();
 
     debug!("returns: {:?}", task);
     Ok(HttpResponse::Accepted().json(task))

meilisearch/src/routes/indexes/mod.rs

@@ -17,7 +17,7 @@ use serde::Serialize;
 use serde_json::json;
 use time::OffsetDateTime;
 
-use super::{Pagination, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT};
+use super::{get_task_id, Pagination, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT};
 use crate::analytics::Analytics;
 use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::{AuthenticationError, GuardedData};
@@ -135,8 +135,9 @@ pub async fn create_index(
         );
 
         let task = KindWithContent::IndexCreation { index_uid: uid.to_string(), primary_key };
+        let uid = get_task_id(&req)?;
         let task: SummarizedTaskView =
-            tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+            tokio::task::spawn_blocking(move || index_scheduler.register(task, uid)).await??.into();
 
         Ok(HttpResponse::Accepted().json(task))
     } else {
@@ -203,8 +204,9 @@ pub async fn update_index(
         primary_key: body.primary_key,
     };
 
+    let uid = get_task_id(&req)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid)).await??.into();
 
     debug!("returns: {:?}", task);
     Ok(HttpResponse::Accepted().json(task))
@@ -213,11 +215,13 @@ pub async fn update_index(
 pub async fn delete_index(
     index_scheduler: GuardedData<ActionPolicy<{ actions::INDEXES_DELETE }>, Data<IndexScheduler>>,
     index_uid: web::Path<String>,
+    req: HttpRequest,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
     let task = KindWithContent::IndexDeletion { index_uid: index_uid.into_inner() };
+    let uid = get_task_id(&req)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid)).await??.into();
 
     Ok(HttpResponse::Accepted().json(task))
 }

meilisearch/src/routes/indexes/settings.rs

@@ -14,7 +14,7 @@ use serde_json::json;
 use crate::analytics::Analytics;
 use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::GuardedData;
-use crate::routes::SummarizedTaskView;
+use crate::routes::{get_task_id, SummarizedTaskView};
 
 #[macro_export]
 macro_rules! make_setting_route {
@@ -33,7 +33,7 @@ macro_rules! make_setting_route {
             use $crate::extractors::authentication::policies::*;
             use $crate::extractors::authentication::GuardedData;
             use $crate::extractors::sequential_extractor::SeqHandler;
-            use $crate::routes::SummarizedTaskView;
+            use $crate::routes::{get_task_id, SummarizedTaskView};
 
             pub async fn delete(
                 index_scheduler: GuardedData<
@@ -41,6 +41,7 @@ macro_rules! make_setting_route {
                     Data<IndexScheduler>,
                 >,
                 index_uid: web::Path<String>,
+                req: HttpRequest,
             ) -> Result<HttpResponse, ResponseError> {
                 let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
@@ -55,8 +56,9 @@ macro_rules! make_setting_route {
                     is_deletion: true,
                     allow_index_creation,
                 };
+                let uid = get_task_id(&req)?;
                 let task: SummarizedTaskView =
-                    tokio::task::spawn_blocking(move || index_scheduler.register(task))
+                    tokio::task::spawn_blocking(move || index_scheduler.register(task, uid))
                         .await??
                         .into();
 
@@ -97,8 +99,9 @@ macro_rules! make_setting_route {
                     is_deletion: false,
                     allow_index_creation,
                 };
+                let uid = get_task_id(&req)?;
                 let task: SummarizedTaskView =
-                    tokio::task::spawn_blocking(move || index_scheduler.register(task))
+                    tokio::task::spawn_blocking(move || index_scheduler.register(task, uid))
                         .await??
                         .into();
 
@@ -664,8 +667,9 @@ pub async fn update_all(
         is_deletion: false,
         allow_index_creation,
     };
+    let uid = get_task_id(&req)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid)).await??.into();
 
     debug!("returns: {:?}", task);
     Ok(HttpResponse::Accepted().json(task))
@@ -687,6 +691,7 @@ pub async fn get_all(
 pub async fn delete_all(
     index_scheduler: GuardedData<ActionPolicy<{ actions::SETTINGS_UPDATE }>, Data<IndexScheduler>>,
     index_uid: web::Path<String>,
+    req: HttpRequest,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
@@ -700,8 +705,9 @@ pub async fn delete_all(
         is_deletion: true,
         allow_index_creation,
     };
+    let uid = get_task_id(&req)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid)).await??.into();
 
     debug!("returns: {:?}", task);
     Ok(HttpResponse::Accepted().json(task))

meilisearch/src/routes/mod.rs

@@ -5,7 +5,7 @@ use actix_web::{web, HttpRequest, HttpResponse};
 use index_scheduler::IndexScheduler;
 use log::debug;
 use meilisearch_auth::AuthController;
-use meilisearch_types::error::ResponseError;
+use meilisearch_types::error::{Code, ResponseError};
 use meilisearch_types::settings::{Settings, Unchecked};
 use meilisearch_types::tasks::{Kind, Status, Task, TaskId};
 use serde::{Deserialize, Serialize};
@@ -24,7 +24,6 @@ pub mod features;
 pub mod indexes;
 mod metrics;
 mod multi_search;
-mod snapshot;
 mod swap_indexes;
 pub mod tasks;
 
@@ -33,7 +32,6 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
         .service(web::resource("/health").route(web::get().to(get_health)))
         .service(web::scope("/keys").configure(api_key::configure))
         .service(web::scope("/dumps").configure(dump::configure))
-        .service(web::scope("/snapshots").configure(snapshot::configure))
         .service(web::resource("/stats").route(web::get().to(get_stats)))
         .service(web::resource("/version").route(web::get().to(get_version)))
         .service(web::scope("/indexes").configure(indexes::configure))
@@ -43,6 +41,34 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
         .service(web::scope("/experimental-features").configure(features::configure));
 }
 
+pub fn get_task_id(req: &HttpRequest) -> Result<Option<TaskId>, ResponseError> {
+    let task_id = req
+        .headers()
+        .get("TaskId")
+        .map(|header| {
+            header.to_str().map_err(|e| {
+                ResponseError::from_msg(
+                    format!("TaskId is not a valid utf-8 string: {e}"),
+                    Code::BadRequest,
+                )
+            })
+        })
+        .transpose()?
+        .map(|s| {
+            s.parse::<TaskId>().map_err(|e| {
+                ResponseError::from_msg(
+                    format!(
+                        "Could not parse the TaskId as a {}: {e}",
+                        std::any::type_name::<TaskId>(),
+                    ),
+                    Code::BadRequest,
+                )
+            })
+        })
+        .transpose()?;
+    Ok(task_id)
+}
+
 #[derive(Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct SummarizedTaskView {

meilisearch/src/routes/snapshot.rs

@@ -1,32 +0,0 @@
-use actix_web::web::Data;
-use actix_web::{web, HttpRequest, HttpResponse};
-use index_scheduler::IndexScheduler;
-use log::debug;
-use meilisearch_types::error::ResponseError;
-use meilisearch_types::tasks::KindWithContent;
-use serde_json::json;
-
-use crate::analytics::Analytics;
-use crate::extractors::authentication::policies::*;
-use crate::extractors::authentication::GuardedData;
-use crate::extractors::sequential_extractor::SeqHandler;
-use crate::routes::SummarizedTaskView;
-
-pub fn configure(cfg: &mut web::ServiceConfig) {
-    cfg.service(web::resource("").route(web::post().to(SeqHandler(create_snapshot))));
-}
-
-pub async fn create_snapshot(
-    index_scheduler: GuardedData<ActionPolicy<{ actions::SNAPSHOTS_CREATE }>, Data<IndexScheduler>>,
-    req: HttpRequest,
-    analytics: web::Data<dyn Analytics>,
-) -> Result<HttpResponse, ResponseError> {
-    analytics.publish("Snapshot Created".to_string(), json!({}), Some(&req));
-
-    let task = KindWithContent::SnapshotCreation;
-    let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
-
-    debug!("returns: {:?}", task);
-    Ok(HttpResponse::Accepted().json(task))
-}

meilisearch/src/routes/swap_indexes.rs

@@ -10,7 +10,7 @@ use meilisearch_types::index_uid::IndexUid;
 use meilisearch_types::tasks::{IndexSwap, KindWithContent};
 use serde_json::json;
 
-use super::SummarizedTaskView;
+use super::{get_task_id, SummarizedTaskView};
 use crate::analytics::Analytics;
 use crate::error::MeilisearchHttpError;
 use crate::extractors::authentication::policies::*;
@@ -60,7 +60,9 @@ pub async fn swap_indexes(
     }
 
     let task = KindWithContent::IndexSwap { swaps };
+
+    let uid = get_task_id(&req)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid)).await??.into();
     Ok(HttpResponse::Accepted().json(task))
 }

meilisearch/src/routes/tasks.rs

@@ -20,13 +20,13 @@ use time::macros::format_description;
 use time::{Date, Duration, OffsetDateTime, Time};
 use tokio::task;
 
-use super::SummarizedTaskView;
+use super::{get_task_id, SummarizedTaskView};
 use crate::analytics::Analytics;
 use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::GuardedData;
 use crate::extractors::sequential_extractor::SeqHandler;
 
-const DEFAULT_LIMIT: u32 = 20;
+const DEFAULT_LIMIT: u64 = 20;
 
 pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(
@@ -175,14 +175,14 @@ impl From<Details> for DetailsView {
 #[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)]
 pub struct TasksFilterQuery {
     #[deserr(default = Param(DEFAULT_LIMIT), error = DeserrQueryParamError<InvalidTaskLimit>)]
-    pub limit: Param<u32>,
+    pub limit: Param<TaskId>,
     #[deserr(default, error = DeserrQueryParamError<InvalidTaskFrom>)]
     pub from: Option<Param<TaskId>>,
 
     #[deserr(default, error = DeserrQueryParamError<InvalidTaskUids>)]
-    pub uids: OptionStarOrList<u32>,
+    pub uids: OptionStarOrList<TaskId>,
     #[deserr(default, error = DeserrQueryParamError<InvalidTaskCanceledBy>)]
-    pub canceled_by: OptionStarOrList<u32>,
+    pub canceled_by: OptionStarOrList<TaskId>,
     #[deserr(default, error = DeserrQueryParamError<InvalidTaskTypes>)]
     pub types: OptionStarOrList<Kind>,
     #[deserr(default, error = DeserrQueryParamError<InvalidTaskStatuses>)]
@@ -249,9 +249,9 @@ impl TaskDeletionOrCancelationQuery {
 #[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)]
 pub struct TaskDeletionOrCancelationQuery {
     #[deserr(default, error = DeserrQueryParamError<InvalidTaskUids>)]
-    pub uids: OptionStarOrList<u32>,
+    pub uids: OptionStarOrList<TaskId>,
     #[deserr(default, error = DeserrQueryParamError<InvalidTaskCanceledBy>)]
-    pub canceled_by: OptionStarOrList<u32>,
+    pub canceled_by: OptionStarOrList<TaskId>,
     #[deserr(default, error = DeserrQueryParamError<InvalidTaskTypes>)]
     pub types: OptionStarOrList<Kind>,
     #[deserr(default, error = DeserrQueryParamError<InvalidTaskStatuses>)]
@@ -333,7 +333,9 @@ async fn cancel_tasks(
     let task_cancelation =
         KindWithContent::TaskCancelation { query: format!("?{}", req.query_string()), tasks };
 
-    let task = task::spawn_blocking(move || index_scheduler.register(task_cancelation)).await??;
+    let uid = get_task_id(&req)?;
+    let task =
+        task::spawn_blocking(move || index_scheduler.register(task_cancelation, uid)).await??;
     let task: SummarizedTaskView = task.into();
 
     Ok(HttpResponse::Ok().json(task))
@@ -378,7 +380,8 @@ async fn delete_tasks(
     let task_deletion =
         KindWithContent::TaskDeletion { query: format!("?{}", req.query_string()), tasks };
 
-    let task = task::spawn_blocking(move || index_scheduler.register(task_deletion)).await??;
+    let uid = get_task_id(&req)?;
+    let task = task::spawn_blocking(move || index_scheduler.register(task_deletion, uid)).await??;
     let task: SummarizedTaskView = task.into();
 
     Ok(HttpResponse::Ok().json(task))
@@ -388,9 +391,9 @@ async fn delete_tasks(
 pub struct AllTasks {
     results: Vec<TaskView>,
     total: u64,
-    limit: u32,
-    from: Option<u32>,
-    next: Option<u32>,
+    limit: TaskId,
+    from: Option<TaskId>,
+    next: Option<TaskId>,
 }
 
 async fn get_tasks(

meilisearch/tests/auth/api_keys.rs

@@ -1,7 +1,8 @@
 use std::{thread, time};
 
-use crate::common::{Server, Value};
-use crate::json;
+use serde_json::{json, Value};
+
+use crate::common::Server;
 
 #[actix_rt::test]
 async fn add_valid_api_key() {
@@ -161,7 +162,7 @@ async fn add_valid_api_key_null_description() {
     server.use_api_key("MASTER_KEY");
 
     let content = json!({
-        "description": json!(null),
+        "description": Value::Null,
         "indexes": ["products"],
         "actions": ["documents.add"],
         "expiresAt": "2050-11-13T00:00:00"
@@ -364,7 +365,7 @@ async fn error_add_api_key_invalid_index_uids() {
     server.use_api_key("MASTER_KEY");
 
     let content = json!({
-        "description": json!(null),
+        "description": Value::Null,
         "indexes": ["invalid index # / \\name with spaces"],
         "actions": [
             "documents.add"
@@ -421,7 +422,7 @@ async fn error_add_api_key_invalid_parameters_actions() {
     meili_snap::snapshot!(code, @"400 Bad Request");
     meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]" }), @r###"
     {
-      "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`",
+      "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`",
       "code": "invalid_api_key_actions",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions"
@@ -506,7 +507,7 @@ async fn error_add_api_key_invalid_parameters_uid() {
 async fn error_add_api_key_parameters_uid_already_exist() {
     let mut server = Server::new_auth().await;
     server.use_api_key("MASTER_KEY");
-    let content: Value = json!({
+    let content = json!({
         "uid": "4bc0887a-0e41-4f3b-935d-0c451dcee9c8",
         "indexes": ["products"],
         "actions": ["search"],
@@ -1145,7 +1146,7 @@ async fn patch_api_key_description() {
     meili_snap::snapshot!(code, @"200 OK");
 
     // Remove the description
-    let content = json!({ "description": null });
+    let content = json!({ "description": serde_json::Value::Null });
 
     let (response, code) = server.patch_api_key(&uid, content).await;
     meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]", ".uid" => "[ignored]", ".key" => "[ignored]" }), @r###"

meilisearch/tests/auth/authorization.rs

@@ -3,10 +3,10 @@ use std::collections::{HashMap, HashSet};
 use ::time::format_description::well_known::Rfc3339;
 use maplit::{hashmap, hashset};
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 use time::{Duration, OffsetDateTime};
 
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 pub static AUTHORIZATIONS: Lazy<HashMap<(&'static str, &'static str), HashSet<&'static str>>> =
     Lazy::new(|| {
@@ -54,7 +54,6 @@ pub static AUTHORIZATIONS: Lazy<HashMap<(&'static str, &'static str), HashSet<&'
             ("GET",     "/indexes/products/stats") =>                          hashset!{"stats.get", "stats.*", "*"},
             ("GET",     "/stats") =>                                           hashset!{"stats.get", "stats.*", "*"},
             ("POST",    "/dumps") =>                                           hashset!{"dumps.create", "dumps.*", "*"},
-            ("POST",    "/snapshots") =>                                       hashset!{"snapshots.create", "snapshots.*", "*"},
             ("GET",     "/version") =>                                         hashset!{"version", "*"},
             ("GET",     "/metrics") =>                                         hashset!{"metrics.get", "metrics.*", "*"},
             ("PATCH",   "/keys/mykey/") =>                                     hashset!{"keys.update", "*"},

meilisearch/tests/auth/errors.rs

@@ -1,8 +1,8 @@
 use meili_snap::*;
+use serde_json::json;
 use uuid::Uuid;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn create_api_key_bad_description() {
@@ -90,7 +90,7 @@ async fn create_api_key_bad_actions() {
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`",
+      "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`",
       "code": "invalid_api_key_actions",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions"

meilisearch/tests/auth/mod.rs

@@ -7,9 +7,9 @@ mod tenant_token;
 mod tenant_token_multi_search;
 
 use actix_web::http::StatusCode;
+use serde_json::{json, Value};
 
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 impl Server {
     pub fn use_api_key(&mut self, api_key: impl AsRef<str>) {

meilisearch/tests/auth/tenant_token.rs

@@ -3,11 +3,11 @@ use std::collections::HashMap;
 use ::time::format_description::well_known::Rfc3339;
 use maplit::hashmap;
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 use time::{Duration, OffsetDateTime};
 
 use super::authorization::{ALL_ACTIONS, AUTHORIZATIONS};
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 fn generate_tenant_token(
     parent_uid: impl AsRef<str>,
@@ -233,31 +233,31 @@ async fn search_authorized_simple_token() {
         },
         hashmap! {
             "searchRules" => json!({"*": {}}),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
-            "exp" => json!(null)
+            "searchRules" => json!({"*": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["*"]),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!({"sales": {}}),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
-            "exp" => json!(null)
+            "searchRules" => json!({"sales": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sales"]),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sa*"]),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
     ];
 
@@ -386,7 +386,7 @@ async fn error_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
+            "searchRules" => json!({"*": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -398,7 +398,7 @@ async fn error_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
+            "searchRules" => json!({"sales": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -428,15 +428,15 @@ async fn error_search_forbidden_token() {
         },
         hashmap! {
             "searchRules" => json!({"products": {}}),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"products": null}),
-            "exp" => json!(null)
+            "searchRules" => json!({"products": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["products"]),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
         // expired token
         hashmap! {
@@ -444,7 +444,7 @@ async fn error_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
+            "searchRules" => json!({"*": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -456,7 +456,7 @@ async fn error_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
+            "searchRules" => json!({"sales": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {

meilisearch/tests/auth/tenant_token_multi_search.rs

@@ -3,11 +3,11 @@ use std::collections::HashMap;
 use ::time::format_description::well_known::Rfc3339;
 use maplit::hashmap;
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 use time::{Duration, OffsetDateTime};
 
 use super::authorization::ALL_ACTIONS;
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 fn generate_tenant_token(
     parent_uid: impl AsRef<str>,
@@ -512,31 +512,31 @@ async fn single_search_authorized_simple_token() {
         },
         hashmap! {
             "searchRules" => json!({"*": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"*": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["*"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!({"sales": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"sales": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sales"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sa*"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
     ];
 
@@ -564,31 +564,31 @@ async fn multi_search_authorized_simple_token() {
         },
         hashmap! {
             "searchRules" => json!({"*": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"*": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["*"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!({"sales": {}, "products": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"sales": null, "products": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"sales": Value::Null, "products": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sales", "products"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sa*", "pro*"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
     ];
 
@@ -823,7 +823,7 @@ async fn error_single_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
+            "searchRules" => json!({"*": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -835,7 +835,7 @@ async fn error_single_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
+            "searchRules" => json!({"sales": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -864,7 +864,7 @@ async fn error_multi_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
+            "searchRules" => json!({"*": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -876,7 +876,7 @@ async fn error_multi_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": null, "products": null}),
+            "searchRules" => json!({"sales": Value::Null, "products": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -919,15 +919,15 @@ async fn error_single_search_forbidden_token() {
         },
         hashmap! {
             "searchRules" => json!({"products": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"products": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"products": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["products"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         // expired token
         hashmap! {
@@ -935,7 +935,7 @@ async fn error_single_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
+            "searchRules" => json!({"*": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -947,7 +947,7 @@ async fn error_single_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
+            "searchRules" => json!({"sales": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -978,15 +978,15 @@ async fn error_multi_search_forbidden_token() {
         },
         hashmap! {
             "searchRules" => json!({"products": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"products": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"products": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["products"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!({"sales": {}}),
@@ -998,15 +998,15 @@ async fn error_multi_search_forbidden_token() {
         },
         hashmap! {
             "searchRules" => json!({"sales": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"sales": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sales"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         // expired token
         hashmap! {
@@ -1014,7 +1014,7 @@ async fn error_multi_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
+            "searchRules" => json!({"*": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -1026,7 +1026,7 @@ async fn error_multi_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": null, "products": {}}),
+            "searchRules" => json!({"sales": Value::Null, "products": {}}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {

meilisearch/tests/common/index.rs

@@ -3,13 +3,12 @@ use std::panic::{catch_unwind, resume_unwind, UnwindSafe};
 use std::time::Duration;
 
 use actix_web::http::StatusCode;
+use serde_json::{json, Value};
 use tokio::time::sleep;
 use urlencoding::encode as urlencode;
 
 use super::encoder::Encoder;
 use super::service::Service;
-use super::Value;
-use crate::json;
 
 pub struct Index<'a> {
     pub uid: String,
@@ -243,9 +242,7 @@ impl Index<'_> {
 
     pub async fn delete_batch(&self, ids: Vec<u64>) -> (Value, StatusCode) {
         let url = format!("/indexes/{}/documents/delete-batch", urlencode(self.uid.as_ref()));
-        self.service
-            .post_encoded(url, serde_json::to_value(&ids).unwrap().into(), self.encoder)
-            .await
+        self.service.post_encoded(url, serde_json::to_value(&ids).unwrap(), self.encoder).await
     }
 
     pub async fn delete_batch_raw(&self, body: Value) -> (Value, StatusCode) {

meilisearch/tests/common/mod.rs

@@ -3,83 +3,9 @@ pub mod index;
 pub mod server;
 pub mod service;
 
-use std::fmt::{self, Display};
-
 pub use index::{GetAllDocumentsOptions, GetDocumentOptions};
-use meili_snap::json_string;
-use serde::{Deserialize, Serialize};
 pub use server::{default_settings, Server};
 
-#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
-pub struct Value(pub serde_json::Value);
-
-impl Value {
-    pub fn uid(&self) -> u64 {
-        if let Some(uid) = self["uid"].as_u64() {
-            uid
-        } else if let Some(uid) = self["taskUid"].as_u64() {
-            uid
-        } else {
-            panic!("Didn't find any task id in: {self}");
-        }
-    }
-}
-
-impl From<serde_json::Value> for Value {
-    fn from(value: serde_json::Value) -> Self {
-        Value(value)
-    }
-}
-
-impl std::ops::Deref for Value {
-    type Target = serde_json::Value;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl PartialEq<serde_json::Value> for Value {
-    fn eq(&self, other: &serde_json::Value) -> bool {
-        &self.0 == other
-    }
-}
-
-impl PartialEq<Value> for serde_json::Value {
-    fn eq(&self, other: &Value) -> bool {
-        self == &other.0
-    }
-}
-
-impl PartialEq<&str> for Value {
-    fn eq(&self, other: &&str) -> bool {
-        self.0.eq(other)
-    }
-}
-
-impl Display for Value {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(
-            f,
-            "{}",
-            json_string!(self, { ".enqueuedAt" => "[date]", ".processedAt" => "[date]", ".finishedAt" => "[date]", ".duration" => "[duration]" })
-        )
-    }
-}
-
-impl From<Vec<Value>> for Value {
-    fn from(value: Vec<Value>) -> Self {
-        Self(value.into_iter().map(|value| value.0).collect::<serde_json::Value>())
-    }
-}
-
-#[macro_export]
-macro_rules! json {
-    ($($json:tt)+) => {
-        $crate::common::Value(serde_json::json!($($json)+))
-    };
-}
-
 /// Performs a search test on both post and get routes
 #[macro_export]
 macro_rules! test_post_get_search {

meilisearch/tests/common/server.rs

@@ -11,14 +11,13 @@ use clap::Parser;
 use meilisearch::option::{IndexerOpts, MaxMemory, Opt};
 use meilisearch::{analytics, create_app, setup_meilisearch};
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 use tempfile::TempDir;
 use tokio::time::sleep;
 
 use super::index::Index;
 use super::service::Service;
 use crate::common::encoder::Encoder;
-use crate::common::Value;
-use crate::json;
 
 pub struct Server {
     pub service: Service,
@@ -157,10 +156,6 @@ impl Server {
         self.service.post("/dumps", json!(null)).await
     }
 
-    pub async fn create_snapshot(&self) -> (Value, StatusCode) {
-        self.service.post("/snapshots", json!(null)).await
-    }
-
     pub async fn index_swap(&self, value: Value) -> (Value, StatusCode) {
         self.service.post("/swap-indexes", value).await
     }
@@ -209,7 +204,7 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
         db_path: dir.as_ref().join("db"),
         dump_dir: dir.as_ref().join("dumps"),
         env: "development".to_owned(),
-        #[cfg(feature = "analytics")]
+        #[cfg(all(not(debug_assertions), feature = "analytics"))]
         no_analytics: true,
         max_index_size: Byte::from_unit(100.0, ByteUnit::MiB).unwrap(),
         max_task_db_size: Byte::from_unit(1.0, ByteUnit::GiB).unwrap(),

meilisearch/tests/common/service.rs

@@ -7,9 +7,9 @@ use actix_web::test::TestRequest;
 use index_scheduler::IndexScheduler;
 use meilisearch::{analytics, create_app, Opt};
 use meilisearch_auth::AuthController;
+use serde_json::Value;
 
 use crate::common::encoder::Encoder;
-use crate::common::Value;
 
 pub struct Service {
     pub index_scheduler: Arc<IndexScheduler>,

meilisearch/tests/content_type.rs

@@ -3,8 +3,9 @@
 mod common;
 
 use actix_web::test;
+use serde_json::{json, Value};
 
-use crate::common::{Server, Value};
+use crate::common::Server;
 
 enum HttpVerb {
     Put,

meilisearch/tests/documents/add_documents.rs

@@ -1,11 +1,11 @@
 use actix_web::test;
 use meili_snap::{json_string, snapshot};
+use serde_json::{json, Value};
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
 use crate::common::encoder::Encoder;
-use crate::common::{GetAllDocumentsOptions, Server, Value};
-use crate::json;
+use crate::common::{GetAllDocumentsOptions, Server};
 
 /// This is the basic usage of our API and every other tests uses the content-type application/json
 #[actix_rt::test]

meilisearch/tests/documents/delete_documents.rs

@@ -1,7 +1,7 @@
 use meili_snap::{json_string, snapshot};
+use serde_json::json;
 
 use crate::common::{GetAllDocumentsOptions, Server};
-use crate::json;
 
 #[actix_rt::test]
 async fn delete_one_document_unexisting_index() {
@@ -154,19 +154,6 @@ async fn delete_document_by_filter() {
         )
         .await;
     index.wait_task(1).await;
-
-    let (stats, _) = index.stats().await;
-    snapshot!(json_string!(stats), @r###"
-    {
-      "numberOfDocuments": 4,
-      "isIndexing": false,
-      "fieldDistribution": {
-        "color": 3,
-        "id": 4
-      }
-    }
-    "###);
-
     let (response, code) =
         index.delete_document_by_filter(json!({ "filter": "color = blue"})).await;
     snapshot!(code, @"202 Accepted");
@@ -201,18 +188,6 @@ async fn delete_document_by_filter() {
     }
     "###);
 
-    let (stats, _) = index.stats().await;
-    snapshot!(json_string!(stats), @r###"
-    {
-      "numberOfDocuments": 2,
-      "isIndexing": false,
-      "fieldDistribution": {
-        "color": 1,
-        "id": 2
-      }
-    }
-    "###);
-
     let (documents, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await;
     snapshot!(code, @"200 OK");
     snapshot!(json_string!(documents), @r###"
@@ -266,18 +241,6 @@ async fn delete_document_by_filter() {
     }
     "###);
 
-    let (stats, _) = index.stats().await;
-    snapshot!(json_string!(stats), @r###"
-    {
-      "numberOfDocuments": 1,
-      "isIndexing": false,
-      "fieldDistribution": {
-        "color": 1,
-        "id": 1
-      }
-    }
-    "###);
-
     let (documents, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await;
     snapshot!(code, @"200 OK");
     snapshot!(json_string!(documents), @r###"

meilisearch/tests/documents/errors.rs

@@ -1,8 +1,8 @@
 use meili_snap::*;
+use serde_json::json;
 use urlencoding::encode;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn get_all_documents_bad_offset() {

meilisearch/tests/documents/get_documents.rs

@@ -1,11 +1,11 @@
 use actix_web::test;
 use http::header::ACCEPT_ENCODING;
 use meili_snap::*;
+use serde_json::{json, Value};
 use urlencoding::encode as urlencode;
 
 use crate::common::encoder::Encoder;
-use crate::common::{GetAllDocumentsOptions, GetDocumentOptions, Server, Value};
-use crate::json;
+use crate::common::{GetAllDocumentsOptions, GetDocumentOptions, Server};
 
 // TODO: partial test since we are testing error, amd error is not yet fully implemented in
 // transplant
@@ -40,7 +40,7 @@ async fn get_document() {
     let server = Server::new().await;
     let index = server.index("test");
     index.create(None).await;
-    let documents = json!([
+    let documents = serde_json::json!([
         {
             "id": 0,
             "nested": { "content": "foobar" },
@@ -53,7 +53,7 @@ async fn get_document() {
     assert_eq!(code, 200);
     assert_eq!(
         response,
-        json!({
+        serde_json::json!({
             "id": 0,
             "nested": { "content": "foobar" },
         })
@@ -64,7 +64,7 @@ async fn get_document() {
     assert_eq!(code, 200);
     assert_eq!(
         response,
-        json!({
+        serde_json::json!({
             "id": 0,
         })
     );
@@ -75,7 +75,7 @@ async fn get_document() {
     assert_eq!(code, 200);
     assert_eq!(
         response,
-        json!({
+        serde_json::json!({
             "nested": { "content": "foobar" },
         })
     );
@@ -122,7 +122,7 @@ async fn get_all_documents_no_options() {
     assert_eq!(code, 200);
     let arr = response["results"].as_array().unwrap();
     assert_eq!(arr.len(), 20);
-    let first = json!({
+    let first = serde_json::json!({
         "id":0,
         "isActive":false,
         "balance":"$2,668.55",

meilisearch/tests/documents/update_documents.rs

@@ -1,8 +1,7 @@
-use meili_snap::snapshot;
+use serde_json::json;
 
 use crate::common::encoder::Encoder;
 use crate::common::{GetAllDocumentsOptions, Server};
-use crate::json;
 
 #[actix_rt::test]
 async fn error_document_update_create_index_bad_uid() {
@@ -85,13 +84,7 @@ async fn update_document() {
 
     let (response, code) = index.get_document(1, None).await;
     assert_eq!(code, 200);
-    snapshot!(response, @r###"
-    {
-      "doc_id": 1,
-      "content": "foo",
-      "other": "bar"
-    }
-    "###);
+    assert_eq!(response.to_string(), r##"{"doc_id":1,"content":"foo","other":"bar"}"##);
 }
 
 #[actix_rt::test]
@@ -129,13 +122,7 @@ async fn update_document_gzip_encoded() {
 
     let (response, code) = index.get_document(1, None).await;
     assert_eq!(code, 200);
-    snapshot!(response, @r###"
-    {
-      "doc_id": 1,
-      "content": "foo",
-      "other": "bar"
-    }
-    "###);
+    assert_eq!(response.to_string(), r##"{"doc_id":1,"content":"foo","other":"bar"}"##);
 }
 
 #[actix_rt::test]

meilisearch/tests/dumps/mod.rs

@@ -2,10 +2,10 @@ mod data;
 
 use meili_snap::{json_string, snapshot};
 use meilisearch::Opt;
+use serde_json::json;
 
 use self::data::GetDump;
 use crate::common::{default_settings, GetAllDocumentsOptions, Server};
-use crate::json;
 
 // all the following test are ignored on windows. See #2364
 #[actix_rt::test]

meilisearch/tests/features/mod.rs

@@ -1,5 +1,6 @@
+use serde_json::json;
+
 use crate::common::Server;
-use crate::json;
 
 /// Feature name to test against.
 /// This will have to be changed by a different one when that feature is stabilized.

meilisearch/tests/index/create_index.rs

@@ -2,10 +2,10 @@ use actix_web::http::header::ContentType;
 use actix_web::test;
 use http::header::ACCEPT_ENCODING;
 use meili_snap::{json_string, snapshot};
+use serde_json::{json, Value};
 
 use crate::common::encoder::Encoder;
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 #[actix_rt::test]
 async fn create_index_no_primary_key() {
@@ -21,7 +21,7 @@ async fn create_index_no_primary_key() {
 
     assert_eq!(response["status"], "succeeded");
     assert_eq!(response["type"], "indexCreation");
-    assert_eq!(response["details"]["primaryKey"], json!(null));
+    assert_eq!(response["details"]["primaryKey"], Value::Null);
 }
 
 #[actix_rt::test]
@@ -38,7 +38,7 @@ async fn create_index_with_gzip_encoded_request() {
 
     assert_eq!(response["status"], "succeeded");
     assert_eq!(response["type"], "indexCreation");
-    assert_eq!(response["details"]["primaryKey"], json!(null));
+    assert_eq!(response["details"]["primaryKey"], Value::Null);
 }
 
 #[actix_rt::test]
@@ -86,7 +86,7 @@ async fn create_index_with_zlib_encoded_request() {
 
     assert_eq!(response["status"], "succeeded");
     assert_eq!(response["type"], "indexCreation");
-    assert_eq!(response["details"]["primaryKey"], json!(null));
+    assert_eq!(response["details"]["primaryKey"], Value::Null);
 }
 
 #[actix_rt::test]
@@ -103,7 +103,7 @@ async fn create_index_with_brotli_encoded_request() {
 
     assert_eq!(response["status"], "succeeded");
     assert_eq!(response["type"], "indexCreation");
-    assert_eq!(response["details"]["primaryKey"], json!(null));
+    assert_eq!(response["details"]["primaryKey"], Value::Null);
 }
 
 #[actix_rt::test]
@@ -136,7 +136,7 @@ async fn create_index_with_invalid_primary_key() {
 
     let (response, code) = index.get().await;
     assert_eq!(code, 200);
-    assert_eq!(response["primaryKey"], json!(null));
+    assert_eq!(response["primaryKey"], Value::Null);
 }
 
 #[actix_rt::test]
@@ -199,3 +199,74 @@ async fn error_create_with_invalid_index_uid() {
     }
     "###);
 }
+
+#[actix_rt::test]
+async fn send_task_id() {
+    let server = Server::new().await;
+    let app = server.init_web_app().await;
+    let index = server.index("catto");
+    let (response, code) = index.create(None).await;
+    snapshot!(code, @"202 Accepted");
+    snapshot!(json_string!(response, { ".enqueuedAt" => "[date]" }), @r###"
+    {
+      "taskUid": 0,
+      "indexUid": "catto",
+      "status": "enqueued",
+      "type": "indexCreation",
+      "enqueuedAt": "[date]"
+    }
+    "###);
+
+    let body = serde_json::to_string(&json!({
+        "uid": "doggo",
+        "primaryKey": None::<&str>,
+    }))
+    .unwrap();
+    let req = test::TestRequest::post()
+        .uri("/indexes")
+        .insert_header(("TaskId", "25"))
+        .insert_header(ContentType::json())
+        .set_payload(body)
+        .to_request();
+
+    let res = test::call_service(&app, req).await;
+    snapshot!(res.status(), @"202 Accepted");
+
+    let bytes = test::read_body(res).await;
+    let response = serde_json::from_slice::<Value>(&bytes).expect("Expecting valid json");
+    snapshot!(json_string!(response, { ".enqueuedAt" => "[date]" }), @r###"
+    {
+      "taskUid": 25,
+      "indexUid": "doggo",
+      "status": "enqueued",
+      "type": "indexCreation",
+      "enqueuedAt": "[date]"
+    }
+    "###);
+
+    let body = serde_json::to_string(&json!({
+        "uid": "girafo",
+        "primaryKey": None::<&str>,
+    }))
+    .unwrap();
+    let req = test::TestRequest::post()
+        .uri("/indexes")
+        .insert_header(("TaskId", "12"))
+        .insert_header(ContentType::json())
+        .set_payload(body)
+        .to_request();
+
+    let res = test::call_service(&app, req).await;
+    snapshot!(res.status(), @"400 Bad Request");
+
+    let bytes = test::read_body(res).await;
+    let response = serde_json::from_slice::<Value>(&bytes).expect("Expecting valid json");
+    snapshot!(json_string!(response), @r###"
+    {
+      "message": "Received bad task id: 12 should be >= to 26.",
+      "code": "bad_request",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#bad_request"
+    }
+    "###);
+}

meilisearch/tests/index/delete_index.rs

@@ -1,5 +1,6 @@
+use serde_json::json;
+
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn create_and_delete_index() {

meilisearch/tests/index/errors.rs

@@ -1,7 +1,7 @@
 use meili_snap::*;
+use serde_json::json;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn get_indexes_bad_offset() {

meilisearch/tests/index/stats.rs

@@ -1,5 +1,6 @@
+use serde_json::json;
+
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn stats() {

meilisearch/tests/index/update_index.rs

@@ -1,9 +1,9 @@
+use serde_json::json;
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
 use crate::common::encoder::Encoder;
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn update_primary_key() {

meilisearch/tests/search/errors.rs

@@ -1,8 +1,8 @@
 use meili_snap::*;
+use serde_json::json;
 
 use super::DOCUMENTS;
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn search_unexisting_index() {

meilisearch/tests/search/facet_search.rs

@@ -1,8 +1,8 @@
 use meili_snap::snapshot;
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 pub(self) static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
     json!([

meilisearch/tests/search/formatted.rs

@@ -1,8 +1,8 @@
 use insta::{allow_duplicates, assert_json_snapshot};
+use serde_json::json;
 
 use super::*;
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn formatted_contain_wildcard() {

meilisearch/tests/search/geo.rs

@@ -1,8 +1,8 @@
 use meili_snap::{json_string, snapshot};
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 pub(self) static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
     json!([

meilisearch/tests/search/mod.rs

@@ -10,9 +10,9 @@ mod pagination;
 mod restrict_searchable;
 
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 pub(self) static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
     json!([
@@ -1104,59 +1104,3 @@ async fn camelcased_words() {
         })
         .await;
 }
-
-#[actix_rt::test]
-async fn simple_search_with_strange_synonyms() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.update_settings(json!({ "synonyms": {"&": ["to"], "to": ["&"]} })).await;
-    let r = index.wait_task(0).await;
-    meili_snap::snapshot!(r["status"], @r###""succeeded""###);
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
-    index
-        .search(json!({"q": "How to train"}), |response, code| {
-            meili_snap::snapshot!(code, @"200 OK");
-            meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###"
-            [
-              {
-                "title": "How to Train Your Dragon: The Hidden World",
-                "id": "166428"
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "How & train"}), |response, code| {
-            meili_snap::snapshot!(code, @"200 OK");
-            meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###"
-            [
-              {
-                "title": "How to Train Your Dragon: The Hidden World",
-                "id": "166428"
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "to"}), |response, code| {
-            meili_snap::snapshot!(code, @"200 OK");
-            meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###"
-            [
-              {
-                "title": "How to Train Your Dragon: The Hidden World",
-                "id": "166428"
-              }
-            ]
-            "###);
-        })
-        .await;
-}

meilisearch/tests/search/multi.rs

@@ -1,8 +1,8 @@
 use meili_snap::{json_string, snapshot};
+use serde_json::json;
 
 use super::{DOCUMENTS, NESTED_DOCUMENTS};
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn search_empty_list() {

meilisearch/tests/search/pagination.rs

@@ -1,5 +1,6 @@
+use serde_json::json;
+
 use crate::common::Server;
-use crate::json;
 use crate::search::DOCUMENTS;
 
 #[actix_rt::test]

meilisearch/tests/search/restrict_searchable.rs

@@ -1,9 +1,9 @@
 use meili_snap::{json_string, snapshot};
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 
 use crate::common::index::Index;
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 async fn index_with_documents<'a>(server: &'a Server, documents: &Value) -> Index<'a> {
     let index = server.index("test");

meilisearch/tests/settings/distinct.rs

@@ -1,5 +1,6 @@
+use serde_json::json;
+
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn set_and_reset_distinct_attribute() {

meilisearch/tests/settings/errors.rs

@@ -1,7 +1,7 @@
 use meili_snap::*;
+use serde_json::json;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn settings_bad_displayed_attributes() {

meilisearch/tests/settings/get_settings.rs

@@ -1,16 +1,16 @@
 use std::collections::HashMap;
 
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 static DEFAULT_SETTINGS_VALUES: Lazy<HashMap<&'static str, Value>> = Lazy::new(|| {
     let mut map = HashMap::new();
     map.insert("displayed_attributes", json!(["*"]));
     map.insert("searchable_attributes", json!(["*"]));
     map.insert("filterable_attributes", json!([]));
-    map.insert("distinct_attribute", json!(null));
+    map.insert("distinct_attribute", json!(Value::Null));
     map.insert(
         "ranking_rules",
         json!(["words", "typo", "proximity", "attribute", "sort", "exactness"]),
@@ -229,7 +229,7 @@ macro_rules! test_setting_routes {
                         .chars()
                         .map(|c| if c == '_' { '-' } else { c })
                         .collect::<String>());
-                    let (response, code) = server.service.$write_method(url, serde_json::Value::Null.into()).await;
+                    let (response, code) = server.service.$write_method(url, serde_json::Value::Null).await;
                     assert_eq!(code, 202, "{}", response);
                     server.index("").wait_task(0).await;
                     let (response, code) = server.index("test").get().await;

meilisearch/tests/settings/tokenizer_customization.rs

@@ -1,7 +1,7 @@
 use meili_snap::{json_string, snapshot};
+use serde_json::json;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn set_and_reset() {

meilisearch/tests/snapshot/mod.rs

@@ -1,13 +1,11 @@
 use std::time::Duration;
 
 use actix_rt::time::sleep;
-use meili_snap::{json_string, snapshot};
 use meilisearch::option::ScheduleSnapshot;
 use meilisearch::Opt;
 
 use crate::common::server::default_settings;
 use crate::common::{GetAllDocumentsOptions, Server};
-use crate::json;
 
 macro_rules! verify_snapshot {
     (
@@ -46,7 +44,7 @@ async fn perform_snapshot() {
 
     let index = server.index("test");
     index
-        .update_settings(json! ({
+        .update_settings(serde_json::json! ({
         "searchableAttributes": [],
         }))
         .await;
@@ -92,95 +90,3 @@ async fn perform_snapshot() {
         server.index("test1").settings(),
     );
 }
-
-#[actix_rt::test]
-async fn perform_on_demand_snapshot() {
-    let temp = tempfile::tempdir().unwrap();
-    let snapshot_dir = tempfile::tempdir().unwrap();
-
-    let options =
-        Opt { snapshot_dir: snapshot_dir.path().to_owned(), ..default_settings(temp.path()) };
-
-    let server = Server::new_with_options(options).await.unwrap();
-
-    let index = server.index("catto");
-    index
-        .update_settings(json! ({
-        "searchableAttributes": [],
-        }))
-        .await;
-
-    index.load_test_set().await;
-
-    server.index("doggo").create(Some("bone")).await;
-    index.wait_task(2).await;
-
-    server.index("doggo").create(Some("bone")).await;
-    index.wait_task(2).await;
-
-    let (task, code) = server.create_snapshot().await;
-    snapshot!(code, @"202 Accepted");
-    snapshot!(json_string!(task, { ".enqueuedAt" => "[date]" }), @r###"
-    {
-      "taskUid": 4,
-      "indexUid": null,
-      "status": "enqueued",
-      "type": "snapshotCreation",
-      "enqueuedAt": "[date]"
-    }
-    "###);
-    let task = index.wait_task(task.uid()).await;
-    snapshot!(json_string!(task, { ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".duration" => "[duration]" }), @r###"
-    {
-      "uid": 4,
-      "indexUid": null,
-      "status": "succeeded",
-      "type": "snapshotCreation",
-      "canceledBy": null,
-      "error": null,
-      "duration": "[duration]",
-      "enqueuedAt": "[date]",
-      "startedAt": "[date]",
-      "finishedAt": "[date]"
-    }
-    "###);
-
-    let temp = tempfile::tempdir().unwrap();
-
-    let snapshots: Vec<String> = std::fs::read_dir(&snapshot_dir)
-        .unwrap()
-        .map(|entry| entry.unwrap().path().file_name().unwrap().to_str().unwrap().to_string())
-        .collect();
-    meili_snap::snapshot!(format!("{snapshots:?}"), @r###"["db.snapshot"]"###);
-
-    let snapshot_path = snapshot_dir.path().to_owned().join("db.snapshot");
-    #[cfg_attr(windows, allow(unused))]
-    let snapshot_meta = std::fs::metadata(&snapshot_path).unwrap();
-
-    #[cfg(unix)]
-    {
-        use std::os::unix::fs::PermissionsExt;
-        let mode = snapshot_meta.permissions().mode();
-        //                                                    rwxrwxrwx
-        meili_snap::snapshot!(format!("{:b}", mode), @"1000000100100100");
-    }
-
-    let options = Opt { import_snapshot: Some(snapshot_path), ..default_settings(temp.path()) };
-
-    let snapshot_server = Server::new_with_options(options).await.unwrap();
-
-    verify_snapshot!(server, snapshot_server, |server| =>
-        server.list_indexes(None, None),
-        // for some reason the db sizes differ. this may be due to the compaction options we have
-        // set when performing the snapshot
-        //server.stats(),
-
-        // The original instance contains the snapshotCreation task, while the snapshotted-instance does not. For this reason we need to compare the task queue **after** the task 4
-        server.tasks_filter("?from=2"),
-
-        server.index("catto").get_all_documents(GetAllDocumentsOptions::default()),
-        server.index("catto").settings(),
-        server.index("doggo").get_all_documents(GetAllDocumentsOptions::default()),
-        server.index("doggo").settings(),
-    );
-}

meilisearch/tests/stats/mod.rs

@@ -1,8 +1,8 @@
+use serde_json::json;
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn get_settings_unexisting_index() {

meilisearch/tests/swap_indexes/errors.rs

@@ -1,7 +1,7 @@
 use meili_snap::*;
+use serde_json::json;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn swap_indexes_bad_format() {

meilisearch/tests/swap_indexes/mod.rs

@@ -1,9 +1,9 @@
 mod errors;
 
 use meili_snap::{json_string, snapshot};
+use serde_json::json;
 
 use crate::common::{GetAllDocumentsOptions, Server};
-use crate::json;
 
 #[actix_rt::test]
 async fn swap_indexes() {

meilisearch/tests/tasks/mod.rs

@@ -1,11 +1,11 @@
 mod errors;
 
 use meili_snap::insta::assert_json_snapshot;
+use serde_json::json;
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn error_get_unexisting_task_status() {
@@ -33,7 +33,7 @@ async fn get_task_status() {
     index.create(None).await;
     index
         .add_documents(
-            json!([{
+            serde_json::json!([{
                 "id": 1,
                 "content": "foobar",
             }]),

milli/Cargo.toml

@@ -79,7 +79,6 @@ big_s = "1.0.2"
 insta = "1.29.0"
 maplit = "1.0.2"
 md5 = "0.7.0"
-meili-snap = { path = "../meili-snap" }
 rand = { version = "0.8.5", features = ["small_rng"] }
 
 [features]

milli/src/heed_codec/mod.rs

@@ -20,7 +20,10 @@ pub use self::beu32_str_codec::BEU32StrCodec;
 pub use self::field_id_word_count_codec::FieldIdWordCountCodec;
 pub use self::fst_set_codec::FstSetCodec;
 pub use self::obkv_codec::ObkvCodec;
-pub use self::roaring_bitmap::{BoRoaringBitmapCodec, CboRoaringBitmapCodec, RoaringBitmapCodec};
+pub use self::roaring_bitmap::{
+    BoRoaringBitmapCodec, CboRoaringBitmapCodec, CboRoaringTreemapCodec, RoaringBitmapCodec,
+    RoaringTreemapCodec,
+};
 pub use self::roaring_bitmap_length::{
     BoRoaringBitmapLenCodec, CboRoaringBitmapLenCodec, RoaringBitmapLenCodec,
 };

milli/src/heed_codec/roaring_bitmap/cbo_roaring_bitmap_codec.rs

@@ -60,16 +60,12 @@ impl CboRoaringBitmapCodec {
     /// if the merged values length is under the threshold, values are directly
     /// serialized in the buffer else a RoaringBitmap is created from the
     /// values and is serialized in the buffer.
-    pub fn merge_into<I, A>(slices: I, buffer: &mut Vec<u8>) -> io::Result<()>
-    where
-        I: IntoIterator<Item = A>,
-        A: AsRef<[u8]>,
-    {
+    pub fn merge_into(slices: &[Cow<[u8]>], buffer: &mut Vec<u8>) -> io::Result<()> {
         let mut roaring = RoaringBitmap::new();
         let mut vec = Vec::new();
 
         for bytes in slices {
-            if bytes.as_ref().len() <= THRESHOLD * size_of::<u32>() {
+            if bytes.len() <= THRESHOLD * size_of::<u32>() {
                 let mut reader = bytes.as_ref();
                 while let Ok(integer) = reader.read_u32::<NativeEndian>() {
                     vec.push(integer);
@@ -89,7 +85,7 @@ impl CboRoaringBitmapCodec {
                 }
             } else {
                 // We can unwrap safely because the vector is sorted upper.
-                let roaring = RoaringBitmap::from_sorted_iter(vec).unwrap();
+                let roaring = RoaringBitmap::from_sorted_iter(vec.into_iter()).unwrap();
                 roaring.serialize_into(buffer)?;
             }
         } else {

milli/src/heed_codec/roaring_bitmap/cbo_roaring_treemap_codec.rs

@@ -0,0 +1,196 @@
+use std::borrow::Cow;
+use std::io;
+use std::mem::size_of;
+
+use byteorder::{NativeEndian, ReadBytesExt, WriteBytesExt};
+use roaring::RoaringTreemap;
+
+use crate::heed_codec::BytesDecodeOwned;
+
+/// This is the limit where using a byteorder became less size efficient
+/// than using a direct roaring encoding, it is also the point where we are able
+/// to determine the encoding used only by using the array of bytes length.
+pub const THRESHOLD: usize = 4;
+
+/// A conditionnal codec that either use the RoaringBitmap
+/// or a lighter ByteOrder en/decoding method.
+pub struct CboRoaringTreemapCodec;
+
+impl CboRoaringTreemapCodec {
+    pub fn serialized_size(roaring: &RoaringTreemap) -> usize {
+        if roaring.len() <= THRESHOLD as u64 {
+            roaring.len() as usize * size_of::<u64>()
+        } else {
+            roaring.serialized_size()
+        }
+    }
+
+    pub fn serialize_into(roaring: &RoaringTreemap, vec: &mut Vec<u8>) {
+        if roaring.len() <= THRESHOLD as u64 {
+            // If the number of items (u32s) to encode is less than or equal to the threshold
+            // it means that it would weigh the same or less than the RoaringBitmap
+            // header, so we directly encode them using ByteOrder instead.
+            for integer in roaring {
+                vec.write_u64::<NativeEndian>(integer).unwrap();
+            }
+        } else {
+            // Otherwise, we use the classic RoaringBitmapCodec that writes a header.
+            roaring.serialize_into(vec).unwrap();
+        }
+    }
+
+    pub fn deserialize_from(mut bytes: &[u8]) -> io::Result<RoaringTreemap> {
+        if bytes.len() <= THRESHOLD * size_of::<u64>() {
+            // If there is threshold or less than threshold integers that can fit into this array
+            // of bytes it means that we used the ByteOrder codec serializer.
+            let mut bitmap = RoaringTreemap::new();
+            while let Ok(integer) = bytes.read_u64::<NativeEndian>() {
+                bitmap.insert(integer);
+            }
+            Ok(bitmap)
+        } else {
+            // Otherwise, it means we used the classic RoaringBitmapCodec and
+            // that the header takes threshold integers.
+            RoaringTreemap::deserialize_unchecked_from(bytes)
+        }
+    }
+
+    /// Merge serialized CboRoaringBitmaps in a buffer.
+    ///
+    /// if the merged values length is under the threshold, values are directly
+    /// serialized in the buffer else a RoaringBitmap is created from the
+    /// values and is serialized in the buffer.
+    pub fn merge_into(slices: &[Cow<[u8]>], buffer: &mut Vec<u8>) -> io::Result<()> {
+        let mut roaring = RoaringTreemap::new();
+        let mut vec = Vec::new();
+
+        for bytes in slices {
+            if bytes.len() <= THRESHOLD * size_of::<u64>() {
+                let mut reader = bytes.as_ref();
+                while let Ok(integer) = reader.read_u64::<NativeEndian>() {
+                    vec.push(integer);
+                }
+            } else {
+                roaring |= RoaringTreemap::deserialize_unchecked_from(bytes.as_ref())?;
+            }
+        }
+
+        if roaring.is_empty() {
+            vec.sort_unstable();
+            vec.dedup();
+
+            if vec.len() <= THRESHOLD {
+                for integer in vec {
+                    buffer.extend_from_slice(&integer.to_ne_bytes());
+                }
+            } else {
+                // We can unwrap safely because the vector is sorted upper.
+                let roaring = RoaringTreemap::from_sorted_iter(vec.into_iter()).unwrap();
+                roaring.serialize_into(buffer)?;
+            }
+        } else {
+            roaring.extend(vec);
+            roaring.serialize_into(buffer)?;
+        }
+
+        Ok(())
+    }
+}
+
+impl heed::BytesDecode<'_> for CboRoaringTreemapCodec {
+    type DItem = RoaringTreemap;
+
+    fn bytes_decode(bytes: &[u8]) -> Option<Self::DItem> {
+        Self::deserialize_from(bytes).ok()
+    }
+}
+
+impl BytesDecodeOwned for CboRoaringTreemapCodec {
+    type DItem = RoaringTreemap;
+
+    fn bytes_decode_owned(bytes: &[u8]) -> Option<Self::DItem> {
+        Self::deserialize_from(bytes).ok()
+    }
+}
+
+impl heed::BytesEncode<'_> for CboRoaringTreemapCodec {
+    type EItem = RoaringTreemap;
+
+    fn bytes_encode(item: &Self::EItem) -> Option<Cow<[u8]>> {
+        let mut vec = Vec::with_capacity(Self::serialized_size(item));
+        Self::serialize_into(item, &mut vec);
+        Some(Cow::Owned(vec))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::iter::FromIterator;
+
+    use heed::{BytesDecode, BytesEncode};
+
+    use super::*;
+
+    #[test]
+    fn verify_encoding_decoding() {
+        let input = RoaringTreemap::from_iter(0..THRESHOLD as u64);
+        let bytes = CboRoaringTreemapCodec::bytes_encode(&input).unwrap();
+        let output = CboRoaringTreemapCodec::bytes_decode(&bytes).unwrap();
+        assert_eq!(input, output);
+    }
+
+    #[test]
+    fn verify_threshold() {
+        let input = RoaringTreemap::from_iter(0..THRESHOLD as u64);
+
+        // use roaring treemap
+        let mut bytes = Vec::new();
+        input.serialize_into(&mut bytes).unwrap();
+        let roaring_size = bytes.len();
+
+        // use byteorder directly
+        let mut bytes = Vec::new();
+        for integer in input {
+            bytes.write_u64::<NativeEndian>(integer).unwrap();
+        }
+        let bo_size = bytes.len();
+
+        assert!(roaring_size > bo_size, "roaring size: {}, bo size {}", roaring_size, bo_size);
+    }
+
+    #[test]
+    fn merge_cbo_roaring_bitmaps() {
+        let mut buffer = Vec::new();
+
+        let small_data = vec![
+            RoaringTreemap::from_sorted_iter(1..4).unwrap(),
+            RoaringTreemap::from_sorted_iter(2..5).unwrap(),
+            RoaringTreemap::from_sorted_iter(4..6).unwrap(),
+            RoaringTreemap::from_sorted_iter(1..3).unwrap(),
+        ];
+
+        let small_data: Vec<_> =
+            small_data.iter().map(|b| CboRoaringTreemapCodec::bytes_encode(b).unwrap()).collect();
+        CboRoaringTreemapCodec::merge_into(small_data.as_slice(), &mut buffer).unwrap();
+        let bitmap = CboRoaringTreemapCodec::deserialize_from(&buffer).unwrap();
+        let expected = RoaringTreemap::from_sorted_iter(1..6).unwrap();
+        assert_eq!(bitmap, expected);
+
+        let medium_data = vec![
+            RoaringTreemap::from_sorted_iter(1..4).unwrap(),
+            RoaringTreemap::from_sorted_iter(2..5).unwrap(),
+            RoaringTreemap::from_sorted_iter(4..8).unwrap(),
+            RoaringTreemap::from_sorted_iter(0..3).unwrap(),
+            RoaringTreemap::from_sorted_iter(7..23).unwrap(),
+        ];
+
+        let medium_data: Vec<_> =
+            medium_data.iter().map(|b| CboRoaringTreemapCodec::bytes_encode(b).unwrap()).collect();
+        buffer.clear();
+        CboRoaringTreemapCodec::merge_into(medium_data.as_slice(), &mut buffer).unwrap();
+
+        let bitmap = CboRoaringTreemapCodec::deserialize_from(&buffer).unwrap();
+        let expected = RoaringTreemap::from_sorted_iter(0..23).unwrap();
+        assert_eq!(bitmap, expected);
+    }
+}

milli/src/heed_codec/roaring_bitmap/mod.rs

@@ -1,7 +1,11 @@
 mod bo_roaring_bitmap_codec;
 pub mod cbo_roaring_bitmap_codec;
+pub mod cbo_roaring_treemap_codec;
 mod roaring_bitmap_codec;
+mod roaring_treemap_codec;
 
 pub use self::bo_roaring_bitmap_codec::BoRoaringBitmapCodec;
 pub use self::cbo_roaring_bitmap_codec::CboRoaringBitmapCodec;
+pub use self::cbo_roaring_treemap_codec::CboRoaringTreemapCodec;
 pub use self::roaring_bitmap_codec::RoaringBitmapCodec;
+pub use self::roaring_treemap_codec::RoaringTreemapCodec;