Merge pull request #5919 from meilisearch/raise-remote-search-timeout

Raise remote search timeout to 30s

.github/ISSUE_TEMPLATE/new_feature_issue.md

@@ -24,11 +24,6 @@ TBD
 - [ ] If not, add the `no db change` label to your PR, and you're good to merge.
 - [ ] If yes, add the `db change` label to your PR. You'll receive a message explaining you what to do.
 
-### Reminders when adding features
-
-- [ ] Write unit tests using insta
-- [ ] Write declarative integration tests in [workloads/tests](https://github.com/meilisearch/meilisearch/tree/main/workloads/test). Specify the routes to call and then call `cargo xtask test workloads/tests/YOUR_TEST.json --update-responses` so that responses are automatically filled.
-
 ### Reminders when modifying the API
 
 - [ ] Update the openAPI file with utoipa:

.github/dependabot.yml

@@ -7,5 +7,6 @@ updates:
     schedule:
       interval: "monthly"
     labels:
+      - 'skip changelog'
       - 'dependencies'
     rebase-strategy: disabled

.github/release-draft-template.yml

@@ -0,0 +1,33 @@
+name-template: 'v$RESOLVED_VERSION'
+tag-template: 'v$RESOLVED_VERSION'
+exclude-labels:
+  - 'skip changelog'
+version-resolver:
+  minor:
+    labels:
+      - 'enhancement'
+  default: patch
+categories:
+  - title: '⚠️ Breaking changes'
+    label: 'breaking-change'
+  - title: '🚀 Enhancements'
+    label: 'enhancement'
+  - title: '🐛 Bug Fixes'
+    label: 'bug'
+  - title: '🔒 Security'
+    label: 'security'
+  - title: '⚙️ Maintenance/misc'
+    label:
+      - 'maintenance'
+      - 'documentation'
+template: |
+  $CHANGES
+
+  ❤️  Huge thanks to our contributors: $CONTRIBUTORS.
+no-changes-template: 'Changes are coming soon 😎'
+sort-direction: 'ascending'
+replacers:
+  - search: '/(?:and )?@dependabot-preview(?:\[bot\])?,?/g'
+    replace: ''
+  - search: '/(?:and )?@dependabot(?:\[bot\])?,?/g'
+    replace: ''

.github/workflows/bench-manual.yml

@@ -18,7 +18,7 @@ jobs:
     timeout-minutes: 180 # 3h
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.91.1
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
 

.github/workflows/bench-pr.yml

@@ -66,7 +66,9 @@ jobs:
           fetch-depth: 0 # fetch full history to be able to get main commit sha
           ref: ${{ steps.comment-branch.outputs.head_ref }}
 
-      - uses: dtolnay/rust-toolchain@1.91.1
+      - uses: dtolnay/rust-toolchain@1.89
+        with:
+          profile: minimal
 
       - name: Run benchmarks on PR ${{ github.event.issue.id }}
         run: |

.github/workflows/bench-push-indexing.yml

@@ -12,7 +12,9 @@ jobs:
     timeout-minutes: 180 # 3h
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.91.1
+      - uses: dtolnay/rust-toolchain@1.89
+        with:
+          profile: minimal
 
       # Run benchmarks
       - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch main - Commit ${{ github.sha }}

.github/workflows/benchmarks-manual.yml

@@ -18,7 +18,7 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.91.1
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
 

.github/workflows/benchmarks-pr.yml

@@ -44,7 +44,7 @@ jobs:
             exit 1
           fi
 
-      - uses: dtolnay/rust-toolchain@1.91.1
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
 

.github/workflows/benchmarks-push-indexing.yml

@@ -16,7 +16,7 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.91.1
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
 

.github/workflows/benchmarks-push-search-geo.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.91.1
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
 

.github/workflows/benchmarks-push-search-songs.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.91.1
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
 

.github/workflows/benchmarks-push-search-wiki.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.91.1
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
 

.github/workflows/db-change-comments.yml

@@ -6,7 +6,7 @@ on:
 
 env:
   MESSAGE: |
-    ### Hello, I'm a bot 🤖
+    ### Hello, I'm a bot 🤖 
 
     You are receiving this message because you declared that this PR make changes to the Meilisearch database.
     Depending on the nature of the change, additional actions might be required on your part. The following sections detail the additional actions depending on the nature of the change, please copy the relevant section in the description of your PR, and make sure to perform the required actions.
@@ -19,7 +19,6 @@ env:
 
     - [ ] Detail the change to the DB format and why they are forward compatible
     - [ ] Forward-compatibility: A database created before this PR and using the features touched by this PR was able to be opened by a Meilisearch produced by the code of this PR.
-    - [ ] Declarative test: add a [declarative test containing a dumpless upgrade](https://github.com/meilisearch/meilisearch/blob/main/TESTING.md#typical-usage)
 
 
     ## This PR makes breaking changes
@@ -36,7 +35,8 @@ env:
       - [ ] Write the code to go from the old database to the new one
         - If the change happened in milli, the upgrade function should be written and called [here](https://github.com/meilisearch/meilisearch/blob/3fd86e8d76d7d468b0095d679adb09211ca3b6c0/crates/milli/src/update/upgrade/mod.rs#L24-L47)
         - If the change happened in the index-scheduler, we've never done it yet, but the right place to do it should be [here](https://github.com/meilisearch/meilisearch/blob/3fd86e8d76d7d468b0095d679adb09211ca3b6c0/crates/index-scheduler/src/scheduler/process_upgrade/mod.rs#L13)
-      - [ ] Declarative test: add a [declarative test containing a dumpless upgrade](https://github.com/meilisearch/meilisearch/blob/main/TESTING.md#typical-usage)
+      - [ ] Write an integration test [here](https://github.com/meilisearch/meilisearch/blob/main/crates/meilisearch/tests/upgrade/mod.rs) ensuring you can read the old database, upgrade to the new database, and read the new database as expected
+    
 
 jobs:
   add-comment:

.github/workflows/flaky-tests.yml

@@ -3,7 +3,7 @@ name: Look for flaky tests
 on:
   workflow_dispatch:
   schedule:
-    - cron: "0 4 * * *" # Every day at 4:00AM
+    - cron: '0 4 * * *' # Every day at 4:00AM
 
 jobs:
   flaky:
@@ -13,17 +13,11 @@ jobs:
       image: ubuntu:22.04
     steps:
       - uses: actions/checkout@v5
-      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
-        run: |
-          sudo rm -rf "/opt/ghc" || true
-          sudo rm -rf "/usr/share/dotnet" || true
-          sudo rm -rf "/usr/local/lib/android" || true
-          sudo rm -rf "/usr/local/share/boost" || true
       - name: Install needed dependencies
         run: |
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
-      - uses: dtolnay/rust-toolchain@1.91.1
+      - uses: dtolnay/rust-toolchain@1.89
       - name: Install cargo-flaky
         run: cargo install cargo-flaky
       - name: Run cargo flaky in the dumps

.github/workflows/fuzzer-indexing.yml

@@ -12,7 +12,9 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.91.1
+      - uses: dtolnay/rust-toolchain@1.89
+        with:
+          profile: minimal
 
       # Run benchmarks
       - name: Run the fuzzer

.github/workflows/publish-apt-brew-pkg.yml

@@ -25,13 +25,7 @@ jobs:
         run: |
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
-      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
+      - uses: dtolnay/rust-toolchain@1.89
-        run: |
-          sudo rm -rf "/opt/ghc" || true
-          sudo rm -rf "/usr/share/dotnet" || true
-          sudo rm -rf "/usr/local/lib/android" || true
-          sudo rm -rf "/usr/local/share/boost" || true
-      - uses: dtolnay/rust-toolchain@1.91.1
       - name: Install cargo-deb
         run: cargo install cargo-deb
       - uses: actions/checkout@v5

.github/workflows/publish-docker-images.yml

@@ -14,105 +14,10 @@ on:
   workflow_dispatch:
 
 jobs:
-  build:
+  docker:
-    runs-on: ${{ matrix.runner }}
+    runs-on: docker
-
-    strategy:
-      matrix:
-        platform: [amd64, arm64]
-        edition: [community, enterprise]
-        include:
-          - platform: amd64
-            runner: ubuntu-24.04
-          - platform: arm64
-            runner: ubuntu-24.04-arm
-          - edition: community
-            registry: getmeili/meilisearch
-            feature-flag: ""
-          - edition: enterprise
-            registry: getmeili/meilisearch-enterprise
-            feature-flag: "--features enterprise"
-
-    permissions: {}
-    steps:
-      - uses: actions/checkout@v5
-
-      - name: Prepare
-        run: |
-          platform=linux/${{ matrix.platform }}
-          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          platforms: linux/${{ matrix.platform }}
-          install: true
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ matrix.registry }}
-          # Prevent `latest` to be updated for each new tag pushed.
-          # We need latest and `vX.Y` tags to only be pushed for the stable Meilisearch releases.
-          flavor: latest=false
-          tags: |
-            type=ref,event=tag
-            type=raw,value=nightly,enable=${{ github.event_name != 'push' }}
-            type=semver,pattern=v{{major}}.{{minor}},enable=${{ steps.check-tag-format.outputs.stable == 'true' }}
-            type=semver,pattern=v{{major}},enable=${{ steps.check-tag-format.outputs.stable == 'true' }}
-            type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable == 'true' && steps.check-tag-format.outputs.latest == 'true' }}
-
-      - name: Build and push by digest
-        uses: docker/build-push-action@v6
-        id: build-and-push
-        with:
-          platforms: linux/${{ matrix.platform }}
-          labels: ${{ steps.meta.outputs.labels }}
-          tags: ${{ matrix.registry }}
-          outputs: type=image,push-by-digest=true,name-canonical=true,push=true
-          build-args: |
-            COMMIT_SHA=${{ github.sha }}
-            COMMIT_DATE=${{ steps.build-metadata.outputs.date }}
-            GIT_TAG=${{ github.ref_name }}
-            EXTRA_ARGS=${{ matrix.feature-flag }}
-
-      - name: Export digest
-        run: |
-          mkdir -p ${{ runner.temp }}/digests
-          digest="${{ steps.build-and-push.outputs.digest }}"
-          touch "${{ runner.temp }}/digests/${digest#sha256:}"
-
-      - name: Upload digest
-        uses: actions/upload-artifact@v4
-        with:
-          name: digests-${{ matrix.edition }}-${{ env.PLATFORM_PAIR }}
-          path: ${{ runner.temp }}/digests/*
-          if-no-files-found: error
-          retention-days: 1
-
-  merge:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        edition: [community, enterprise]
-        include:
-          - edition: community
-            registry: getmeili/meilisearch
-          - edition: enterprise
-            registry: getmeili/meilisearch-enterprise
-    needs:
-      - build
-
     permissions:
       id-token: write # This is needed to use Cosign in keyless mode
-
     steps:
       - uses: actions/checkout@v5
 
@@ -153,15 +58,14 @@ jobs:
 
           echo "date=$commit_date" >> $GITHUB_OUTPUT
 
-      - name: Install cosign
+      - name: Set up QEMU
-        uses: sigstore/cosign-installer@d7543c93d881b35a8faa02e8e3605f69b7a1ce62 # tag=v3.10.0
+        uses: docker/setup-qemu-action@v3
 
-      - name: Download digests
+      - name: Set up Docker Buildx
-        uses: actions/download-artifact@v4
+        uses: docker/setup-buildx-action@v3
-        with:
+
-          path: ${{ runner.temp }}/digests
+      - name: Install cosign
-          pattern: digests-${{ matrix.edition }}-*
+        uses: sigstore/cosign-installer@d58896d6a1865668819e1d91763c7751a165e159 # tag=v3.9.2
-          merge-multiple: true
 
       - name: Login to Docker Hub
         uses: docker/login-action@v3
@@ -169,14 +73,11 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v5
         with:
-          images: ${{ matrix.registry }}
+          images: getmeili/meilisearch
           # Prevent `latest` to be updated for each new tag pushed.
           # We need latest and `vX.Y` tags to only be pushed for the stable Meilisearch releases.
           flavor: latest=false
@@ -187,31 +88,33 @@ jobs:
             type=semver,pattern=v{{major}},enable=${{ steps.check-tag-format.outputs.stable == 'true' }}
             type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable == 'true' && steps.check-tag-format.outputs.latest == 'true' }}
 
-      - name: Create manifest list and push
+      - name: Build and push
-        working-directory: ${{ runner.temp }}/digests
+        uses: docker/build-push-action@v6
-        run: |
+        id: build-and-push
-          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+        with:
-            $(printf '${{ matrix.registry }}@sha256:%s ' *)
+          push: true
-
+          platforms: linux/amd64,linux/arm64
-      - name: Inspect image to fetch digest to sign
+          tags: ${{ steps.meta.outputs.tags }}
-        run: |
+          build-args: |
-          digest=$(docker buildx imagetools inspect --format='{{ json .Manifest }}' ${{ matrix.registry }}:${{ steps.meta.outputs.version }} | jq -r '.digest')
+            COMMIT_SHA=${{ github.sha }}
-          echo "DIGEST=${digest}" >> $GITHUB_ENV
+            COMMIT_DATE=${{ steps.build-metadata.outputs.date }}
+            GIT_TAG=${{ github.ref_name }}
 
       - name: Sign the images with GitHub OIDC Token
         env:
+          DIGEST: ${{ steps.build-and-push.outputs.digest }}
           TAGS: ${{ steps.meta.outputs.tags }}
         run: |
           images=""
           for tag in ${TAGS}; do
-            images+="${tag}@${{ env.DIGEST }} "
+            images+="${tag}@${DIGEST} "
           done
           cosign sign --yes ${images}
 
-      # /!\ Don't touch this without checking with engineers working on the Cloud code base on #discussion-engineering Slack channel
+      # /!\ Don't touch this without checking with Cloud team
-      - name: Notify meilisearch-cloud 
+      - name: Send CI information to Cloud team
         # Do not send if nightly build (i.e. 'schedule' or 'workflow_dispatch' event)
-        if: ${{ (github.event_name == 'push') && (matrix.edition == 'enterprise') }}
+        if: github.event_name == 'push'
         uses: peter-evans/repository-dispatch@v3
         with:
           token: ${{ secrets.MEILI_BOT_GH_PAT }}
@@ -219,13 +122,21 @@ jobs:
           event-type: cloud-docker-build
           client-payload: '{ "meilisearch_version": "${{ github.ref_name }}", "stable": "${{ steps.check-tag-format.outputs.stable }}" }'
 
-      # /!\ Don't touch this without checking with integration team members on #discussion-integrations Slack channel 
+      # Send notification to Swarmia to notify of a deployment: https://app.swarmia.com
-      - name: Notify meilisearch-kubernetes
+      # - name: 'Setup jq'
-        # Do not send if nightly build (i.e. 'schedule' or 'workflow_dispatch' event), or if not stable
+      #   uses: dcarbone/install-jq-action
-        if: ${{ github.event_name == 'push' && matrix.edition == 'community' && steps.check-tag-format.outputs.stable == 'true' }}
+      # - name: Send deployment to Swarmia
-        uses: peter-evans/repository-dispatch@v3
+      #   if: github.event_name == 'push' && success()
-        with:
+      #   run: |
-          token: ${{ secrets.MEILI_BOT_GH_PAT }}
+      #     JSON_STRING=$( jq --null-input --compact-output \
-          repository: meilisearch/meilisearch-kubernetes
+      #     --arg version "${{ github.ref_name }}" \
-          event-type: meilisearch-release
+      #     --arg appName "meilisearch" \
-          client-payload: '{ "version": "${{ github.ref_name }}" }'
+      #     --arg environment "production" \
+      #     --arg commitSha "${{ github.sha }}" \
+      #     --arg repositoryFullName "${{ github.repository }}" \
+      #     '{"version": $version, "appName": $appName, "environment": $environment, "commitSha": $commitSha, "repositoryFullName": $repositoryFullName}' )
+
+      #     curl -H "Authorization: ${{ secrets.SWARMIA_DEPLOYMENTS_AUTHORIZATION }}" \
+      #       -H "Content-Type: application/json" \
+      #       -d "$JSON_STRING" \
+      #       https://hook.swarmia.com/deployments

.github/workflows/publish-release-assets.yml

@@ -11,7 +11,7 @@ jobs:
   check-version:
     name: Check the version validity
     runs-on: ubuntu-latest
-    # No need to check the version for dry run (cron or workflow_dispatch)
+    # No need to check the version for dry run (cron)
     steps:
       - uses: actions/checkout@v5
       # Check if the tag has the v<nmumber>.<number>.<number> format.
@@ -32,66 +32,161 @@ jobs:
         if: github.event_name == 'release' && steps.check-tag-format.outputs.stable == 'true'
         run: bash .github/scripts/check-release.sh
 
-  publish-binaries:
+  publish-linux:
-    name: Publish binary for ${{ matrix.release }} ${{ matrix.edition }} edition
+    name: Publish binary for Linux
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        edition: [community, enterprise]
-        release:
-          [macos-amd64, macos-aarch64, windows, linux-amd64, linux-aarch64]
-        include:
-          - edition: "community"
-            feature-flag: ""
-            edition-suffix: ""
-          - edition: "enterprise"
-            feature-flag: "--features enterprise"
-            edition-suffix: "enterprise-"
-          - release: macos-amd64
-            os: macos-15-intel
-            binary_path: release/meilisearch
-            asset_name: macos-amd64
-            extra-args: ""
-          - release: macos-aarch64
-            os: macos-14
-            binary_path: aarch64-apple-darwin/release/meilisearch
-            asset_name: macos-apple-silicon
-            extra-args: "--target aarch64-apple-darwin"
-          - release: windows
-            os: windows-2022
-            binary_path: release/meilisearch.exe
-            asset_name: windows-amd64.exe
-            extra-args: ""
-          - release: linux-amd64
-            os: ubuntu-22.04
-            binary_path: x86_64-unknown-linux-gnu/release/meilisearch
-            asset_name: linux-amd64
-            extra-args: "--target x86_64-unknown-linux-gnu"
-          - release: linux-aarch64
-            os: ubuntu-22.04-arm
-            binary_path: aarch64-unknown-linux-gnu/release/meilisearch
-            asset_name: linux-aarch64
-            extra-args: "--target aarch64-unknown-linux-gnu"
     needs: check-version
+    container:
+      # Use ubuntu-22.04 to compile with glibc 2.35
+      image: ubuntu:22.04
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.91.1
+      - name: Install needed dependencies
+        run: |
+          apt-get update && apt-get install -y curl
+          apt-get install build-essential -y
+      - uses: dtolnay/rust-toolchain@1.89
       - name: Build
-        run: cargo build --release --locked ${{ matrix.feature-flag }} ${{ matrix.extra-args }}
+        run: cargo build --release --locked
-      # No need to upload binaries for dry run (cron or workflow_dispatch)
+      # No need to upload binaries for dry run (cron)
       - name: Upload binaries to release
         if: github.event_name == 'release'
         uses: svenstaro/upload-release-action@2.11.2
         with:
           repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
-          file: target/${{ matrix.binary_path }}
+          file: target/release/meilisearch
-          asset_name: meilisearch-${{ matrix.edition-suffix }}${{ matrix.asset_name }}
+          asset_name: meilisearch-linux-amd64
+          tag: ${{ github.ref }}
+
+  publish-macos-windows:
+    name: Publish binary for ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    needs: check-version
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-13, windows-2022]
+        include:
+          - os: macos-13
+            artifact_name: meilisearch
+            asset_name: meilisearch-macos-amd64
+          - os: windows-2022
+            artifact_name: meilisearch.exe
+            asset_name: meilisearch-windows-amd64.exe
+    steps:
+      - uses: actions/checkout@v5
+      - uses: dtolnay/rust-toolchain@1.89
+      - name: Build
+        run: cargo build --release --locked
+      # No need to upload binaries for dry run (cron)
+      - name: Upload binaries to release
+        if: github.event_name == 'release'
+        uses: svenstaro/upload-release-action@2.11.2
+        with:
+          repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
+          file: target/release/${{ matrix.artifact_name }}
+          asset_name: ${{ matrix.asset_name }}
+          tag: ${{ github.ref }}
+
+  publish-macos-apple-silicon:
+    name: Publish binary for macOS silicon
+    runs-on: macos-13
+    needs: check-version
+    strategy:
+      matrix:
+        include:
+          - target: aarch64-apple-darwin
+            asset_name: meilisearch-macos-apple-silicon
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+      - name: Installing Rust toolchain
+        uses: dtolnay/rust-toolchain@1.89
+        with:
+          profile: minimal
+          target: ${{ matrix.target }}
+      - name: Cargo build
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --release --target ${{ matrix.target }}
+      - name: Upload the binary to release
+        # No need to upload binaries for dry run (cron)
+        if: github.event_name == 'release'
+        uses: svenstaro/upload-release-action@2.11.2
+        with:
+          repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
+          file: target/${{ matrix.target }}/release/meilisearch
+          asset_name: ${{ matrix.asset_name }}
+          tag: ${{ github.ref }}
+
+  publish-aarch64:
+    name: Publish binary for aarch64
+    runs-on: ubuntu-latest
+    needs: check-version
+    env:
+      DEBIAN_FRONTEND: noninteractive
+    container:
+      # Use ubuntu-22.04 to compile with glibc 2.35
+      image: ubuntu:22.04
+    strategy:
+      matrix:
+        include:
+          - target: aarch64-unknown-linux-gnu
+            asset_name: meilisearch-linux-aarch64
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+      - name: Install needed dependencies
+        run: |
+          apt-get update -y && apt upgrade -y
+          apt-get install -y curl build-essential gcc-aarch64-linux-gnu
+      - name: Set up Docker for cross compilation
+        run: |
+          apt-get install -y curl apt-transport-https ca-certificates software-properties-common
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+          add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+          apt-get update -y && apt-get install -y docker-ce
+      - name: Installing Rust toolchain
+        uses: dtolnay/rust-toolchain@1.89
+        with:
+          profile: minimal
+          target: ${{ matrix.target }}
+      - name: Configure target aarch64 GNU
+        ## Environment variable is not passed using env:
+        ## LD gold won't work with MUSL
+        # env:
+        #   JEMALLOC_SYS_WITH_LG_PAGE: 16
+        #   RUSTFLAGS: '-Clink-arg=-fuse-ld=gold'
+        run: |
+          echo '[target.aarch64-unknown-linux-gnu]' >> ~/.cargo/config
+          echo 'linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config
+          echo 'JEMALLOC_SYS_WITH_LG_PAGE=16' >> $GITHUB_ENV
+      - name: Install a default toolchain that will be used to build cargo cross
+        run: |
+          rustup default stable
+      - name: Cargo build
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          use-cross: true
+          args: --release --target ${{ matrix.target }}
+        env:
+          CROSS_DOCKER_IN_DOCKER: true
+      - name: List target output files
+        run: ls -lR ./target
+      - name: Upload the binary to release
+        # No need to upload binaries for dry run (cron)
+        if: github.event_name == 'release'
+        uses: svenstaro/upload-release-action@2.11.2
+        with:
+          repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
+          file: target/${{ matrix.target }}/release/meilisearch
+          asset_name: ${{ matrix.asset_name }}
           tag: ${{ github.ref }}
 
   publish-openapi-file:
     name: Publish OpenAPI file
-    needs: check-version
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -106,7 +201,7 @@ jobs:
           cd crates/openapi-generator
           cargo run --release -- --pretty --output ../../meilisearch.json
       - name: Upload OpenAPI to Release
-        # No need to upload for dry run (cron or workflow_dispatch)
+        # No need to upload for dry run (cron)
         if: github.event_name == 'release'
         uses: svenstaro/upload-release-action@2.11.2
         with:

.github/workflows/release-drafter.yml

@@ -0,0 +1,20 @@
+name: Release Drafter
+
+permissions:
+  contents: read
+  pull-requests: write
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  update_release_draft:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: release-drafter/release-drafter@v6
+        with:
+          config-name: release-draft-template.yml
+        env:
+          GITHUB_TOKEN: ${{ secrets.RELEASE_DRAFTER_TOKEN }}

.github/workflows/sdks-tests.yml

@@ -25,18 +25,14 @@ jobs:
       - uses: actions/checkout@v5
       - name: Define the Docker image we need to use
         id: define-image
-        env:
-          EVENT_NAME: ${{ github.event_name }}
-          DOCKER_IMAGE_INPUT: ${{ github.event.inputs.docker_image }}
         run: |
+          event=${{ github.event_name }}
           echo "docker-image=nightly" >> $GITHUB_OUTPUT
-          if [[ "$EVENT_NAME" == 'workflow_dispatch' ]]; then
+          if [[ $event == 'workflow_dispatch' ]]; then
-            echo "docker-image=$DOCKER_IMAGE_INPUT" >> $GITHUB_OUTPUT
+            echo "docker-image=${{ github.event.inputs.docker_image }}" >> $GITHUB_OUTPUT
           fi
       - name: Docker image is ${{ steps.define-image.outputs.docker-image }}
-        env:
+        run: echo "Docker image is ${{ steps.define-image.outputs.docker-image }}"
-          DOCKER_IMAGE: ${{ steps.define-image.outputs.docker-image }}
-        run: echo "Docker image is $DOCKER_IMAGE"
 
 ##########
 ## SDKs ##
@@ -54,7 +50,7 @@ jobs:
         with:
           repository: meilisearch/meilisearch-dotnet
       - name: Setup .NET Core
-        uses: actions/setup-dotnet@v5
+        uses: actions/setup-dotnet@v4
         with:
           dotnet-version: "8.0.x"
       - name: Install dependencies
@@ -72,7 +68,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch-enterprise:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
@@ -96,7 +92,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch-enterprise:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
@@ -104,7 +100,7 @@ jobs:
           - '7700:7700'
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v5
         with:
           go-version: stable
       - uses: actions/checkout@v5
@@ -126,7 +122,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch-enterprise:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
@@ -139,13 +135,13 @@ jobs:
       - name: Set up Java
         uses: actions/setup-java@v5
         with:
-          java-version: 17
+          java-version: 8
-          distribution: 'temurin'
+          distribution: 'zulu'
           cache: gradle
       - name: Grant execute permission for gradlew
         run: chmod +x gradlew
       - name: Build and run unit and integration tests
-        run: ./gradlew build integrationTest --info
+        run: ./gradlew build integrationTest
 
   meilisearch-js-tests:
     needs: define-docker-image
@@ -153,7 +149,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch-enterprise:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
@@ -164,7 +160,7 @@ jobs:
         with:
           repository: meilisearch/meilisearch-js
       - name: Setup node
-        uses: actions/setup-node@v5
+        uses: actions/setup-node@v4
         with:
           cache: 'yarn'
       - name: Install dependencies
@@ -188,7 +184,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch-enterprise:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
@@ -217,7 +213,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch-enterprise:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
@@ -228,7 +224,7 @@ jobs:
         with:
           repository: meilisearch/meilisearch-python
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v5
       - name: Install pipenv
         uses: dschep/install-pipenv-action@v1
       - name: Install dependencies
@@ -242,7 +238,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch-enterprise:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
@@ -267,7 +263,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch-enterprise:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
@@ -288,7 +284,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch-enterprise:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
@@ -311,7 +307,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch-enterprise:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
@@ -322,7 +318,7 @@ jobs:
         with:
           repository: meilisearch/meilisearch-js-plugins
       - name: Setup node
-        uses: actions/setup-node@v5
+        uses: actions/setup-node@v4
         with:
           cache: yarn
       - name: Install dependencies
@@ -342,7 +338,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch-enterprise:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
@@ -374,7 +370,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       meilisearch:
-        image: getmeili/meilisearch-enterprise:${{ needs.define-docker-image.outputs.docker-image }}
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
         env:
           MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
           MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}

.github/workflows/test-suite.yml

@@ -15,40 +15,31 @@ env:
 
 jobs:
   test-linux:
-    name: Tests on Ubuntu
+    name: Tests on ubuntu-22.04
-    runs-on: ${{ matrix.runner }}
+    runs-on: ubuntu-latest
-    strategy:
+    container:
-      matrix:
+      # Use ubuntu-22.04 to compile with glibc 2.35
-        runner: [ubuntu-22.04, ubuntu-22.04-arm]
+      image: ubuntu:22.04
-        features: ["", "--features enterprise"]
     steps:
       - uses: actions/checkout@v5
-      - name: check free space before
+      - name: Install needed dependencies
-        run: df -h
-      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
         run: |
-          sudo rm -rf "/opt/ghc" || true
+          apt-get update && apt-get install -y curl
-          sudo rm -rf "/usr/share/dotnet" || true
+          apt-get install build-essential -y
-          sudo rm -rf "/usr/local/lib/android" || true
-          sudo rm -rf "/usr/local/share/boost" || true
-      - name: check free space after
-        run: df -h
       - name: Setup test with Rust stable
-        uses: dtolnay/rust-toolchain@1.91.1
+        uses: dtolnay/rust-toolchain@1.89
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.8.0
-        with:
+      - name: Run cargo check without any default features
-          key: ${{ matrix.features }}
-      - name: Run cargo build without any default features
         uses: actions-rs/cargo@v1
         with:
           command: build
-          args: --locked --no-default-features --all
+          args: --locked --release --no-default-features --all
       - name: Run cargo test
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --locked --all ${{ matrix.features }}
+          args: --locked --release --all
 
   test-others:
     name: Tests on ${{ matrix.os }}
@@ -56,58 +47,51 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-14, windows-2022]
+        os: [macos-13, windows-2022]
-        features: ["", "--features enterprise"]
-    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     steps:
       - uses: actions/checkout@v5
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.8.0
-      - uses: dtolnay/rust-toolchain@1.91.1
+      - uses: dtolnay/rust-toolchain@1.89
-      - name: Run cargo build without any default features
+      - name: Run cargo check without any default features
         uses: actions-rs/cargo@v1
         with:
           command: build
-          args: --locked --no-default-features --all
+          args: --locked --release --no-default-features --all
       - name: Run cargo test
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --locked --all ${{ matrix.features }}
+          args: --locked --release --all
 
   test-all-features:
     name: Tests almost all features
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
+    container:
+      # Use ubuntu-22.04 to compile with glibc 2.35
+      image: ubuntu:22.04
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     steps:
       - uses: actions/checkout@v5
-      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
+      - name: Install needed dependencies
         run: |
-          sudo rm -rf "/opt/ghc" || true
+          apt-get update
-          sudo rm -rf "/usr/share/dotnet" || true
+          apt-get install --assume-yes build-essential curl
-          sudo rm -rf "/usr/local/lib/android" || true
+      - uses: dtolnay/rust-toolchain@1.89
-          sudo rm -rf "/usr/local/share/boost" || true
-      - uses: dtolnay/rust-toolchain@1.91.1
       - name: Run cargo build with almost all features
         run: |
-          cargo build --workspace --locked --features "$(cargo xtask list-features --exclude-feature cuda,test-ollama)"
+          cargo build --workspace --locked --release --features "$(cargo xtask list-features --exclude-feature cuda,test-ollama)"
       - name: Run cargo test with almost all features
         run: |
-          cargo test --workspace --locked --features "$(cargo xtask list-features --exclude-feature cuda,test-ollama)"
+          cargo test --workspace --locked --release --features "$(cargo xtask list-features --exclude-feature cuda,test-ollama)"
 
   ollama-ubuntu:
     name: Test with Ollama
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     env:
       MEILI_TEST_OLLAMA_SERVER: "http://localhost:11434"
     steps:
       - uses: actions/checkout@v5
-      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
-        run: |
-          sudo rm -rf "/opt/ghc" || true
-          sudo rm -rf "/usr/share/dotnet" || true
-          sudo rm -rf "/usr/local/lib/android" || true
-          sudo rm -rf "/usr/local/share/boost" || true
       - name: Install Ollama
         run: |
           curl -fsSL https://ollama.com/install.sh | sudo -E sh
@@ -131,21 +115,21 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --locked -p meilisearch --features test-ollama ollama
+          args: --locked --release --all --features test-ollama ollama
 
   test-disabled-tokenization:
     name: Test disabled tokenization
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
+    container:
+      image: ubuntu:22.04
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     steps:
       - uses: actions/checkout@v5
-      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
+      - name: Install needed dependencies
         run: |
-          sudo rm -rf "/opt/ghc" || true
+          apt-get update
-          sudo rm -rf "/usr/share/dotnet" || true
+          apt-get install --assume-yes build-essential curl
-          sudo rm -rf "/usr/local/lib/android" || true
+      - uses: dtolnay/rust-toolchain@1.89
-          sudo rm -rf "/usr/local/share/boost" || true
-      - uses: dtolnay/rust-toolchain@1.91.1
       - name: Run cargo tree without default features and check lindera is not present
         run: |
           if cargo tree -f '{p} {f}' -e normal --no-default-features | grep -qz lindera; then
@@ -156,39 +140,36 @@ jobs:
         run: |
           cargo tree -f '{p} {f}' -e normal | grep lindera -qz
 
-  build:
+  # We run tests in debug also, to make sure that the debug_assertions are hit
-    name: Build in release
+  test-debug:
-    runs-on: ubuntu-22.04
+    name: Run tests in debug
+    runs-on: ubuntu-latest
+    container:
+      # Use ubuntu-22.04 to compile with glibc 2.35
+      image: ubuntu:22.04
     steps:
       - uses: actions/checkout@v5
-      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
+      - name: Install needed dependencies
         run: |
-          sudo rm -rf "/opt/ghc" || true
+          apt-get update && apt-get install -y curl
-          sudo rm -rf "/usr/share/dotnet" || true
+          apt-get install build-essential -y
-          sudo rm -rf "/usr/local/lib/android" || true
+      - uses: dtolnay/rust-toolchain@1.89
-          sudo rm -rf "/usr/local/share/boost" || true
-      - uses: dtolnay/rust-toolchain@1.91.1
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.8.0
-      - name: Build
+      - name: Run tests in debug
-        run: cargo build --release --locked --target x86_64-unknown-linux-gnu
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --locked --all
 
   clippy:
     name: Run Clippy
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        features: ["", "--features enterprise"]
     steps:
       - uses: actions/checkout@v5
-      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
+      - uses: dtolnay/rust-toolchain@1.89
-        run: |
-          sudo rm -rf "/opt/ghc" || true
-          sudo rm -rf "/usr/share/dotnet" || true
-          sudo rm -rf "/usr/local/lib/android" || true
-          sudo rm -rf "/usr/local/share/boost" || true
-      - uses: dtolnay/rust-toolchain@1.91.1
         with:
+          profile: minimal
           components: clippy
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.8.0
@@ -196,21 +177,18 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: --all-targets ${{ matrix.features }} -- --deny warnings -D clippy::todo
+          args: --all-targets -- --deny warnings
 
   fmt:
     name: Run Rustfmt
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
-      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
+      - uses: dtolnay/rust-toolchain@1.89
-        run: |
-          sudo rm -rf "/opt/ghc" || true
-          sudo rm -rf "/usr/share/dotnet" || true
-          sudo rm -rf "/usr/local/lib/android" || true
-          sudo rm -rf "/usr/local/share/boost" || true
-      - uses: dtolnay/rust-toolchain@1.91.1
         with:
+          profile: minimal
+          toolchain: nightly-2024-07-09
+          override: true
           components: rustfmt
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.8.0
@@ -221,23 +199,3 @@ jobs:
         run: |
           echo -ne "\n" > crates/benchmarks/benches/datasets_paths.rs
           cargo fmt --all -- --check
-
-  declarative-tests:
-    name: Run declarative tests
-    runs-on: ubuntu-22.04-arm
-    permissions:
-      contents: read
-    steps:
-      - uses: actions/checkout@v5
-      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
-        run: |
-          sudo rm -rf "/opt/ghc" || true
-          sudo rm -rf "/usr/share/dotnet" || true
-          sudo rm -rf "/usr/local/lib/android" || true
-          sudo rm -rf "/usr/local/share/boost" || true
-      - uses: dtolnay/rust-toolchain@1.91.1
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.8.0
-      - name: Run declarative tests
-        run: |
-          cargo xtask test workloads/tests/*.json

.github/workflows/update-cargo-toml-version.yml

@@ -18,13 +18,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
-      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
+      - uses: dtolnay/rust-toolchain@1.89
-        run: |
+        with:
-          sudo rm -rf "/opt/ghc" || true
+          profile: minimal
-          sudo rm -rf "/usr/share/dotnet" || true
-          sudo rm -rf "/usr/local/lib/android" || true
-          sudo rm -rf "/usr/local/share/boost" || true
-      - uses: dtolnay/rust-toolchain@1.91.1
       - name: Install sd
         run: cargo install sd
       - name: Update Cargo.toml file

BENCHMARKS.md

@@ -124,7 +124,6 @@ They are JSON files with the following structure (comments are not actually supp
 {
   // Name of the workload. Must be unique to the workload, as it will be used to group results on the dashboard.
   "name": "hackernews.ndjson_1M,no-threads",
-  "type": "bench",
   // Number of consecutive runs of the commands that should be performed.
   // Each run uses a fresh instance of Meilisearch and a fresh database.
   // Each run produces its own report file.

Cargo.toml

@@ -23,7 +23,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.30.0"
+version = "1.22.2"
 authors = [
     "Quentin de Quelen <quentin@dequelen.me>",
     "Clément Renault <clement@meilisearch.com>",
@@ -50,5 +50,3 @@ opt-level = 3
 opt-level = 3
 [profile.dev.package.roaring]
 opt-level = 3
-[profile.dev.package.gemm-f16]
-opt-level = 3

Cross.toml

@@ -0,0 +1,7 @@
+[build.env]
+passthrough = [
+    "RUST_BACKTRACE",
+    "CARGO_TERM_COLOR",
+    "RUSTFLAGS",
+    "JEMALLOC_SYS_WITH_LG_PAGE"
+]

Dockerfile

@@ -1,5 +1,5 @@
 # Compile
-FROM    rust:1.89-alpine3.22 AS compiler
+FROM    rust:1.89-alpine3.20 AS compiler
 
 RUN     apk add -q --no-cache build-base openssl-dev
 
@@ -8,17 +8,19 @@ WORKDIR /
 ARG     COMMIT_SHA
 ARG     COMMIT_DATE
 ARG     GIT_TAG
-ARG     EXTRA_ARGS
 ENV     VERGEN_GIT_SHA=${COMMIT_SHA} VERGEN_GIT_COMMIT_TIMESTAMP=${COMMIT_DATE} VERGEN_GIT_DESCRIBE=${GIT_TAG}
 ENV     RUSTFLAGS="-C target-feature=-crt-static"
 
 COPY    . .
 RUN     set -eux; \
         apkArch="$(apk --print-arch)"; \
-        cargo build --release -p meilisearch -p meilitool ${EXTRA_ARGS}
+        if [ "$apkArch" = "aarch64" ]; then \
+            export JEMALLOC_SYS_WITH_LG_PAGE=16; \
+        fi && \
+        cargo build --release -p meilisearch -p meilitool
 
 # Run
-FROM    alpine:3.22
+FROM    alpine:3.20
 LABEL   org.opencontainers.image.source="https://github.com/meilisearch/meilisearch"
 
 ENV     MEILI_HTTP_ADDR 0.0.0.0:7700

LICENSE

@@ -1,9 +1,29 @@
-# License
+MIT License
 
 Copyright (c) 2019-2025 Meili SAS
 
-Part of this work fall under the Meilisearch Enterprise Edition (EE) and are licensed under the Business Source License 1.1, please refer to [LICENSE-EE](./LICENSE-EE) for details.
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
 
-The other parts of this work are licensed under the [MIT license](./LICENSE-MIT).
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
 
-`SPDX-License-Identifier: MIT AND BUSL-1.1`
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+---
+
+🔒 Meilisearch Enterprise Edition (EE)
+
+Certain parts of this codebase are not licensed under the MIT license and governed by the Business Source License 1.1.
+
+See the LICENSE-EE file for details.

LICENSE-MIT

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2019-2025 Meili SAS
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

README.md

@@ -39,7 +39,6 @@
 ## 🖥 Examples
 
 - [**Movies**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=organization) — An application to help you find streaming platforms to watch movies using [hybrid search](https://www.meilisearch.com/solutions/hybrid-search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos).
-- [**Flickr**](https://flickr.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=organization) — Search and explore one hundred million Flickr images with semantic search.
 - [**Ecommerce**](https://ecommerce.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Ecommerce website using disjunctive [facets](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos), range and rating filtering, and pagination.
 - [**Songs**](https://music.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search through 47 million of songs.
 - [**SaaS**](https://saas.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search for contacts, deals, and companies in this [multi-tenant](https://www.meilisearch.com/docs/learn/security/multitenancy_tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) CRM application.
@@ -122,7 +121,7 @@ If you want to know more about the kind of data we collect and what we use it fo
 
 Meilisearch is a search engine created by [Meili](https://www.meilisearch.com/careers), a software development company headquartered in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact)
 
-🗞 [Subscribe to our newsletter](https://share-eu1.hsforms.com/1LN5N0x_GQgq7ss7tXmSykwfg3aq) if you don't want to miss any updates! We promise we won't clutter your mailbox: we only send one edition every two months.
+🗞 [Subscribe to our newsletter](https://meilisearch.us2.list-manage.com/subscribe?u=27870f7b71c908a8b359599fb&id=79582d828e) if you don't want to miss any updates! We promise we won't clutter your mailbox: we only send one edition every two months.
 
 💌 Want to make a suggestion or give feedback? Here are some of the channels where you can reach us:
 

TESTING.md

@@ -1,326 +0,0 @@
-# Declarative tests
-
-Declarative tests ensure that Meilisearch features remain stable across versions.
-
-While we already have unit tests, those are run against **temporary databases** that are created fresh each time and therefore never risk corruption.
-
-Declarative tests instead **simulate the lifetime of a database**: they chain together commands and requests to change the binary, verifying that database state and API responses remain consistent.
-
-## Basic example
-
-```jsonc
-{
-  "type": "test",
-  "name": "api-keys",
-  "binary": { // the first command will run on the binary following this specification.
-    "source": "release", // get the binary as a release from GitHub
-    "version": "1.19.0", // version to fetch
-    "edition": "community" // edition to fetch
-  },
-  "commands": []
-}
-```
-
-This example defines a no-op test (it does nothing).
-
-If the file is saved at `workloads/tests/example.json`, you can run it with:
-
-```bash
-cargo xtask test workloads/tests/example.json
-```
-
-## Commands
-
-Commands represent API requests sent to Meilisearch endpoints during a test.
-
-They are executed sequentially, and their responses can be validated to ensure consistent behavior across upgrades.
-
-```jsonc
-
-{
-  "route": "keys",
-  "method": "POST",
-  "body": {
-    "inline": {
-      "actions": [
-        "search",
-        "documents.add"
-      ],
-      "description": "Test API Key",
-      "expiresAt": null,
-      "indexes": [ "movies" ]
-    }
-  }
-}
-```
-
-This command issues a `POST /keys` request, creating an API key with permissions to search and add documents in the `movies` index.
-
-### Using assets in commands
-
-To keep tests concise and reusable, you can define **assets** at the root of the workload file.
-
-Assets are external data sources (such as datasets) that are cached between runs, making tests faster and easier to read.
-
-```jsonc
-{
-  "type": "test",
-  "name": "movies",
-  "binary": {
-    "source": "release",
-    "version": "1.19.0",
-    "edition": "community"
-  },
-  "assets": {
-    "movies.json": {
-      "local_location": null,
-      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/movies.json",
-      "sha256": "5b6e4cb660bc20327776e8a33ea197b43d9ec84856710ead1cc87ab24df77de1"
-    }
-  },
-  "commands": [
-    {
-      "route": "indexes/movies/documents",
-      "method": "POST",
-      "body": {
-        "asset": "movies.json"
-      }
-    }
-  ]
-}
-```
-
-In this example:
-- The `movies.json` dataset is defined as an asset, pointing to a remote URL.
-- The SHA-256 checksum ensures integrity.
-- The `POST /indexes/movies/documents` command uses this asset as the request body.
-
-This makes the test much cleaner than inlining a large dataset directly into the command.
-
-For asset handling, please refer to the [declarative benchmarks documentation](/BENCHMARKS.md#adding-new-assets).
-
-### Asserting responses
-
-Commands can specify both the **expected status code** and the **expected response body**.
-
-```jsonc
-{
-  "route": "indexes/movies/documents",
-  "method": "POST",
-  "body": {
-    "asset": "movies.json"
-  },
-  "expectedStatus": 202,
-  "expectedResponse": {
-    "enqueuedAt": "[timestamp]", // Set to a bracketed string to ignore the value
-    "indexUid": "movies",
-    "status": "enqueued",
-    "taskUid": 1,
-    "type": "documentAdditionOrUpdate"
-  },
-  "synchronous": "WaitForTask"
-}
-```
-
-Manually writing `expectedResponse` fields can be tedious.
-
-Instead, you can let the test runner populate them automatically:
-
-```bash
-# Run the workload to populate expected fields. Only adds the missing ones, doesn't change existing data
-cargo xtask test workloads/tests/example.json --add-missing-responses
-
-# OR
-
-# Run the workload to populate expected fields. Updates all fields including existing ones
-cargo xtask test workloads/tests/example.json --update-responses
-```
-
-This workflow is recommended:
-
-1. Write the test without expected fields.
-2. Run it with `--add-missing-responses` to capture the actual responses.
-3. Review and commit the generated expectations.
-
-## Changing binary
-
-It is possible to insert an instruction to change the current Meilisearch instance from one binary specification to another during a test.
-
-When executed, such an instruction will:
-1. Stop the current Meilisearch instance.
-2. Fetch the binary specified by the instruction.
-3. Restart the server with the specified binary on the same database.
-
-```jsonc
-{
-  "type": "test",
-  "name": "movies",
-  "binary": {
-    "source": "release",
-    "version": "1.19.0", // start with version v1.19.0
-    "edition": "community"
-  },
-  "assets": {
-    "movies.json": {
-      "local_location": null,
-      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/movies.json",
-      "sha256": "5b6e4cb660bc20327776e8a33ea197b43d9ec84856710ead1cc87ab24df77de1"
-    }
-  },
-  "commands": [
-    // setup some data
-    {
-      "route": "indexes/movies/documents",
-      "method": "POST",
-      "body": {
-        "asset": "movies.json"
-      }
-    },
-    // switch binary to v1.24.0
-    {
-      "binary": {
-        "source": "release",
-        "version": "1.24.0",
-        "edition": "community"
-      }
-    }
-  ]
-}
-```
-
-### Typical Usage
-
-In most cases, the change binary instruction will be used to update a database.
-
-- **Set up** some data using commands on an older version.
-- **Upgrade** to the latest version.
-- **Assert** that the data and API behavior remain correct after the upgrade.
-
-To properly test the dumpless upgrade, one should typically:
-
-1. Open the database without processing the update task: Use a `binary` instruction to switch to the desired version, passing `--experimental-dumpless-upgrade` and `--experimental-max-number-of-batched-tasks=0` as extra CLI arguments
-2. Check that the search, stats and task queue still work.
-3. Open the database and process the update task: Use a `binary` instruction to switch to the desired version, passing `--experimental-dumpless-upgrade` as the extra CLI argument. Use a `health` command to wait for the upgrade task to finish.
-4. Check that the indexing, search, stats, and task queue still work.
-
-```jsonc
-{
-  "type": "test",
-  "name": "movies",
-  "binary": {
-    "source": "release",
-    "version": "1.12.0",
-    "edition": "community"
-  },
-  "commands": [
-    // 0. Run commands to populate the database
-    {
-      // ..
-    },
-    // 1. Open the database with new MS without processing the update task
-    {
-      "binary": {
-        "source": "build", // build the binary from the sources in the current git repository
-        "edition": "community",
-        "extraCliArgs": [
-          "--experimental-dumpless-upgrade", // allows to open with a newer MS
-          "--experimental-max-number-of-batched-tasks=0" // prevent processing of the update task
-        ]
-      }
-    },
-    // 2. Check the search etc.
-    {
-      // ..
-    },
-    // 3. Open the database with new MS and processing the update task
-    {
-      "binary": {
-        "source": "build", // build the binary from the sources in the current git repository
-        "edition": "community",
-        "extraCliArgs": [
-          "--experimental-dumpless-upgrade" // allows to open with a newer MS
-          // no `--experimental-max-number-of-batched-tasks=0`
-        ]
-      }
-    },
-    // 4. Check the indexing, search, etc.
-    {
-      // ..
-    }
-  ]
-}
-```
-
-This ensures backward compatibility: databases created with older Meilisearch versions should remain functional and consistent after an upgrade.
-
-## Variables
-
-Sometimes a command needs to use a value returned by a **previous response**.
-These values can be captured and reused using the register field.
-
-```jsonc
-{
-  "route": "keys",
-  "method": "POST",
-  "body": {
-    "inline": {
-        "actions": [
-        "search",
-        "documents.add"
-        ],
-        "description": "Test API Key",
-        "expiresAt": null,
-        "indexes": [ "movies" ]
-    }
-  },
-  "expectedResponse": {
-      "key": "c6f64630bad2996b1f675007c8800168e14adf5d6a7bb1a400a6d2b158050eaf",
-      // ...
-  },
-  "register": {
-    "key": "/key"
-  },
-  "synchronous": "WaitForResponse"
-}
-```
-
-The `register` field captures the value at the JSON path `/key` from the response.
-Paths follow the **JavaScript Object Notation Pointer (RFC 6901)** format.
-Registered variables are available for all subsequent commands.
-
-Registered variables can be referenced by wrapping their name in double curly braces:
-
-In the route/path:
-
-```jsonc
-{
-  "route": "tasks/{{ task_id }}",
-  "method": "GET"
-}
-```
-
-In the request body:
-
-```jsonc
-{
-  "route": "indexes/movies/documents",
-  "method": "PATCH",
-  "body": {
-    "inline": {
-      "id": "{{ document_id }}",
-      "overview": "Shazam turns evil and the world is in danger.",
-    }
-  }
-}
-```
-
-Or they can be referenced by their name (**without curly braces**) as an API key:
-
-```jsonc
-{
-  "route": "indexes/movies/documents",
-  "method": "POST",
-  "body": { /* ... */ },
-  "apiKeyVariable": "key" // The **content** of the key variable will be used as an API key
-}
-```

crates/benchmarks/Cargo.toml

@@ -11,27 +11,27 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.100"
+anyhow = "1.0.98"
-bumpalo = "3.19.0"
+bumpalo = "3.18.1"
-csv = "1.4.0"
+csv = "1.3.1"
-memmap2 = "0.9.9"
+memmap2 = "0.9.7"
 milli = { path = "../milli" }
-mimalloc = { version = "0.1.48", default-features = false }
+mimalloc = { version = "0.1.47", default-features = false }
-serde_json = { version = "1.0.145", features = ["preserve_order"] }
+serde_json = { version = "1.0.140", features = ["preserve_order"] }
-tempfile = "3.23.0"
+tempfile = "3.20.0"
 
 [dev-dependencies]
-criterion = { version = "0.7.0", features = ["html_reports"] }
+criterion = { version = "0.6.0", features = ["html_reports"] }
 rand = "0.8.5"
 rand_chacha = "0.3.1"
 roaring = "0.10.12"
 
 [build-dependencies]
-anyhow = "1.0.100"
+anyhow = "1.0.98"
-bytes = "1.11.0"
+bytes = "1.10.1"
-convert_case = "0.9.0"
+convert_case = "0.8.0"
-flate2 = "1.1.5"
+flate2 = "1.1.2"
-reqwest = { version = "0.12.24", features = ["blocking", "rustls-tls"], default-features = false }
+reqwest = { version = "0.12.20", features = ["blocking", "rustls-tls"], default-features = false }
 
 [features]
 default = ["milli/all-tokenizations"]

crates/benchmarks/benches/indexing.rs

@@ -21,10 +21,6 @@ use roaring::RoaringBitmap;
 #[global_allocator]
 static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
-fn no_cancel() -> bool {
-    false
-}
-
 const BENCHMARK_ITERATION: usize = 10;
 
 fn setup_dir(path: impl AsRef<Path>) {
@@ -69,7 +65,7 @@ fn setup_settings<'t>(
     let sortable_fields = sortable_fields.iter().map(|s| s.to_string()).collect();
     builder.set_sortable_fields(sortable_fields);
 
-    builder.execute(&no_cancel, &Progress::default(), Default::default()).unwrap();
+    builder.execute(&|| false, &Progress::default(), Default::default()).unwrap();
 }
 
 fn setup_index_with_settings(
@@ -156,7 +152,7 @@ fn indexing_songs_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -172,7 +168,7 @@ fn indexing_songs_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -224,7 +220,7 @@ fn reindexing_songs_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -240,7 +236,7 @@ fn reindexing_songs_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -270,7 +266,7 @@ fn reindexing_songs_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -286,7 +282,7 @@ fn reindexing_songs_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -340,7 +336,7 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -356,7 +352,7 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -418,7 +414,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -434,7 +430,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -464,7 +460,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -480,7 +476,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -506,7 +502,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -522,7 +518,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -575,7 +571,7 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -591,7 +587,7 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -643,7 +639,7 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -659,7 +655,7 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -711,7 +707,7 @@ fn indexing_wiki(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -727,7 +723,7 @@ fn indexing_wiki(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -778,7 +774,7 @@ fn reindexing_wiki(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -794,7 +790,7 @@ fn reindexing_wiki(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -824,7 +820,7 @@ fn reindexing_wiki(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -840,7 +836,7 @@ fn reindexing_wiki(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -893,7 +889,7 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -909,7 +905,7 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -971,7 +967,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -987,7 +983,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1018,7 +1014,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -1034,7 +1030,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1061,7 +1057,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -1077,7 +1073,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1129,7 +1125,7 @@ fn indexing_movies_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -1145,7 +1141,7 @@ fn indexing_movies_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1196,7 +1192,7 @@ fn reindexing_movies_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -1212,7 +1208,7 @@ fn reindexing_movies_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1242,7 +1238,7 @@ fn reindexing_movies_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -1258,7 +1254,7 @@ fn reindexing_movies_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1311,7 +1307,7 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -1327,7 +1323,7 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1376,7 +1372,7 @@ fn delete_documents_from_ids(index: Index, document_ids_to_delete: Vec<RoaringBi
             Some(primary_key),
             &document_changes,
             RuntimeEmbedders::default(),
-            &no_cancel,
+            &|| false,
             &Progress::default(),
             &Default::default(),
         )
@@ -1426,7 +1422,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -1442,7 +1438,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1472,7 +1468,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -1488,7 +1484,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1514,7 +1510,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -1530,7 +1526,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1605,7 +1601,7 @@ fn indexing_nested_movies_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -1621,7 +1617,7 @@ fn indexing_nested_movies_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1697,7 +1693,7 @@ fn deleting_nested_movies_in_batches_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -1713,7 +1709,7 @@ fn deleting_nested_movies_in_batches_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1781,7 +1777,7 @@ fn indexing_nested_movies_without_faceted_fields(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -1797,7 +1793,7 @@ fn indexing_nested_movies_without_faceted_fields(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1849,7 +1845,7 @@ fn indexing_geo(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -1865,7 +1861,7 @@ fn indexing_geo(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1916,7 +1912,7 @@ fn reindexing_geo(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -1932,7 +1928,7 @@ fn reindexing_geo(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1962,7 +1958,7 @@ fn reindexing_geo(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -1978,7 +1974,7 @@ fn reindexing_geo(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -2031,7 +2027,7 @@ fn deleting_geo_in_batches_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &no_cancel,
+                        &|| false,
                         Progress::default(),
                         None,
                     )
@@ -2047,7 +2043,7 @@ fn deleting_geo_in_batches_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &no_cancel,
+                    &|| false,
                     &Progress::default(),
                     &Default::default(),
                 )

crates/build-info/Cargo.toml

@@ -11,8 +11,8 @@ license.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-time = { version = "0.3.44", features = ["parsing"] }
+time = { version = "0.3.41", features = ["parsing"] }
 
 [build-dependencies]
-anyhow = "1.0.100"
+anyhow = "1.0.98"
-vergen-gitcl = "1.0.8"
+vergen-git2 = "1.0.7"

crates/build-info/build.rs

@@ -15,7 +15,7 @@ fn emit_git_variables() -> anyhow::Result<()> {
     // Note: any code that needs VERGEN_ environment variables should take care to define them manually in the Dockerfile and pass them
     // in the corresponding GitHub workflow (publish_docker.yml).
     // This is due to the Dockerfile building the binary outside of the git directory.
-    let mut builder = vergen_gitcl::GitclBuilder::default();
+    let mut builder = vergen_git2::Git2Builder::default();
 
     builder.branch(true);
     builder.commit_timestamp(true);
@@ -25,5 +25,5 @@ fn emit_git_variables() -> anyhow::Result<()> {
 
     let git2 = builder.build()?;
 
-    vergen_gitcl::Emitter::default().fail_on_error().add_instructions(&git2)?.emit()
+    vergen_git2::Emitter::default().fail_on_error().add_instructions(&git2)?.emit()
 }

crates/build-info/src/main.rs

@@ -1,6 +0,0 @@
-use build_info::BuildInfo;
-
-fn main() {
-    let info = BuildInfo::from_build();
-    dbg!(info);
-}

crates/dump/Cargo.toml

@@ -11,27 +11,24 @@ readme.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.100"
+anyhow = "1.0.98"
-flate2 = "1.1.5"
+flate2 = "1.1.2"
 http = "1.3.1"
 meilisearch-types = { path = "../meilisearch-types" }
 once_cell = "1.21.3"
-regex = "1.12.2"
+regex = "1.11.1"
 roaring = { version = "0.10.12", features = ["serde"] }
-serde = { version = "1.0.228", features = ["derive"] }
+serde = { version = "1.0.219", features = ["derive"] }
-serde_json = { version = "1.0.145", features = ["preserve_order"] }
+serde_json = { version = "1.0.140", features = ["preserve_order"] }
 tar = "0.4.44"
-tempfile = "3.23.0"
+tempfile = "3.20.0"
-thiserror = "2.0.17"
+thiserror = "2.0.12"
-time = { version = "0.3.44", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+time = { version = "0.3.41", features = ["serde-well-known", "formatting", "parsing", "macros"] }
 tracing = "0.1.41"
-uuid = { version = "1.18.1", features = ["serde", "v4"] }
+uuid = { version = "1.17.0", features = ["serde", "v4"] }
 
 [dev-dependencies]
 big_s = "1.0.2"
 maplit = "1.0.2"
 meili-snap = { path = "../meili-snap" }
 meilisearch-types = { path = "../meilisearch-types" }
-
-[features]
-enterprise = ["meilisearch-types/enterprise"]

crates/dump/src/lib.rs

@@ -9,9 +9,8 @@ use meilisearch_types::error::ResponseError;
 use meilisearch_types::keys::Key;
 use meilisearch_types::milli::update::IndexDocumentsMethod;
 use meilisearch_types::settings::Unchecked;
-use meilisearch_types::tasks::network::{DbTaskNetwork, NetworkTopologyChange};
 use meilisearch_types::tasks::{
-    Details, ExportIndexSettings, IndexSwap, KindWithContent, Status, Task, TaskId,
+    Details, ExportIndexSettings, IndexSwap, KindWithContent, Status, Task, TaskId, TaskNetwork,
 };
 use meilisearch_types::InstanceUid;
 use roaring::RoaringBitmap;
@@ -96,9 +95,7 @@ pub struct TaskDump {
     )]
     pub finished_at: Option<OffsetDateTime>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub network: Option<DbTaskNetwork>,
+    pub network: Option<TaskNetwork>,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub custom_metadata: Option<String>,
 }
 
 // A `Kind` specific version made for the dump. If modified you may break the dump.
@@ -161,10 +158,6 @@ pub enum KindDump {
     UpgradeDatabase {
         from: (u32, u32, u32),
     },
-    IndexCompaction {
-        index_uid: String,
-    },
-    NetworkTopologyChange(NetworkTopologyChange),
 }
 
 impl From<Task> for TaskDump {
@@ -182,7 +175,6 @@ impl From<Task> for TaskDump {
             started_at: task.started_at,
             finished_at: task.finished_at,
             network: task.network,
-            custom_metadata: task.custom_metadata,
         }
     }
 }
@@ -248,12 +240,6 @@ impl From<KindWithContent> for KindDump {
             KindWithContent::UpgradeDatabase { from: version } => {
                 KindDump::UpgradeDatabase { from: version }
             }
-            KindWithContent::IndexCompaction { index_uid } => {
-                KindDump::IndexCompaction { index_uid }
-            }
-            KindWithContent::NetworkTopologyChange(network_topology_change) => {
-                KindDump::NetworkTopologyChange(network_topology_change)
-            }
         }
     }
 }
@@ -267,13 +253,13 @@ pub(crate) mod test {
     use big_s::S;
     use maplit::{btreemap, btreeset};
     use meilisearch_types::batches::{Batch, BatchEnqueuedAt, BatchStats};
+    use meilisearch_types::enterprise_edition::network::{Network, Remote};
     use meilisearch_types::facet_values_sort::FacetValuesSort;
     use meilisearch_types::features::RuntimeTogglableFeatures;
     use meilisearch_types::index_uid_pattern::IndexUidPattern;
     use meilisearch_types::keys::{Action, Key};
     use meilisearch_types::milli::update::Setting;
     use meilisearch_types::milli::{self, FilterableAttributesRule};
-    use meilisearch_types::network::{Network, Remote};
     use meilisearch_types::settings::{Checked, FacetingSettings, Settings};
     use meilisearch_types::task_view::DetailsView;
     use meilisearch_types::tasks::{BatchStopReason, Details, Kind, Status};
@@ -404,7 +390,6 @@ pub(crate) mod test {
                     started_at: Some(datetime!(2022-11-20 0:00 UTC)),
                     finished_at: Some(datetime!(2022-11-21 0:00 UTC)),
                     network: None,
-                    custom_metadata: None,
                 },
                 None,
             ),
@@ -430,7 +415,6 @@ pub(crate) mod test {
                     started_at: None,
                     finished_at: None,
                     network: None,
-                    custom_metadata: None,
                 },
                 Some(vec![
                     json!({ "id": 4, "race": "leonberg" }).as_object().unwrap().clone(),
@@ -451,7 +435,6 @@ pub(crate) mod test {
                     started_at: None,
                     finished_at: None,
                     network: None,
-                    custom_metadata: None,
                 },
                 None,
             ),
@@ -565,8 +548,7 @@ pub(crate) mod test {
         Network {
             local: Some("myself".to_string()),
             remotes: maplit::btreemap! {"other".to_string() => Remote { url: "http://test".to_string(), search_api_key: Some("apiKey".to_string()), write_api_key: Some("docApiKey".to_string()) }},
-            leader: None,
+            sharding: false,
-            version: Default::default(),
         }
     }
 
@@ -620,10 +602,7 @@ pub(crate) mod test {
         assert_eq!(dump.features().unwrap().unwrap(), expected);
 
         // ==== checking the network
-        let mut expected = create_test_network();
+        let expected = create_test_network();
-        // from v1.29, we drop `leader` and `local` on import
-        expected.leader = None;
-        expected.local = None;
         assert_eq!(&expected, dump.network().unwrap().unwrap());
     }
 }

crates/dump/src/reader/compat/v5_to_v6.rs

@@ -164,7 +164,6 @@ impl CompatV5ToV6 {
                     started_at: task_view.started_at,
                     finished_at: task_view.finished_at,
                     network: None,
-                    custom_metadata: None,
                 };
 
                 (task, content_file)

crates/dump/src/reader/mod.rs

@@ -434,11 +434,7 @@ pub(crate) mod test {
         // network
 
         let network = dump.network().unwrap().unwrap();
-
+        insta::assert_snapshot!(network.local.as_ref().unwrap(), @"ms-0");
-        // since v1.29 we are dropping `local` and `leader` on import
-        insta::assert_snapshot!(network.local.is_none(), @"true");
-        insta::assert_snapshot!(network.leader.is_none(), @"true");
-
         insta::assert_snapshot!(network.remotes.get("ms-0").as_ref().unwrap().url, @"http://localhost:7700");
         insta::assert_snapshot!(network.remotes.get("ms-0").as_ref().unwrap().search_api_key.is_none(), @"true");
         insta::assert_snapshot!(network.remotes.get("ms-1").as_ref().unwrap().url, @"http://localhost:7701");

crates/dump/src/reader/v2/settings.rs

@@ -107,14 +107,19 @@ impl Settings<Unchecked> {
     }
 }
 
-#[derive(Default, Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq)]
 pub enum Setting<T> {
     Set(T),
     Reset,
-    #[default]
     NotSet,
 }
 
+impl<T> Default for Setting<T> {
+    fn default() -> Self {
+        Self::NotSet
+    }
+}
+
 impl<T> Setting<T> {
     pub const fn is_not_set(&self) -> bool {
         matches!(self, Self::NotSet)

crates/dump/src/reader/v3/settings.rs

@@ -161,14 +161,19 @@ pub struct Facets {
     pub min_level_size: Option<NonZeroUsize>,
 }
 
-#[derive(Default, Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Setting<T> {
     Set(T),
     Reset,
-    #[default]
     NotSet,
 }
 
+impl<T> Default for Setting<T> {
+    fn default() -> Self {
+        Self::NotSet
+    }
+}
+
 impl<T> Setting<T> {
     pub fn map<U, F>(self, f: F) -> Setting<U>
     where

crates/dump/src/reader/v4/meta.rs

@@ -1,7 +1,9 @@
 use std::fmt::{self, Display, Formatter};
+use std::marker::PhantomData;
 use std::str::FromStr;
 
-use serde::Deserialize;
+use serde::de::Visitor;
+use serde::{Deserialize, Deserializer};
 use uuid::Uuid;
 
 use super::settings::{Settings, Unchecked};
@@ -80,3 +82,59 @@ impl Display for IndexUidFormatError {
 }
 
 impl std::error::Error for IndexUidFormatError {}
+
+/// A type that tries to match either a star (*) or
+/// any other thing that implements `FromStr`.
+#[derive(Debug)]
+#[cfg_attr(test, derive(serde::Serialize))]
+pub enum StarOr<T> {
+    Star,
+    Other(T),
+}
+
+impl<'de, T, E> Deserialize<'de> for StarOr<T>
+where
+    T: FromStr<Err = E>,
+    E: Display,
+{
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        /// Serde can't differentiate between `StarOr::Star` and `StarOr::Other` without a tag.
+        /// Simply using `#[serde(untagged)]` + `#[serde(rename="*")]` will lead to attempting to
+        /// deserialize everything as a `StarOr::Other`, including "*".
+        /// [`#[serde(other)]`](https://serde.rs/variant-attrs.html#other) might have helped but is
+        /// not supported on untagged enums.
+        struct StarOrVisitor<T>(PhantomData<T>);
+
+        impl<T, FE> Visitor<'_> for StarOrVisitor<T>
+        where
+            T: FromStr<Err = FE>,
+            FE: Display,
+        {
+            type Value = StarOr<T>;
+
+            fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result {
+                formatter.write_str("a string")
+            }
+
+            fn visit_str<SE>(self, v: &str) -> Result<Self::Value, SE>
+            where
+                SE: serde::de::Error,
+            {
+                match v {
+                    "*" => Ok(StarOr::Star),
+                    v => {
+                        let other = FromStr::from_str(v).map_err(|e: T::Err| {
+                            SE::custom(format!("Invalid `other` value: {}", e))
+                        })?;
+                        Ok(StarOr::Other(other))
+                    }
+                }
+            }
+        }
+
+        deserializer.deserialize_str(StarOrVisitor(PhantomData))
+    }
+}

crates/dump/src/reader/v4/settings.rs

@@ -192,14 +192,19 @@ pub struct Facets {
     pub min_level_size: Option<NonZeroUsize>,
 }
 
-#[derive(Default, Debug, Clone, PartialEq, Eq, Copy)]
+#[derive(Debug, Clone, PartialEq, Eq, Copy)]
 pub enum Setting<T> {
     Set(T),
     Reset,
-    #[default]
     NotSet,
 }
 
+impl<T> Default for Setting<T> {
+    fn default() -> Self {
+        Self::NotSet
+    }
+}
+
 impl<T> Setting<T> {
     pub fn set(self) -> Option<T> {
         match self {

crates/dump/src/reader/v5/settings.rs

@@ -47,15 +47,20 @@ pub struct Settings<T> {
     pub _kind: PhantomData<T>,
 }
 
-#[derive(Default, Debug, Clone, PartialEq, Eq, Copy)]
+#[derive(Debug, Clone, PartialEq, Eq, Copy)]
 #[cfg_attr(test, derive(serde::Serialize))]
 pub enum Setting<T> {
     Set(T),
     Reset,
-    #[default]
     NotSet,
 }
 
+impl<T> Default for Setting<T> {
+    fn default() -> Self {
+        Self::NotSet
+    }
+}
+
 impl<T> Setting<T> {
     pub fn set(self) -> Option<T> {
         match self {

crates/dump/src/reader/v5/tasks.rs

@@ -322,7 +322,7 @@ impl From<Task> for TaskView {
             _ => None,
         });
 
-        let duration = finished_at.zip(started_at).map(|(tf, ts)| tf - ts);
+        let duration = finished_at.zip(started_at).map(|(tf, ts)| (tf - ts));
 
         Self {
             uid: id,

crates/dump/src/reader/v6/mod.rs

@@ -24,7 +24,7 @@ pub type Batch = meilisearch_types::batches::Batch;
 pub type Key = meilisearch_types::keys::Key;
 pub type ChatCompletionSettings = meilisearch_types::features::ChatCompletionSettings;
 pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures;
-pub type Network = meilisearch_types::network::Network;
+pub type Network = meilisearch_types::enterprise_edition::network::Network;
 pub type Webhooks = meilisearch_types::webhooks::WebhooksDumpView;
 
 // ===== Other types to clarify the code of the compat module
@@ -95,26 +95,17 @@ impl V6Reader {
             Err(e) => return Err(e.into()),
         };
 
-        let mut network: Option<meilisearch_types::network::Network> =
+        let network = match fs::read(dump.path().join("network.json")) {
-            match fs::read(dump.path().join("network.json")) {
+            Ok(network_file) => Some(serde_json::from_reader(&*network_file)?),
-                Ok(network_file) => Some(serde_json::from_reader(&*network_file)?),
+            Err(error) => match error.kind() {
-                Err(error) => match error.kind() {
+                // Allows the file to be missing, this will only result in all experimental features disabled.
-                    // Allows the file to be missing, this will only result in all experimental features disabled.
+                ErrorKind::NotFound => {
-                    ErrorKind::NotFound => {
+                    debug!("`network.json` not found in dump");
-                        debug!("`network.json` not found in dump");
+                    None
-                        None
+                }
-                    }
+                _ => return Err(error.into()),
-                    _ => return Err(error.into()),
+            },
-                },
+        };
-            };
-
-        if let Some(network) = &mut network {
-            // as dumps are typically imported in a different machine as the emitter (otherwise dumpless upgrade would be used),
-            // we decide to remove the self to avoid alias issues
-            network.local = None;
-            // for the same reason we disable automatic sharding
-            network.leader = None;
-        }
 
         let webhooks = match fs::read(dump.path().join("webhooks.json")) {
             Ok(webhooks_file) => Some(serde_json::from_reader(&*webhooks_file)?),

crates/dump/src/writer.rs

@@ -5,9 +5,9 @@ use std::path::PathBuf;
 use flate2::write::GzEncoder;
 use flate2::Compression;
 use meilisearch_types::batches::Batch;
+use meilisearch_types::enterprise_edition::network::Network;
 use meilisearch_types::features::{ChatCompletionSettings, RuntimeTogglableFeatures};
 use meilisearch_types::keys::Key;
-use meilisearch_types::network::Network;
 use meilisearch_types::settings::{Checked, Settings};
 use meilisearch_types::webhooks::WebhooksDumpView;
 use serde_json::{Map, Value};

crates/file-store/Cargo.toml

@@ -11,7 +11,7 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-tempfile = "3.23.0"
+tempfile = "3.20.0"
-thiserror = "2.0.17"
+thiserror = "2.0.12"
 tracing = "0.1.41"
-uuid = { version = "1.18.1", features = ["serde", "v4"] }
+uuid = { version = "1.17.0", features = ["serde", "v4"] }

crates/file-store/src/lib.rs

@@ -60,7 +60,7 @@ impl FileStore {
 
     /// Returns the file corresponding to the requested uuid.
     pub fn get_update(&self, uuid: Uuid) -> Result<StdFile> {
-        let path = self.update_path(uuid);
+        let path = self.get_update_path(uuid);
         let file = match StdFile::open(path) {
             Ok(file) => file,
             Err(e) => {
@@ -72,7 +72,7 @@ impl FileStore {
     }
 
     /// Returns the path that correspond to this uuid, the path could not exists.
-    pub fn update_path(&self, uuid: Uuid) -> PathBuf {
+    pub fn get_update_path(&self, uuid: Uuid) -> PathBuf {
         self.path.join(uuid.to_string())
     }
 

crates/flatten-serde-json/Cargo.toml

@@ -16,7 +16,7 @@ license.workspace = true
 serde_json = "1.0"
 
 [dev-dependencies]
-criterion = { version = "0.7.0", features = ["html_reports"] }
+criterion = { version = "0.6.0", features = ["html_reports"] }
 
 [[bench]]
 name = "benchmarks"

crates/fuzzers/Cargo.toml

@@ -11,12 +11,12 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-arbitrary = { version = "1.4.2", features = ["derive"] }
+arbitrary = { version = "1.4.1", features = ["derive"] }
-bumpalo = "3.19.0"
+bumpalo = "3.18.1"
-clap = { version = "4.5.52", features = ["derive"] }
+clap = { version = "4.5.40", features = ["derive"] }
 either = "1.15.0"
 fastrand = "2.3.0"
 milli = { path = "../milli" }
-serde = { version = "1.0.228", features = ["derive"] }
+serde = { version = "1.0.219", features = ["derive"] }
-serde_json = { version = "1.0.145", features = ["preserve_order"] }
+serde_json = { version = "1.0.140", features = ["preserve_order"] }
-tempfile = "3.23.0"
+tempfile = "3.20.0"

crates/index-scheduler/Cargo.toml

@@ -11,34 +11,31 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.100"
+anyhow = "1.0.98"
 bincode = "1.3.3"
 byte-unit = "5.1.6"
-bytes = "1.11.0"
+bumpalo = "3.18.1"
-bumpalo = "3.19.0"
 bumparaw-collections = "0.1.4"
-convert_case = "0.9.0"
+convert_case = "0.8.0"
-csv = "1.4.0"
+csv = "1.3.1"
 derive_builder = "0.20.2"
 dump = { path = "../dump" }
-enum-iterator = "2.3.0"
+enum-iterator = "2.1.0"
 file-store = { path = "../file-store" }
-flate2 = "1.1.5"
+flate2 = "1.1.2"
-hashbrown = "0.15.5"
+indexmap = "2.9.0"
-indexmap = "2.12.0"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
-memmap2 = "0.9.9"
+memmap2 = "0.9.7"
 page_size = "0.6.0"
-rayon = "1.11.0"
+rayon = "1.10.0"
 roaring = { version = "0.10.12", features = ["serde"] }
-serde = { version = "1.0.228", features = ["derive"] }
+serde = { version = "1.0.219", features = ["derive"] }
-serde_json = { version = "1.0.145", features = ["preserve_order"] }
+serde_json = { version = "1.0.140", features = ["preserve_order"] }
-tar = "0.4.44"
 synchronoise = "1.0.1"
-tempfile = "3.23.0"
+tempfile = "3.20.0"
-thiserror = "2.0.17"
+thiserror = "2.0.12"
-time = { version = "0.3.44", features = [
+time = { version = "0.3.41", features = [
     "serde-well-known",
     "formatting",
     "parsing",
@@ -46,15 +43,8 @@ time = { version = "0.3.44", features = [
 ] }
 tracing = "0.1.41"
 ureq = "2.12.1"
-uuid = { version = "1.18.1", features = ["serde", "v4"] }
+uuid = { version = "1.17.0", features = ["serde", "v4"] }
 backoff = "0.4.0"
-reqwest = { version = "0.12.24", features = [
-    "rustls-tls",
-    "http2",
-], default-features = false }
-rusty-s3 = "0.8.1"
-tokio = { version = "1.48.0", features = ["full"] }
-urlencoding = "2.1.3"
 
 [dev-dependencies]
 big_s = "1.0.2"
@@ -63,6 +53,3 @@ crossbeam-channel = "0.5.15"
 insta = { version = "=1.39.0", features = ["json", "redactions"] }
 maplit = "1.0.2"
 meili-snap = { path = "../meili-snap" }
-
-[features]
-enterprise = ["meilisearch-types/enterprise"]

crates/index-scheduler/src/dump.rs

@@ -150,7 +150,6 @@ impl<'a> Dump<'a> {
             details: task.details,
             status: task.status,
             network: task.network,
-            custom_metadata: task.custom_metadata,
             kind: match task.kind {
                 KindDump::DocumentImport {
                     primary_key,
@@ -235,12 +234,6 @@ impl<'a> Dump<'a> {
                     }
                 }
                 KindDump::UpgradeDatabase { from } => KindWithContent::UpgradeDatabase { from },
-                KindDump::IndexCompaction { index_uid } => {
-                    KindWithContent::IndexCompaction { index_uid }
-                }
-                KindDump::NetworkTopologyChange(network_topology_change) => {
-                    KindWithContent::NetworkTopologyChange(network_topology_change)
-                }
             },
         };
 

crates/index-scheduler/src/error.rs

@@ -3,13 +3,9 @@ use std::fmt::Display;
 use meilisearch_types::batches::BatchId;
 use meilisearch_types::error::{Code, ErrorCode};
 use meilisearch_types::milli::index::RollbackOutcome;
-use meilisearch_types::milli::DocumentId;
-use meilisearch_types::tasks::network::ReceiveTaskError;
 use meilisearch_types::tasks::{Kind, Status};
 use meilisearch_types::{heed, milli};
-use reqwest::StatusCode;
 use thiserror::Error;
-use uuid::Uuid;
 
 use crate::TaskId;
 
@@ -131,14 +127,6 @@ pub enum Error {
     #[error("Aborted task")]
     AbortedTask,
 
-    #[error("S3 error: status: {status}, body: {body}")]
-    S3Error { status: StatusCode, body: String },
-    #[error("S3 HTTP error: {0}")]
-    S3HttpError(reqwest::Error),
-    #[error("S3 XML error: {0}")]
-    S3XmlError(Box<dyn std::error::Error + Send + Sync>),
-    #[error("S3 bucket error: {0}")]
-    S3BucketError(rusty_s3::BucketError),
     #[error(transparent)]
     Dump(#[from] dump::Error),
     #[error(transparent)]
@@ -194,17 +182,6 @@ pub enum Error {
     #[error(transparent)]
     HeedTransaction(heed::Error),
 
-    #[error("No network topology change task is currently enqueued or processing")]
-    ImportTaskWithoutNetworkTask,
-    #[error("The network task version (`{network_task}`) does not match the import task version (`{import_task}`)")]
-    NetworkVersionMismatch { network_task: Uuid, import_task: Uuid },
-    #[error("The import task emanates from an unknown remote `{0}`")]
-    ImportTaskUnknownRemote(String),
-    #[error("The import task with key `{0}` was already received")]
-    ImportTaskAlreadyReceived(DocumentId),
-    #[error("{action} requires the Enterprise Edition")]
-    RequiresEnterpriseEdition { action: &'static str },
-
     #[cfg(test)]
     #[error("Planned failure for tests.")]
     PlannedFailure,
@@ -249,10 +226,6 @@ impl Error {
             | Error::TaskCancelationWithEmptyQuery
             | Error::FromRemoteWhenExporting { .. }
             | Error::AbortedTask
-            | Error::S3Error { .. }
-            | Error::S3HttpError(_)
-            | Error::S3XmlError(_)
-            | Error::S3BucketError(_)
             | Error::Dump(_)
             | Error::Heed(_)
             | Error::Milli { .. }
@@ -262,11 +235,6 @@ impl Error {
             | Error::Persist(_)
             | Error::FeatureNotEnabled(_)
             | Error::Export(_)
-            | Error::ImportTaskWithoutNetworkTask
-            | Error::NetworkVersionMismatch { .. }
-            | Error::ImportTaskAlreadyReceived(_)
-            | Error::ImportTaskUnknownRemote(_)
-            | Error::RequiresEnterpriseEdition { .. }
             | Error::Anyhow(_) => true,
             Error::CreateBatch(_)
             | Error::CorruptedTaskQueue
@@ -325,19 +293,8 @@ impl ErrorCode for Error {
             Error::BatchNotFound(_) => Code::BatchNotFound,
             Error::TaskDeletionWithEmptyQuery => Code::MissingTaskFilters,
             Error::TaskCancelationWithEmptyQuery => Code::MissingTaskFilters,
+            // TODO: not sure of the Code to use
             Error::NoSpaceLeftInTaskQueue => Code::NoSpaceLeftOnDevice,
-            Error::ImportTaskWithoutNetworkTask => Code::ImportTaskWithoutNetworkTask,
-            Error::NetworkVersionMismatch { .. } => Code::NetworkVersionMismatch,
-            Error::ImportTaskAlreadyReceived(_) => Code::ImportTaskAlreadyReceived,
-            Error::ImportTaskUnknownRemote(_) => Code::ImportTaskUnknownRemote,
-            Error::RequiresEnterpriseEdition { .. } => Code::RequiresEnterpriseEdition,
-            Error::S3Error { status, .. } if status.is_client_error() => {
-                Code::InvalidS3SnapshotRequest
-            }
-            Error::S3Error { .. } => Code::S3SnapshotServerError,
-            Error::S3HttpError(_) => Code::S3SnapshotServerError,
-            Error::S3XmlError(_) => Code::S3SnapshotServerError,
-            Error::S3BucketError(_) => Code::InvalidS3SnapshotParameters,
             Error::Dump(e) => e.error_code(),
             Error::Milli { error, .. } => error.error_code(),
             Error::ProcessBatchPanicked(_) => Code::Internal,
@@ -369,12 +326,3 @@ impl ErrorCode for Error {
         }
     }
 }
-
-impl From<ReceiveTaskError> for Error {
-    fn from(value: ReceiveTaskError) -> Self {
-        match value {
-            ReceiveTaskError::UnknownRemote(unknown) => Error::ImportTaskUnknownRemote(unknown),
-            ReceiveTaskError::DuplicateTask(dup) => Error::ImportTaskAlreadyReceived(dup),
-        }
-    }
-}

crates/index-scheduler/src/features.rs

@@ -1,9 +1,9 @@
 use std::sync::{Arc, RwLock};
 
+use meilisearch_types::enterprise_edition::network::Network;
 use meilisearch_types::features::{InstanceTogglableFeatures, RuntimeTogglableFeatures};
 use meilisearch_types::heed::types::{SerdeJson, Str};
 use meilisearch_types::heed::{Database, Env, RwTxn, WithoutTls};
-use meilisearch_types::network::Network;
 
 use crate::error::FeatureNotEnabledError;
 use crate::Result;
@@ -38,10 +38,6 @@ impl RoFeatures {
         Self { runtime }
     }
 
-    pub fn from_runtime_features(features: RuntimeTogglableFeatures) -> Self {
-        Self { runtime: features }
-    }
-
     pub fn runtime_features(&self) -> RuntimeTogglableFeatures {
         self.runtime
     }

crates/index-scheduler/src/index_mapper/mod.rs

@@ -199,7 +199,7 @@ impl IndexMapper {
                 let uuid = Uuid::new_v4();
                 self.index_mapping.put(&mut wtxn, name, &uuid)?;
 
-                let index_path = self.index_path(uuid);
+                let index_path = self.base_path.join(uuid.to_string());
                 fs::create_dir_all(&index_path)?;
 
                 // Error if the UUIDv4 somehow already exists in the map, since it should be fresh.
@@ -286,7 +286,7 @@ impl IndexMapper {
         };
 
         let index_map = self.index_map.clone();
-        let index_path = self.index_path(uuid);
+        let index_path = self.base_path.join(uuid.to_string());
         let index_name = name.to_string();
         thread::Builder::new()
             .name(String::from("index_deleter"))
@@ -341,32 +341,6 @@ impl IndexMapper {
         Ok(())
     }
 
-    /// Closes the specified index.
-    ///
-    /// This operation involves closing the underlying environment and so can take a long time to complete.
-    ///
-    /// # Panics
-    ///
-    /// - If the Index corresponding to the passed name is concurrently being deleted/resized or cannot be found in the
-    ///   in memory hash map.
-    pub fn close_index(&self, rtxn: &RoTxn, name: &str) -> Result<()> {
-        let uuid = self
-            .index_mapping
-            .get(rtxn, name)?
-            .ok_or_else(|| Error::IndexNotFound(name.to_string()))?;
-
-        // We remove the index from the in-memory index map.
-        self.index_map.write().unwrap().close_for_resize(&uuid, self.enable_mdb_writemap, 0);
-
-        Ok(())
-    }
-
-    /// The number of indexes in the database
-    #[cfg(feature = "enterprise")] // only used in enterprise edition for now
-    pub fn index_count(&self, rtxn: &RoTxn) -> Result<u64> {
-        Ok(self.index_mapping.len(rtxn)?)
-    }
-
     /// Return an index, may open it if it wasn't already opened.
     pub fn index(&self, rtxn: &RoTxn, name: &str) -> Result<Index> {
         if let Some((current_name, current_index)) =
@@ -414,7 +388,7 @@ impl IndexMapper {
                     } else {
                         continue;
                     };
-                    let index_path = self.index_path(uuid);
+                    let index_path = self.base_path.join(uuid.to_string());
                     // take the lock to reopen the environment.
                     reopen
                         .reopen(&mut self.index_map.write().unwrap(), &index_path)
@@ -431,7 +405,7 @@ impl IndexMapper {
                     // if it's not already there.
                     match index_map.get(&uuid) {
                         Missing => {
-                            let index_path = self.index_path(uuid);
+                            let index_path = self.base_path.join(uuid.to_string());
 
                             break index_map
                                 .create(
@@ -458,14 +432,6 @@ impl IndexMapper {
         Ok(index)
     }
 
-    /// Returns the path of the index.
-    ///
-    /// The folder located at this path is containing the data.mdb,
-    /// the lock.mdb and an optional data.mdb.cpy file.
-    pub fn index_path(&self, uuid: Uuid) -> PathBuf {
-        self.base_path.join(uuid.to_string())
-    }
-
     pub fn rollback_index(
         &self,
         rtxn: &RoTxn,
@@ -506,7 +472,7 @@ impl IndexMapper {
             };
         }
 
-        let index_path = self.index_path(uuid);
+        let index_path = self.base_path.join(uuid.to_string());
         Index::rollback(milli::heed::EnvOpenOptions::new().read_txn_without_tls(), index_path, to)
             .map_err(|err| crate::Error::from_milli(err, Some(name.to_string())))
     }

crates/index-scheduler/src/insta_snapshot.rs

@@ -6,7 +6,7 @@ use meilisearch_types::heed::types::{SerdeBincode, SerdeJson, Str};
 use meilisearch_types::heed::{Database, RoTxn};
 use meilisearch_types::milli::{CboRoaringBitmapCodec, RoaringBitmapCodec, BEU32};
 use meilisearch_types::tasks::{Details, Kind, Status, Task};
-use meilisearch_types::versioning::{self, VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH};
+use meilisearch_types::versioning;
 use roaring::RoaringBitmap;
 
 use crate::index_mapper::IndexMapper;
@@ -27,7 +27,6 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
         queue,
         scheduler,
         persisted,
-        export_default_payload_size_bytes: _,
 
         index_mapper,
         features: _,
@@ -37,7 +36,6 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
         run_loop_iteration: _,
         embedders: _,
         chat_settings: _,
-        runtime: _,
     } = scheduler;
 
     let rtxn = env.read_txn().unwrap();
@@ -233,7 +231,6 @@ pub fn snapshot_task(task: &Task) -> String {
         status,
         kind,
         network,
-        custom_metadata,
     } = task;
     snap.push('{');
     snap.push_str(&format!("uid: {uid}, "));
@@ -254,9 +251,6 @@ pub fn snapshot_task(task: &Task) -> String {
     if let Some(network) = network {
         snap.push_str(&format!("network: {network:?}, "))
     }
-    if let Some(custom_metadata) = custom_metadata {
-        snap.push_str(&format!("custom_metadata: {custom_metadata:?}"))
-    }
 
     snap.push('}');
     snap
@@ -321,17 +315,7 @@ fn snapshot_details(d: &Details) -> String {
             format!("{{ url: {url:?}, api_key: {api_key:?}, payload_size: {payload_size:?}, indexes: {indexes:?} }}")
         }
         Details::UpgradeDatabase { from, to } => {
-            if to == &(VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) {
+            format!("{{ from: {from:?}, to: {to:?} }}")
-                format!("{{ from: {from:?}, to: [current version] }}")
-            } else {
-                format!("{{ from: {from:?}, to: {to:?} }}")
-            }
-        }
-        Details::IndexCompaction { index_uid, pre_compaction_size, post_compaction_size } => {
-            format!("{{ index_uid: {index_uid:?}, pre_compaction_size: {pre_compaction_size:?}, post_compaction_size: {post_compaction_size:?} }}")
-        }
-        Details::NetworkTopologyChange { moved_documents, message } => {
-            format!("{{ moved_documents: {moved_documents:?}, message: {message:?}")
         }
     }
 }
@@ -408,21 +392,7 @@ pub fn snapshot_batch(batch: &Batch) -> String {
 
     snap.push('{');
     snap.push_str(&format!("uid: {uid}, "));
-    let details = if let Some(upgrade_to) = &details.upgrade_to {
+    snap.push_str(&format!("details: {}, ", serde_json::to_string(details).unwrap()));
-        if upgrade_to.as_str()
-            == format!("v{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_PATCH}").as_str()
-        {
-            let mut details = details.clone();
-
-            details.upgrade_to = Some("[current version]".into());
-            serde_json::to_string(&details).unwrap()
-        } else {
-            serde_json::to_string(details).unwrap()
-        }
-    } else {
-        serde_json::to_string(details).unwrap()
-    };
-    snap.push_str(&format!("details: {details}, "));
     snap.push_str(&format!("stats: {}, ", serde_json::to_string(&stats).unwrap()));
     if !embedder_stats.skip_serializing() {
         snap.push_str(&format!(

crates/index-scheduler/src/lib.rs

@@ -48,13 +48,13 @@ use std::path::{Path, PathBuf};
 use std::sync::{Arc, RwLock};
 use std::time::Duration;
 
-use byte_unit::Byte;
 use dump::Dump;
 pub use error::Error;
 pub use features::RoFeatures;
 use flate2::bufread::GzEncoder;
 use flate2::Compression;
 use meilisearch_types::batches::Batch;
+use meilisearch_types::enterprise_edition::network::Network;
 use meilisearch_types::features::{
     ChatCompletionSettings, InstanceTogglableFeatures, RuntimeTogglableFeatures,
 };
@@ -67,14 +67,11 @@ use meilisearch_types::milli::vector::{
     Embedder, EmbedderOptions, RuntimeEmbedder, RuntimeEmbedders, RuntimeFragment,
 };
 use meilisearch_types::milli::{self, Index};
-use meilisearch_types::network::Network;
 use meilisearch_types::task_view::TaskView;
-use meilisearch_types::tasks::network::{
+use meilisearch_types::tasks::{KindWithContent, Task, TaskNetwork};
-    DbTaskNetwork, ImportData, ImportMetadata, Origin, TaskNetwork,
-};
-use meilisearch_types::tasks::{KindWithContent, Task};
 use meilisearch_types::webhooks::{Webhook, WebhooksDumpView, WebhooksView};
 use milli::vector::db::IndexEmbeddingConfig;
+use processing::ProcessingTasks;
 pub use queue::Query;
 use queue::Queue;
 use roaring::RoaringBitmap;
@@ -85,7 +82,6 @@ use uuid::Uuid;
 use versioning::Versioning;
 
 use crate::index_mapper::IndexMapper;
-use crate::processing::ProcessingTasks;
 use crate::utils::clamp_to_page_size;
 
 pub(crate) type BEI128 = I128<BE>;
@@ -148,11 +144,9 @@ pub struct IndexSchedulerOptions {
     /// If the autobatcher is allowed to automatically batch tasks
     /// it will only batch this defined maximum size (in bytes) of tasks at once.
     pub batched_tasks_size_limit: u64,
-    /// The maximum size of the default payload for exporting documents, in bytes
-    pub export_default_payload_size_bytes: Byte,
     /// The experimental features enabled for this instance.
     pub instance_features: InstanceTogglableFeatures,
-    /// Whether the index scheduler is able to auto upgrade or not.
+    /// The experimental features enabled for this instance.
     pub auto_upgrade: bool,
     /// The maximal number of entries in the search query cache of an embedder.
     ///
@@ -205,9 +199,6 @@ pub struct IndexScheduler {
     /// to the same embeddings for the same input text.
     embedders: Arc<RwLock<HashMap<EmbedderOptions, Arc<Embedder>>>>,
 
-    /// The maximum size of the default payload for exporting documents, in bytes
-    pub export_default_payload_size_bytes: Byte,
-
     // ================= test
     // The next entry is dedicated to the tests.
     /// Provide a way to set a breakpoint in multiple part of the scheduler.
@@ -225,9 +216,6 @@ pub struct IndexScheduler {
     /// A counter that is incremented before every call to [`tick`](IndexScheduler::tick)
     #[cfg(test)]
     run_loop_iteration: Arc<RwLock<usize>>,
-
-    /// The tokio runtime used for asynchronous tasks.
-    runtime: Option<tokio::runtime::Handle>,
 }
 
 impl IndexScheduler {
@@ -243,7 +231,6 @@ impl IndexScheduler {
             cleanup_enabled: self.cleanup_enabled,
             experimental_no_edition_2024_for_dumps: self.experimental_no_edition_2024_for_dumps,
             persisted: self.persisted,
-            export_default_payload_size_bytes: self.export_default_payload_size_bytes,
 
             webhooks: self.webhooks.clone(),
             embedders: self.embedders.clone(),
@@ -255,7 +242,6 @@ impl IndexScheduler {
             run_loop_iteration: self.run_loop_iteration.clone(),
             features: self.features.clone(),
             chat_settings: self.chat_settings,
-            runtime: self.runtime.clone(),
         }
     }
 
@@ -269,23 +255,13 @@ impl IndexScheduler {
     }
 
     /// Create an index scheduler and start its run loop.
+    #[allow(private_interfaces)] // because test_utils is private
     pub fn new(
         options: IndexSchedulerOptions,
         auth_env: Env<WithoutTls>,
         from_db_version: (u32, u32, u32),
-        runtime: Option<tokio::runtime::Handle>,
+        #[cfg(test)] test_breakpoint_sdr: crossbeam_channel::Sender<(test_utils::Breakpoint, bool)>,
-    ) -> Result<Self> {
+        #[cfg(test)] planned_failures: Vec<(usize, test_utils::FailureLocation)>,
-        let this = Self::new_without_run(options, auth_env, from_db_version, runtime)?;
-
-        this.run();
-        Ok(this)
-    }
-
-    fn new_without_run(
-        options: IndexSchedulerOptions,
-        auth_env: Env<WithoutTls>,
-        from_db_version: (u32, u32, u32),
-        runtime: Option<tokio::runtime::Handle>,
     ) -> Result<Self> {
         std::fs::create_dir_all(&options.tasks_path)?;
         std::fs::create_dir_all(&options.update_file_path)?;
@@ -340,7 +316,8 @@ impl IndexScheduler {
 
         wtxn.commit()?;
 
-        Ok(Self {
+        // allow unreachable_code to get rids of the warning in the case of a test build.
+        let this = Self {
             processing_tasks: Arc::new(RwLock::new(ProcessingTasks::new())),
             version,
             queue,
@@ -355,34 +332,16 @@ impl IndexScheduler {
             persisted,
             webhooks: Arc::new(webhooks),
             embedders: Default::default(),
-            export_default_payload_size_bytes: options.export_default_payload_size_bytes,
 
-            #[cfg(test)] // Will be replaced in `new_tests` in test environments
+            #[cfg(test)]
-            test_breakpoint_sdr: crossbeam_channel::bounded(0).0,
+            test_breakpoint_sdr,
-            #[cfg(test)] // Will be replaced in `new_tests` in test environments
+            #[cfg(test)]
-            planned_failures: Default::default(),
+            planned_failures,
             #[cfg(test)]
             run_loop_iteration: Arc::new(RwLock::new(0)),
             features,
             chat_settings,
-            runtime,
+        };
-        })
-    }
-
-    /// Create an index scheduler and start its run loop.
-    #[cfg(test)]
-    fn new_test(
-        options: IndexSchedulerOptions,
-        auth_env: Env<WithoutTls>,
-        from_db_version: (u32, u32, u32),
-        runtime: Option<tokio::runtime::Handle>,
-        test_breakpoint_sdr: crossbeam_channel::Sender<(test_utils::Breakpoint, bool)>,
-        planned_failures: Vec<(usize, test_utils::FailureLocation)>,
-    ) -> Result<Self> {
-        let mut this = Self::new_without_run(options, auth_env, from_db_version, runtime)?;
-
-        this.test_breakpoint_sdr = test_breakpoint_sdr;
-        this.planned_failures = planned_failures;
 
         this.run();
         Ok(this)
@@ -711,14 +670,14 @@ impl IndexScheduler {
         self.queue.get_task_ids_from_authorized_indexes(&rtxn, query, filters, &processing)
     }
 
-    pub fn set_task_network(&self, task_id: TaskId, network: DbTaskNetwork) -> Result<Task> {
+    pub fn set_task_network(&self, task_id: TaskId, network: TaskNetwork) -> Result<()> {
         let mut wtxn = self.env.write_txn()?;
         let mut task =
             self.queue.tasks.get_task(&wtxn, task_id)?.ok_or(Error::TaskNotFound(task_id))?;
         task.network = Some(network);
         self.queue.tasks.all_tasks.put(&mut wtxn, &task_id, &task)?;
         wtxn.commit()?;
-        Ok(task)
+        Ok(())
     }
 
     /// Return the batches matching the query from the user's point of view along
@@ -767,31 +726,6 @@ impl IndexScheduler {
         kind: KindWithContent,
         task_id: Option<TaskId>,
         dry_run: bool,
-    ) -> Result<Task> {
-        self.register_with_custom_metadata(kind, task_id, None, dry_run, None)
-    }
-
-    /// Register a new task in the scheduler, with metadata.
-    ///
-    /// If it fails and data was associated with the task, it tries to delete the associated data.
-    ///
-    /// # Parameters
-    ///
-    /// - task_network: network of the task to check.
-    ///
-    /// If the task is an import task, only accept it if:
-    ///
-    /// 1. There is an ongoing network topology change task
-    /// 2. The task to register matches the network version of the network topology change task
-    ///
-    /// Always accept the task if it is not an import task.
-    pub fn register_with_custom_metadata(
-        &self,
-        kind: KindWithContent,
-        task_id: Option<TaskId>,
-        custom_metadata: Option<String>,
-        dry_run: bool,
-        task_network: Option<TaskNetwork>,
     ) -> Result<Task> {
         // if the task doesn't delete or cancel anything and 40% of the task queue is full, we must refuse to enqueue the incoming task
         if !matches!(&kind, KindWithContent::TaskDeletion { tasks, .. } | KindWithContent::TaskCancelation { tasks, .. } if !tasks.is_empty())
@@ -802,19 +736,7 @@ impl IndexScheduler {
         }
 
         let mut wtxn = self.env.write_txn()?;
-
+        let task = self.queue.register(&mut wtxn, &kind, task_id, dry_run)?;
-        if let Some(TaskNetwork::Import { import_from, network_change, metadata }) = &task_network {
-            self.update_network_task(&mut wtxn, import_from, network_change, metadata)?;
-        }
-
-        let task = self.queue.register(
-            &mut wtxn,
-            &kind,
-            task_id,
-            custom_metadata,
-            dry_run,
-            task_network.map(DbTaskNetwork::from),
-        )?;
 
         // If the registered task is a task cancelation
         // we inform the processing tasks to stop (if necessary).
@@ -836,91 +758,6 @@ impl IndexScheduler {
         Ok(task)
     }
 
-    pub fn network_no_index_for_remote(
-        &self,
-        remote_name: String,
-        origin: Origin,
-    ) -> Result<(), Error> {
-        let mut wtxn = self.env.write_txn()?;
-
-        self.update_network_task(
-            &mut wtxn,
-            &ImportData { remote_name, index_name: None, document_count: 0 },
-            &origin,
-            &ImportMetadata { index_count: 0, task_key: None, total_index_documents: 0 },
-        )?;
-
-        wtxn.commit()?;
-
-        // wake up the scheduler as the task state has changed
-        self.scheduler.wake_up.signal();
-
-        Ok(())
-    }
-
-    fn update_network_task(
-        &self,
-        wtxn: &mut heed::RwTxn<'_>,
-        import_from: &ImportData,
-        network_change: &Origin,
-        metadata: &ImportMetadata,
-    ) -> Result<(), Error> {
-        let mut network_tasks = self
-            .queue
-            .tasks
-            .get_kind(&*wtxn, meilisearch_types::tasks::Kind::NetworkTopologyChange)?;
-        if network_tasks.is_empty() {
-            return Err(Error::ImportTaskWithoutNetworkTask);
-        }
-        let network_task = {
-            let processing = self.processing_tasks.read().unwrap().processing.clone();
-            if processing.is_disjoint(&network_tasks) {
-                let enqueued = self
-                    .queue
-                    .tasks
-                    .get_status(&*wtxn, meilisearch_types::tasks::Status::Enqueued)?;
-
-                network_tasks &= enqueued;
-                if let Some(network_task) = network_tasks.into_iter().next() {
-                    network_task
-                } else {
-                    return Err(Error::ImportTaskWithoutNetworkTask);
-                }
-            } else {
-                network_tasks &= &*processing;
-                network_tasks.into_iter().next().unwrap()
-            }
-        };
-        let mut network_task = self.queue.tasks.get_task(&*wtxn, network_task)?.unwrap();
-        let network_task_version = network_task
-            .network
-            .as_ref()
-            .map(|network| network.network_version())
-            .unwrap_or_default();
-        if network_task_version != network_change.network_version {
-            return Err(Error::NetworkVersionMismatch {
-                network_task: network_task_version,
-                import_task: network_change.network_version,
-            });
-        }
-        let KindWithContent::NetworkTopologyChange(network_topology_change) =
-            &mut network_task.kind
-        else {
-            tracing::error!("unexpected network kind for network task while registering task");
-            return Err(Error::CorruptedTaskQueue);
-        };
-        network_topology_change.receive_remote_task(
-            &import_from.remote_name,
-            import_from.index_name.as_deref(),
-            metadata.task_key,
-            import_from.document_count,
-            metadata.index_count,
-            metadata.total_index_documents,
-        )?;
-        self.queue.tasks.update_task(wtxn, &mut network_task)?;
-        Ok(())
-    }
-
     /// Register a new task coming from a dump in the scheduler.
     /// By taking a mutable ref we're pretty sure no one will ever import a dump while actix is running.
     pub fn register_dumped_task(&mut self) -> Result<Dump<'_>> {

crates/index-scheduler/src/processing.rs

@@ -42,10 +42,12 @@ impl ProcessingTasks {
 
     /// Set the processing tasks to an empty list
     pub fn stop_processing(&mut self) -> Self {
+        self.progress = None;
+
         Self {
             batch: std::mem::take(&mut self.batch),
             processing: std::mem::take(&mut self.processing),
-            progress: std::mem::take(&mut self.progress),
+            progress: None,
         }
     }
 
@@ -73,7 +75,6 @@ make_enum_progress! {
     pub enum TaskCancelationProgress {
         RetrievingTasks,
         CancelingUpgrade,
-        CleaningCompactionLeftover,
         UpdatingTasks,
     }
 }
@@ -137,17 +138,6 @@ make_enum_progress! {
     }
 }
 
-make_enum_progress! {
-    pub enum IndexCompaction {
-        RetrieveTheIndex,
-        CreateTemporaryFile,
-        CopyAndCompactTheIndex,
-        PersistTheCompactedIndex,
-        CloseTheIndex,
-        ReopenTheIndex,
-    }
-}
-
 make_enum_progress! {
     pub enum InnerSwappingTwoIndexes {
         RetrieveTheTasks,

crates/index-scheduler/src/queue/mod.rs

@@ -15,7 +15,6 @@ use file_store::FileStore;
 use meilisearch_types::batches::BatchId;
 use meilisearch_types::heed::{Database, Env, RoTxn, RwTxn, WithoutTls};
 use meilisearch_types::milli::{CboRoaringBitmapCodec, BEU32};
-use meilisearch_types::tasks::network::DbTaskNetwork;
 use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task};
 use roaring::RoaringBitmap;
 use time::format_description::well_known::Rfc3339;
@@ -258,9 +257,7 @@ impl Queue {
         wtxn: &mut RwTxn,
         kind: &KindWithContent,
         task_id: Option<TaskId>,
-        custom_metadata: Option<String>,
         dry_run: bool,
-        network: Option<DbTaskNetwork>,
     ) -> Result<Task> {
         let next_task_id = self.tasks.next_task_id(wtxn)?;
 
@@ -282,8 +279,7 @@ impl Queue {
             details: kind.default_details(),
             status: Status::Enqueued,
             kind: kind.clone(),
-            network,
+            network: None,
-            custom_metadata,
         };
         // For deletion and cancelation tasks, we want to make extra sure that they
         // don't attempt to delete/cancel tasks that are newer than themselves.
@@ -348,9 +344,7 @@ impl Queue {
                 tasks: to_delete,
             },
             None,
-            None,
             false,
-            None,
         )?;
 
         Ok(())

crates/index-scheduler/src/queue/tasks.rs

@@ -3,8 +3,7 @@ use std::ops::{Bound, RangeBounds};
 use meilisearch_types::heed::types::{DecodeIgnore, SerdeBincode, SerdeJson, Str};
 use meilisearch_types::heed::{Database, Env, RoTxn, RwTxn, WithoutTls};
 use meilisearch_types::milli::{CboRoaringBitmapCodec, RoaringBitmapCodec, BEU32};
-use meilisearch_types::tasks::network::DbTaskNetwork;
+use meilisearch_types::tasks::{Kind, Status, Task};
-use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task};
 use roaring::{MultiOps, RoaringBitmap};
 use time::OffsetDateTime;
 
@@ -115,15 +114,14 @@ impl TaskQueue {
     /// - CorruptedTaskQueue: The task doesn't exist in the database
     pub(crate) fn update_task(&self, wtxn: &mut RwTxn, task: &mut Task) -> Result<()> {
         let old_task = self.get_task(wtxn, task.uid)?.ok_or(Error::CorruptedTaskQueue)?;
-        // network topology tasks may be processed multiple times.
+        let reprocessing = old_task.status != Status::Enqueued;
-        let maybe_reprocessing = old_task.status != Status::Enqueued
-            || task.kind.as_kind() == Kind::NetworkTopologyChange;
 
+        debug_assert!(old_task != *task);
         debug_assert_eq!(old_task.uid, task.uid);
 
         // If we're processing a task that failed it may already contains a batch_uid
         debug_assert!(
-            maybe_reprocessing || (old_task.batch_uid.is_none() && task.batch_uid.is_some()),
+            reprocessing || (old_task.batch_uid.is_none() && task.batch_uid.is_some()),
             "\n==> old: {old_task:?}\n==> new: {task:?}"
         );
 
@@ -145,24 +143,13 @@ impl TaskQueue {
             })?;
         }
 
-        // Avoids rewriting part of the network topology change because of TOCTOU errors
-        if let (
-            KindWithContent::NetworkTopologyChange(old_state),
-            KindWithContent::NetworkTopologyChange(new_state),
-        ) = (old_task.kind, &mut task.kind)
-        {
-            new_state.merge(old_state);
-            // the state possibly just changed, rewrite the details
-            task.details = Some(new_state.to_details());
-        }
-
         assert_eq!(
             old_task.enqueued_at, task.enqueued_at,
             "Cannot update a task's enqueued_at time"
         );
         if old_task.started_at != task.started_at {
             assert!(
-                maybe_reprocessing || old_task.started_at.is_none(),
+                reprocessing || old_task.started_at.is_none(),
                 "Cannot update a task's started_at time"
             );
             if let Some(started_at) = old_task.started_at {
@@ -174,7 +161,7 @@ impl TaskQueue {
         }
         if old_task.finished_at != task.finished_at {
             assert!(
-                maybe_reprocessing || old_task.finished_at.is_none(),
+                reprocessing || old_task.finished_at.is_none(),
                 "Cannot update a task's finished_at time"
             );
             if let Some(finished_at) = old_task.finished_at {
@@ -188,16 +175,7 @@ impl TaskQueue {
         task.network = match (old_task.network, task.network.take()) {
             (None, None) => None,
             (None, Some(network)) | (Some(network), None) => Some(network),
-            (Some(left), Some(right)) => Some(match (left, right) {
+            (Some(_), Some(network)) => Some(network),
-                (
-                    DbTaskNetwork::Remotes { remote_tasks: mut left, network_version: _ },
-                    DbTaskNetwork::Remotes { remote_tasks: mut right, network_version },
-                ) => {
-                    left.append(&mut right);
-                    DbTaskNetwork::Remotes { remote_tasks: left, network_version }
-                }
-                (_, right) => right,
-            }),
         };
 
         self.all_tasks.put(wtxn, &task.uid, task)?;

crates/index-scheduler/src/queue/test.rs

@@ -203,30 +203,26 @@ fn test_disable_auto_deletion_of_tasks() {
         )
         .unwrap();
 
-    {
+    let rtxn = index_scheduler.env.read_txn().unwrap();
-        let rtxn = index_scheduler.env.read_txn().unwrap();
+    let proc = index_scheduler.processing_tasks.read().unwrap();
-        let proc = index_scheduler.processing_tasks.read().unwrap();
+    let tasks =
-        let tasks = index_scheduler
+        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
-            .queue
+    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
-            .get_task_ids(&rtxn, &Query { ..Default::default() }, &proc)
+    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]" }), name: "task_queue_is_full");
-            .unwrap();
+    drop(rtxn);
-        let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
+    drop(proc);
-        snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]" }), name: "task_queue_is_full");
-    }
 
     // now we're above the max number of tasks
     // and if we try to advance in the tick function no new task deletion should be enqueued
     handle.advance_till([Start, BatchCreated]);
-    {
+    let rtxn = index_scheduler.env.read_txn().unwrap();
-        let rtxn = index_scheduler.env.read_txn().unwrap();
+    let proc = index_scheduler.processing_tasks.read().unwrap();
-        let proc = index_scheduler.processing_tasks.read().unwrap();
+    let tasks =
-        let tasks = index_scheduler
+        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
-            .queue
+    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
-            .get_task_ids(&rtxn, &Query { ..Default::default() }, &proc)
+    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "task_deletion_have_not_been_enqueued");
-            .unwrap();
+    drop(rtxn);
-        let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
+    drop(proc);
-        snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "task_deletion_have_not_been_enqueued");
-    }
 }
 
 #[test]
@@ -271,69 +267,59 @@ fn test_auto_deletion_of_tasks() {
         )
         .unwrap();
 
-    {
+    let rtxn = index_scheduler.env.read_txn().unwrap();
-        let rtxn = index_scheduler.env.read_txn().unwrap();
+    let proc = index_scheduler.processing_tasks.read().unwrap();
-        let proc = index_scheduler.processing_tasks.read().unwrap();
+    let tasks =
-        let tasks = index_scheduler
+        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
-            .queue
+    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
-            .get_task_ids(&rtxn, &Query { ..Default::default() }, &proc)
+    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]" }), name: "task_queue_is_full");
-            .unwrap();
+    drop(rtxn);
-        let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
+    drop(proc);
-        snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]" }), name: "task_queue_is_full");
-    }
 
-    {
+    // now we're above the max number of tasks
-        // now we're above the max number of tasks
+    // and if we try to advance in the tick function a new task deletion should be enqueued
-        // and if we try to advance in the tick function a new task deletion should be enqueued
+    handle.advance_till([Start, BatchCreated]);
-        handle.advance_till([Start, BatchCreated]);
+    let rtxn = index_scheduler.env.read_txn().unwrap();
-        let rtxn = index_scheduler.env.read_txn().unwrap();
+    let proc = index_scheduler.processing_tasks.read().unwrap();
-        let proc = index_scheduler.processing_tasks.read().unwrap();
+    let tasks =
-        let tasks = index_scheduler
+        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
-            .queue
+    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
-            .get_task_ids(&rtxn, &Query { ..Default::default() }, &proc)
+    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "task_deletion_have_been_enqueued");
-            .unwrap();
+    drop(rtxn);
-        let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
+    drop(proc);
-        snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "task_deletion_have_been_enqueued");
-    }
 
-    {
+    handle.advance_till([InsideProcessBatch, ProcessBatchSucceeded, AfterProcessing]);
-        handle.advance_till([InsideProcessBatch, ProcessBatchSucceeded, AfterProcessing]);
+    let rtxn = index_scheduler.env.read_txn().unwrap();
-        let rtxn = index_scheduler.env.read_txn().unwrap();
+    let proc = index_scheduler.processing_tasks.read().unwrap();
-        let proc = index_scheduler.processing_tasks.read().unwrap();
+    let tasks =
-        let tasks = index_scheduler
+        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
-            .queue
+    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
-            .get_task_ids(&rtxn, &Query { ..Default::default() }, &proc)
+    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "task_deletion_have_been_processed");
-            .unwrap();
+    drop(rtxn);
-        let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
+    drop(proc);
-        snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "task_deletion_have_been_processed");
-    }
 
     handle.advance_one_failed_batch();
     // a new task deletion has been enqueued
     handle.advance_one_successful_batch();
-    {
+    let rtxn = index_scheduler.env.read_txn().unwrap();
-        let rtxn = index_scheduler.env.read_txn().unwrap();
+    let proc = index_scheduler.processing_tasks.read().unwrap();
-        let proc = index_scheduler.processing_tasks.read().unwrap();
+    let tasks =
-        let tasks = index_scheduler
+        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
-            .queue
+    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
-            .get_task_ids(&rtxn, &Query { ..Default::default() }, &proc)
+    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "after_the_second_task_deletion");
-            .unwrap();
+    drop(rtxn);
-        let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
+    drop(proc);
-        snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "after_the_second_task_deletion");
-    }
 
     handle.advance_one_failed_batch();
     handle.advance_one_successful_batch();
-    {
+    let rtxn = index_scheduler.env.read_txn().unwrap();
-        let rtxn = index_scheduler.env.read_txn().unwrap();
+    let proc = index_scheduler.processing_tasks.read().unwrap();
-        let proc = index_scheduler.processing_tasks.read().unwrap();
+    let tasks =
-        let tasks = index_scheduler
+        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
-            .queue
+    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
-            .get_task_ids(&rtxn, &Query { ..Default::default() }, &proc)
+    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "everything_has_been_processed");
-            .unwrap();
+    drop(rtxn);
-        let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
+    drop(proc);
-        snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "everything_has_been_processed");
-    }
 }
 
 #[test]

crates/index-scheduler/src/scheduler/autobatcher.rs

@@ -68,15 +68,13 @@ impl From<KindWithContent> for AutobatchKind {
             KindWithContent::IndexCreation { .. } => AutobatchKind::IndexCreation,
             KindWithContent::IndexUpdate { .. } => AutobatchKind::IndexUpdate,
             KindWithContent::IndexSwap { .. } => AutobatchKind::IndexSwap,
-            KindWithContent::IndexCompaction { .. }
+            KindWithContent::TaskCancelation { .. }
-            | KindWithContent::TaskCancelation { .. }
             | KindWithContent::TaskDeletion { .. }
             | KindWithContent::DumpCreation { .. }
             | KindWithContent::Export { .. }
             | KindWithContent::UpgradeDatabase { .. }
-            | KindWithContent::NetworkTopologyChange(_)
             | KindWithContent::SnapshotCreation => {
-                panic!("The autobatcher should never be called with tasks with special priority or that don't apply to an index.")
+                panic!("The autobatcher should never be called with tasks that don't apply to an index.")
             }
         }
     }
@@ -289,10 +287,8 @@ impl BatchKind {
         };
 
         match (self, autobatch_kind) {
-            // We don't batch any of these operations
+            // We don't batch any of these operations  
-            (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentEdition) => {
+            (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentEdition) => Break((this, BatchStopReason::TaskCannotBeBatched { kind, id })),
-                Break((this, BatchStopReason::TaskCannotBeBatched { kind, id }))
-            },
             // We must not batch tasks that don't have the same index creation rights if the index doesn't already exists.
             (this, kind) if !index_already_exists && this.allow_index_creation() == Some(false) && kind.allow_index_creation() == Some(true) => {
                 Break((this, BatchStopReason::IndexCreationMismatch { id }))

crates/index-scheduler/src/scheduler/community_edition.rs

@@ -1,27 +0,0 @@
-use meilisearch_types::milli::progress::Progress;
-use meilisearch_types::tasks::Task;
-
-use super::create_batch::Batch;
-use crate::scheduler::process_batch::ProcessBatchInfo;
-use crate::utils::ProcessingBatch;
-use crate::{Error, IndexScheduler, Result};
-
-impl IndexScheduler {
-    pub(super) fn process_network_index_batch(
-        &self,
-        _network_task: Task,
-        _inner_batch: Box<Batch>,
-        _current_batch: &mut ProcessingBatch,
-        _progress: Progress,
-    ) -> Result<(Vec<Task>, ProcessBatchInfo)> {
-        Err(Error::RequiresEnterpriseEdition { action: "processing a network task" })
-    }
-
-    pub(super) fn process_network_ready(
-        &self,
-        _task: Task,
-        _progress: Progress,
-    ) -> Result<(Vec<Task>, ProcessBatchInfo)> {
-        Err(Error::RequiresEnterpriseEdition { action: "processing a network task" })
-    }
-}

crates/index-scheduler/src/scheduler/create_batch.rs

@@ -4,7 +4,6 @@ use std::io::ErrorKind;
 use meilisearch_types::heed::RoTxn;
 use meilisearch_types::milli::update::IndexDocumentsMethod;
 use meilisearch_types::settings::{Settings, Unchecked};
-use meilisearch_types::tasks::network::NetworkTopologyState;
 use meilisearch_types::tasks::{BatchStopReason, Kind, KindWithContent, Status, Task};
 use roaring::RoaringBitmap;
 use uuid::Uuid;
@@ -56,18 +55,6 @@ pub(crate) enum Batch {
     UpgradeDatabase {
         tasks: Vec<Task>,
     },
-    IndexCompaction {
-        index_uid: String,
-        task: Task,
-    },
-    #[allow(clippy::enum_variant_names)] // warranted because we are executing an inner index batch
-    NetworkIndexBatch {
-        network_task: Task,
-        inner_batch: Box<Batch>,
-    },
-    NetworkReady {
-        task: Task,
-    },
 }
 
 #[derive(Debug)]
@@ -123,8 +110,7 @@ impl Batch {
             | Batch::Dump(task)
             | Batch::IndexCreation { task, .. }
             | Batch::Export { task }
-            | Batch::IndexUpdate { task, .. }
+            | Batch::IndexUpdate { task, .. } => {
-            | Batch::IndexCompaction { task, .. } => {
                 RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap()
             }
             Batch::SnapshotCreation(tasks)
@@ -149,14 +135,9 @@ impl Batch {
                     ..
                 } => RoaringBitmap::from_iter(tasks.iter().chain(other).map(|task| task.uid)),
             },
-            Batch::IndexSwap { task } | Batch::NetworkReady { task } => {
+            Batch::IndexSwap { task } => {
                 RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap()
             }
-            Batch::NetworkIndexBatch { network_task, inner_batch } => {
-                let mut tasks = inner_batch.ids();
-                tasks.insert(network_task.uid);
-                tasks
-            }
         }
     }
 
@@ -170,14 +151,11 @@ impl Batch {
             | Dump(_)
             | Export { .. }
             | UpgradeDatabase { .. }
-            | NetworkReady { .. }
             | IndexSwap { .. } => None,
             IndexOperation { op, .. } => Some(op.index_uid()),
             IndexCreation { index_uid, .. }
             | IndexUpdate { index_uid, .. }
-            | IndexDeletion { index_uid, .. }
+            | IndexDeletion { index_uid, .. } => Some(index_uid),
-            | IndexCompaction { index_uid, .. } => Some(index_uid),
-            NetworkIndexBatch { network_task: _, inner_batch } => inner_batch.index_uid(),
         }
     }
 }
@@ -197,11 +175,8 @@ impl fmt::Display for Batch {
             Batch::IndexUpdate { .. } => f.write_str("IndexUpdate")?,
             Batch::IndexDeletion { .. } => f.write_str("IndexDeletion")?,
             Batch::IndexSwap { .. } => f.write_str("IndexSwap")?,
-            Batch::IndexCompaction { .. } => f.write_str("IndexCompaction")?,
             Batch::Export { .. } => f.write_str("Export")?,
             Batch::UpgradeDatabase { .. } => f.write_str("UpgradeDatabase")?,
-            Batch::NetworkIndexBatch { .. } => f.write_str("NetworkTopologyChange")?,
-            Batch::NetworkReady { .. } => f.write_str("NetworkTopologyChange")?,
         };
         match index_uid {
             Some(name) => f.write_fmt(format_args!(" on {name:?} from tasks: {tasks:?}")),
@@ -470,7 +445,6 @@ impl IndexScheduler {
     pub(crate) fn create_next_batch(
         &self,
         rtxn: &RoTxn,
-        processing_network_tasks: &RoaringBitmap,
     ) -> Result<Option<(Batch, ProcessingBatch)>> {
         #[cfg(test)]
         self.maybe_fail(crate::test_utils::FailureLocation::InsideCreateBatch)?;
@@ -479,6 +453,7 @@ impl IndexScheduler {
         let mut current_batch = ProcessingBatch::new(batch_id);
 
         let enqueued = &self.queue.tasks.get_status(rtxn, Status::Enqueued)?;
+        let count_total_enqueued = enqueued.len();
         let failed = &self.queue.tasks.get_status(rtxn, Status::Failed)?;
 
         // 0. we get the last task to cancel.
@@ -527,15 +502,7 @@ impl IndexScheduler {
             )));
         }
 
-        // 2. Check for enqueued network topology changes
+        // 2. we get the next task to delete
-        let network_changes = self.queue.tasks.get_kind(rtxn, Kind::NetworkTopologyChange)?
-            & (enqueued | processing_network_tasks);
-        if let Some(task_id) = network_changes.iter().next() {
-            let task = self.queue.tasks.get_task(rtxn, task_id)?.unwrap();
-            return self.start_processing_network(rtxn, task, enqueued, current_batch);
-        }
-
-        // 3. we get the next task to delete
         let to_delete = self.queue.tasks.get_kind(rtxn, Kind::TaskDeletion)? & enqueued;
         if !to_delete.is_empty() {
             let mut tasks = self.queue.tasks.get_existing_tasks(rtxn, to_delete)?;
@@ -545,33 +512,17 @@ impl IndexScheduler {
             return Ok(Some((Batch::TaskDeletions(tasks), current_batch)));
         }
 
-        // 4. we get the next task to compact
+        // 3. we batch the export.
-        let to_compact = self.queue.tasks.get_kind(rtxn, Kind::IndexCompaction)? & enqueued;
-        if let Some(task_id) = to_compact.min() {
-            let mut task =
-                self.queue.tasks.get_task(rtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?;
-            current_batch.processing(Some(&mut task));
-            current_batch.reason(BatchStopReason::TaskCannotBeBatched {
-                kind: Kind::IndexCompaction,
-                id: task_id,
-            });
-            let index_uid =
-                task.index_uid().expect("Compaction task must have an index uid").to_owned();
-            return Ok(Some((Batch::IndexCompaction { index_uid, task }, current_batch)));
-        }
-
-        // 5. we batch the export.
         let to_export = self.queue.tasks.get_kind(rtxn, Kind::Export)? & enqueued;
         if !to_export.is_empty() {
             let task_id = to_export.iter().next().expect("There must be at least one export task");
             let mut task = self.queue.tasks.get_task(rtxn, task_id)?.unwrap();
             current_batch.processing([&mut task]);
-            current_batch
+            current_batch.reason(BatchStopReason::TaskKindCannotBeBatched { kind: Kind::Export });
-                .reason(BatchStopReason::TaskCannotBeBatched { kind: Kind::Export, id: task_id });
             return Ok(Some((Batch::Export { task }, current_batch)));
         }
 
-        // 6. we batch the snapshot.
+        // 4. we batch the snapshot.
         let to_snapshot = self.queue.tasks.get_kind(rtxn, Kind::SnapshotCreation)? & enqueued;
         if !to_snapshot.is_empty() {
             let mut tasks = self.queue.tasks.get_existing_tasks(rtxn, to_snapshot)?;
@@ -581,7 +532,7 @@ impl IndexScheduler {
             return Ok(Some((Batch::SnapshotCreation(tasks), current_batch)));
         }
 
-        // 7. we batch the dumps.
+        // 5. we batch the dumps.
         let to_dump = self.queue.tasks.get_kind(rtxn, Kind::DumpCreation)? & enqueued;
         if let Some(to_dump) = to_dump.min() {
             let mut task =
@@ -594,66 +545,25 @@ impl IndexScheduler {
             return Ok(Some((Batch::Dump(task), current_batch)));
         }
 
-        let network = self.network();
+        // 6. We make a batch from the unprioritised tasks. Start by taking the next enqueued task.
+        let task_id = if let Some(task_id) = enqueued.min() { task_id } else { return Ok(None) };
+        let mut task =
+            self.queue.tasks.get_task(rtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?;
 
-        // 8. We make a batch from the unprioritised tasks.
+        // If the task is not associated with any index, verify that it is an index swap and
-        let (batch, current_batch) =
+        // create the batch directly. Otherwise, get the index name associated with the task
-            self.create_next_batch_unprioritized(rtxn, enqueued, current_batch, |task| {
+        // and use the autobatcher to batch the enqueued tasks associated with it
-                // We want to execute all tasks, except those that have a version strictly higher than the network version
 
-                let Some(task_version) =
+        let index_name = if let Some(&index_name) = task.indexes().first() {
-                    task.network.as_ref().map(|tastk_network| tastk_network.network_version())
+            index_name
-                else {
+        } else {
-                    // do not skip tasks that have no network version, otherwise we will never execute them
+            assert!(matches!(&task.kind, KindWithContent::IndexSwap { swaps } if swaps.is_empty()));
-                    return false;
+            current_batch.processing(Some(&mut task));
-                };
+            current_batch.reason(BatchStopReason::TaskCannotBeBatched {
-
+                kind: Kind::IndexSwap,
-                // skip tasks with a version strictly higher than the network version
+                id: task.uid,
-                task_version > network.version
+            });
-            })?;
+            return Ok(Some((Batch::IndexSwap { task }, current_batch)));
-        Ok(batch.map(|batch| (batch, current_batch)))
-    }
-
-    fn create_next_batch_unprioritized<F>(
-        &self,
-        rtxn: &RoTxn,
-        enqueued: &RoaringBitmap,
-        mut current_batch: ProcessingBatch,
-        mut skip_if: F,
-    ) -> Result<(Option<Batch>, ProcessingBatch)>
-    where
-        F: FnMut(&Task) -> bool,
-    {
-        let count_total_enqueued = enqueued.len();
-
-        let mut enqueued_it = enqueued.iter();
-        let mut task;
-        let index_name = loop {
-            let Some(task_id) = enqueued_it.next() else {
-                return Ok((None, current_batch));
-            };
-            task = self.queue.tasks.get_task(rtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?;
-
-            if skip_if(&task) {
-                continue;
-            }
-            // If the task is not associated with any index, verify that it is an index swap and
-            // create the batch directly. Otherwise, get the index name associated with the task
-            // and use the autobatcher to batch the enqueued tasks associated with it
-
-            if let Some(&index_name) = task.indexes().first() {
-                break index_name;
-            } else {
-                assert!(
-                    matches!(&task.kind, KindWithContent::IndexSwap { swaps } if swaps.is_empty())
-                );
-                current_batch.processing(Some(&mut task));
-                current_batch.reason(BatchStopReason::TaskCannotBeBatched {
-                    kind: Kind::IndexSwap,
-                    id: task.uid,
-                });
-                return Ok((Some(Batch::IndexSwap { task }), current_batch));
-            };
         };
 
         let index_already_exists = self.index_mapper.exists(rtxn, index_name)?;
@@ -688,10 +598,6 @@ impl IndexScheduler {
                 .get_task(rtxn, task_id)
                 .and_then(|task| task.ok_or(Error::CorruptedTaskQueue))?;
 
-            if skip_if(&task) {
-                continue;
-            }
-
             if let Some(uuid) = task.content_uuid() {
                 let content_size = match self.queue.file_store.compute_size(uuid) {
                     Ok(content_size) => content_size,
@@ -722,127 +628,19 @@ impl IndexScheduler {
             autobatcher::autobatch(enqueued, index_already_exists, primary_key.as_deref())
         {
             current_batch.reason(autobatch_stop_reason.unwrap_or(stop_reason));
-            let batch = self.create_next_batch_index(
+            return Ok(self
-                rtxn,
+                .create_next_batch_index(
-                index_name.to_string(),
+                    rtxn,
-                batchkind,
+                    index_name.to_string(),
-                &mut current_batch,
+                    batchkind,
-                create_index,
+                    &mut current_batch,
-            )?;
+                    create_index,
-            return Ok((batch, current_batch));
+                )?
+                .map(|batch| (batch, current_batch)));
         }
 
         // If we found no tasks then we were notified for something that got autobatched
         // somehow and there is nothing to do.
-        Ok((None, current_batch))
+        Ok(None)
-    }
-
-    fn start_processing_network(
-        &self,
-        rtxn: &RoTxn,
-        mut task: Task,
-        enqueued: &RoaringBitmap,
-        mut current_batch: ProcessingBatch,
-    ) -> Result<Option<(Batch, ProcessingBatch)>> {
-        current_batch.processing(Some(&mut task));
-        current_batch.reason(BatchStopReason::NetworkTask { id: task.uid });
-
-        let change_version =
-            task.network.as_ref().map(|network| network.network_version()).unwrap_or_default();
-        let KindWithContent::NetworkTopologyChange(network_topology_change) = &task.kind else {
-            panic!("inconsistent kind with content")
-        };
-
-        match network_topology_change.state() {
-            NetworkTopologyState::WaitingForOlderTasks => {
-                let res =
-                    self.create_next_batch_unprioritized(rtxn, enqueued, current_batch, |task| {
-                        // in this limited mode of execution, we only want to run tasks:
-                        // 0. with an index
-                        // 1. with a version
-                        // 2. that version strictly lower than the network task version
-
-                        // 0. skip indexless tasks that are not index swap
-                        if task.index_uid().is_none() && task.kind.as_kind() != Kind::IndexSwap {
-                            return true;
-                        }
-
-                        // 1. skip tasks without version
-                        let Some(task_version) =
-                            task.network.as_ref().map(|network| network.network_version())
-                        else {
-                            return true;
-                        };
-
-                        // 2. skip tasks with a version equal or higher to the network task version
-                        task_version >= change_version
-                    });
-
-                let (batch, mut current_batch) = res?;
-
-                let batch = match batch {
-                    Some(batch) => {
-                        let inner_batch = Box::new(batch);
-                        let inner_reason = current_batch.reason.to_string();
-                        current_batch.reason(BatchStopReason::NetworkTaskOlderTasks {
-                            id: task.uid,
-                            inner_reason,
-                        });
-
-                        Batch::NetworkIndexBatch { network_task: task, inner_batch }
-                    }
-                    None => Batch::NetworkReady { task },
-                };
-
-                Ok(Some((batch, current_batch)))
-            }
-            NetworkTopologyState::ImportingDocuments => {
-                // if the import is done we need to go to the next state
-                if network_topology_change.is_import_finished() {
-                    return Ok(Some((Batch::NetworkReady { task }, current_batch)));
-                }
-
-                let res =
-                    self.create_next_batch_unprioritized(rtxn, enqueued, current_batch, |task| {
-                        // in this limited mode of execution, we only want to run tasks:
-                        // 0. with an index
-                        // 1. with a version
-                        // 2. that version equal to the network task version
-
-                        // 0. skip indexless tasks
-                        if task.index_uid().is_none() && task.kind.as_kind() != Kind::IndexSwap {
-                            return true;
-                        }
-
-                        // 1. skip tasks without version
-                        let Some(task_version) =
-                            task.network.as_ref().map(|network| network.network_version())
-                        else {
-                            return true;
-                        };
-
-                        // 2. skip tasks with a version different from the network task version
-                        task_version != change_version
-                    });
-
-                let (batch, mut current_batch) = res?;
-
-                let batch = batch.map(|batch| {
-                    let inner_batch = Box::new(batch);
-                    let inner_reason = current_batch.reason.to_string();
-                    current_batch.reason(BatchStopReason::NetworkTaskImportTasks {
-                        id: task.uid,
-                        inner_reason,
-                    });
-
-                    (Batch::NetworkIndexBatch { network_task: task, inner_batch }, current_batch)
-                });
-
-                Ok(batch)
-            }
-            NetworkTopologyState::ExportingDocuments | NetworkTopologyState::Finished => {
-                Ok(Some((Batch::NetworkReady { task }, current_batch)))
-            }
-        }
     }
 }

crates/index-scheduler/src/scheduler/enterprise_edition/mod.rs

@@ -1,308 +0,0 @@
-// Copyright © 2025 Meilisearch Some Rights Reserved
-// This file is part of Meilisearch Enterprise Edition (EE).
-// Use of this source code is governed by the Business Source License 1.1,
-// as found in the LICENSE-EE file or at <https://mariadb.com/bsl11>
-
-use std::time::Duration;
-
-use bumpalo::Bump;
-use meilisearch_types::milli::documents::PrimaryKey;
-use meilisearch_types::milli::progress::{EmbedderStats, Progress};
-use meilisearch_types::milli::update::new::indexer;
-use meilisearch_types::milli::update::new::indexer::current_edition::sharding::Shards;
-use meilisearch_types::milli::{self};
-use meilisearch_types::network::Remote;
-use meilisearch_types::tasks::network::{NetworkTopologyState, Origin};
-use meilisearch_types::tasks::{KindWithContent, Status, Task};
-use roaring::RoaringBitmap;
-
-use super::create_batch::Batch;
-use crate::scheduler::process_batch::ProcessBatchInfo;
-use crate::scheduler::process_export::{ExportContext, ExportOptions, TargetInstance};
-use crate::utils::ProcessingBatch;
-use crate::{Error, IndexScheduler, Result};
-
-impl IndexScheduler {
-    pub(super) fn process_network_index_batch(
-        &self,
-        mut network_task: Task,
-        inner_batch: Box<Batch>,
-        current_batch: &mut ProcessingBatch,
-        progress: Progress,
-    ) -> Result<(Vec<Task>, ProcessBatchInfo)> {
-        let KindWithContent::NetworkTopologyChange(network_topology_change) =
-            &mut network_task.kind
-        else {
-            tracing::error!("unexpected network kind for network task while processing batch");
-            return Err(Error::CorruptedTaskQueue);
-        };
-
-        let network = network_topology_change.network_for_state();
-
-        let (mut tasks, info) =
-            self.process_batch(*inner_batch, current_batch, progress, network)?;
-
-        for task in &tasks {
-            let Some(network) = task.network.as_ref() else {
-                continue;
-            };
-            let Some(import) = network.import_data() else {
-                continue;
-            };
-            if let Some(index_name) = import.index_name.as_deref() {
-                network_topology_change.process_remote_tasks(
-                    &import.remote_name,
-                    index_name,
-                    import.document_count,
-                );
-            }
-        }
-        network_task.details = Some(network_topology_change.to_details());
-
-        tasks.push(network_task);
-        Ok((tasks, info))
-    }
-
-    pub(super) fn process_network_ready(
-        &self,
-        mut task: Task,
-        progress: Progress,
-    ) -> Result<(Vec<Task>, ProcessBatchInfo)> {
-        let KindWithContent::NetworkTopologyChange(network_topology_change) = &mut task.kind else {
-            tracing::error!("network topology change task has the wrong kind with content");
-            return Err(Error::CorruptedTaskQueue);
-        };
-
-        let Some(task_network) = &task.network else {
-            tracing::error!("network topology change task has no network");
-            return Err(Error::CorruptedTaskQueue);
-        };
-
-        let origin;
-        let origin = match task_network.origin() {
-            Some(origin) => origin,
-            None => {
-                let myself = network_topology_change.in_name().expect("origin is not the leader");
-                origin = Origin {
-                    remote_name: myself.to_string(),
-                    task_uid: task.uid,
-                    network_version: task_network.network_version(),
-                };
-                &origin
-            }
-        };
-
-        let mut moved_documents = None;
-        if let (Some((remotes, out_name)), Some(new_shards)) =
-            (network_topology_change.export_to_process(), network_topology_change.new_shards())
-        {
-            moved_documents = Some(self.balance_documents(
-                remotes,
-                out_name,
-                new_shards,
-                origin,
-                &progress,
-                &self.scheduler.must_stop_processing,
-            )?);
-        }
-        if let Some(moved_documents) = moved_documents {
-            // we need the mut moved documents to avoid a lifetime error in the previous if let.
-            network_topology_change.set_moved(moved_documents);
-        }
-        network_topology_change.update_state();
-        if network_topology_change.state() == NetworkTopologyState::Finished {
-            task.status = Status::Succeeded;
-        }
-
-        task.details = Some(network_topology_change.to_details());
-        Ok((vec![task], Default::default()))
-    }
-
-    fn balance_documents<'a, I: Iterator<Item = (&'a str, &'a Remote)> + Clone>(
-        &self,
-        remotes: I,
-        out_name: &str,
-        new_shards: Shards,
-        network_change_origin: &Origin,
-        progress: &Progress,
-        must_stop_processing: &crate::scheduler::MustStopProcessing,
-    ) -> crate::Result<u64> {
-        // TECHDEBT: this spawns a `ureq` agent additionally to `reqwest`. We probably want to harmonize all of this.
-        let agent = ureq::AgentBuilder::new().timeout(Duration::from_secs(5)).build();
-
-        let mut indexer_alloc = Bump::new();
-
-        let scheduler_rtxn = self.env.read_txn()?;
-
-        let index_count = self.index_mapper.index_count(&scheduler_rtxn)?;
-
-        // when the instance is empty, we still need to tell that to remotes, as they cannot know of that fact and will be waiting for
-        // data
-        if index_count == 0 {
-            for (remote_name, remote) in remotes {
-                let target = TargetInstance {
-                    remote_name: Some(remote_name),
-                    base_url: &remote.url,
-                    api_key: remote.write_api_key.as_deref(),
-                };
-
-                let res = self.export_no_index(
-                    target,
-                    out_name,
-                    network_change_origin,
-                    &agent,
-                    must_stop_processing,
-                );
-
-                if let Err(err) = res {
-                    tracing::warn!("Could not signal not to wait documents to `{remote_name}` due to error: {err}");
-                }
-            }
-            return Ok(0);
-        }
-
-        let mut total_moved_documents = 0;
-
-        self.index_mapper.try_for_each_index::<(), ()>(
-            &scheduler_rtxn,
-            |index_uid, index| -> crate::Result<()> {
-                indexer_alloc.reset();
-                let err = |err| Error::from_milli(err, Some(index_uid.to_string()));
-                let index_rtxn = index.read_txn()?;
-                let all_docids = index.external_documents_ids();
-                let mut documents_to_move_to =
-                    hashbrown::HashMap::<String, RoaringBitmap>::new();
-                let mut documents_to_delete = RoaringBitmap::new();
-
-                for res in all_docids.iter(&index_rtxn)? {
-                    let (external_docid, docid) = res?;
-                    match new_shards.processing_shard(external_docid) {
-                        Some(shard) if shard.is_own => continue,
-                        Some(shard) => {
-                            documents_to_move_to.entry_ref(&shard.name).or_default().insert(docid);
-                        }
-                        None => {
-                            documents_to_delete.insert(docid);
-                        }
-                    }
-                }
-
-                let fields_ids_map = index.fields_ids_map(&index_rtxn)?;
-
-                for (remote_name, remote) in remotes.clone() {
-                    let documents_to_move =
-                        documents_to_move_to.remove(remote_name).unwrap_or_default();
-
-                    let target = TargetInstance {
-                        remote_name: Some(remote_name),
-                        base_url: &remote.url,
-                        api_key: remote.write_api_key.as_deref(),
-                    };
-                    let options = ExportOptions {
-                        index_uid,
-                        payload_size: None,
-                        override_settings: false,
-                        export_mode: super::process_export::ExportMode::NetworkBalancing {
-                            index_count,
-                            export_old_remote_name: out_name,
-                            network_change_origin,
-                        },
-                    };
-                    let ctx = ExportContext {
-                        index,
-                        index_rtxn: &index_rtxn,
-                        universe: &documents_to_move,
-                        progress,
-                        agent: &agent,
-                        must_stop_processing,
-                    };
-
-                    let res = self.export_one_index(target, options, ctx);
-
-                    match res {
-                        Ok(_) =>{ documents_to_delete |= documents_to_move;}
-                        Err(err) => {
-                            tracing::warn!("Could not export documents to `{remote_name}` due to error: {err}\n  - Note: Documents will be kept");
-                        }
-                    }
-
-
-                }
-
-                if documents_to_delete.is_empty() {
-                    return Ok(());
-                }
-
-                total_moved_documents += documents_to_delete.len();
-
-                self.delete_documents_from_index(progress, must_stop_processing, &indexer_alloc, index_uid, index, &err, index_rtxn, documents_to_delete, fields_ids_map)
-            },
-        )?;
-
-        Ok(total_moved_documents)
-    }
-
-    #[allow(clippy::too_many_arguments)]
-    fn delete_documents_from_index(
-        &self,
-        progress: &Progress,
-        must_stop_processing: &super::MustStopProcessing,
-        indexer_alloc: &Bump,
-        index_uid: &str,
-        index: &milli::Index,
-        err: &impl Fn(milli::Error) -> Error,
-        index_rtxn: milli::heed::RoTxn<'_, milli::heed::WithoutTls>,
-        documents_to_delete: RoaringBitmap,
-        fields_ids_map: milli::FieldsIdsMap,
-    ) -> std::result::Result<(), Error> {
-        let mut new_fields_ids_map = fields_ids_map.clone();
-
-        // candidates not empty => index not empty => a primary key is set
-        let primary_key = index.primary_key(&index_rtxn)?.unwrap();
-
-        let primary_key = PrimaryKey::new_or_insert(primary_key, &mut new_fields_ids_map)
-            .map_err(milli::Error::from)
-            .map_err(err)?;
-
-        let mut index_wtxn = index.write_txn()?;
-
-        let mut indexer = indexer::DocumentDeletion::new();
-        indexer.delete_documents_by_docids(documents_to_delete);
-        let document_changes = indexer.into_changes(indexer_alloc, primary_key);
-        let embedders = index
-            .embedding_configs()
-            .embedding_configs(&index_wtxn)
-            .map_err(milli::Error::from)
-            .map_err(err)?;
-        let embedders = self.embedders(index_uid.to_string(), embedders)?;
-        let indexer_config = self.index_mapper.indexer_config();
-        let pool = &indexer_config.thread_pool;
-
-        indexer::index(
-            &mut index_wtxn,
-            index,
-            pool,
-            indexer_config.grenad_parameters(),
-            &fields_ids_map,
-            new_fields_ids_map,
-            None, // document deletion never changes primary key
-            &document_changes,
-            embedders,
-            &|| must_stop_processing.get(),
-            progress,
-            &EmbedderStats::default(),
-        )
-        .map_err(err)?;
-
-        // update stats
-        let mut mapper_wtxn = self.env.write_txn()?;
-        let stats = crate::index_mapper::IndexStats::new(index, &index_wtxn).map_err(err)?;
-        self.index_mapper.store_stats_of(&mut mapper_wtxn, index_uid, &stats)?;
-
-        index_wtxn.commit()?;
-        // update stats after committing changes to index
-        mapper_wtxn.commit()?;
-
-        Ok(())
-    }
-}

crates/index-scheduler/src/scheduler/mod.rs

@@ -1,12 +1,7 @@
 mod autobatcher;
 #[cfg(test)]
 mod autobatcher_test;
-#[cfg(not(feature = "enterprise"))]
-mod community_edition;
 mod create_batch;
-#[cfg(feature = "enterprise")]
-mod enterprise_edition;
-
 mod process_batch;
 mod process_dump_creation;
 mod process_export;
@@ -26,10 +21,10 @@ use std::path::PathBuf;
 use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
 use std::sync::Arc;
 
+use convert_case::{Case, Casing as _};
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::heed::{Env, WithoutTls};
 use meilisearch_types::milli;
-use meilisearch_types::milli::update::S3SnapshotOptions;
 use meilisearch_types::tasks::Status;
 use process_batch::ProcessBatchInfo;
 use rayon::current_num_threads;
@@ -92,14 +87,11 @@ pub struct Scheduler {
 
     /// Snapshot compaction status.
     pub(crate) experimental_no_snapshot_compaction: bool,
-
-    /// S3 Snapshot options.
-    pub(crate) s3_snapshot_options: Option<S3SnapshotOptions>,
 }
 
 impl Scheduler {
-    pub(crate) fn private_clone(&self) -> Self {
+    pub(crate) fn private_clone(&self) -> Scheduler {
-        Self {
+        Scheduler {
             must_stop_processing: self.must_stop_processing.clone(),
             wake_up: self.wake_up.clone(),
             autobatching_enabled: self.autobatching_enabled,
@@ -111,53 +103,23 @@ impl Scheduler {
             version_file_path: self.version_file_path.clone(),
             embedding_cache_cap: self.embedding_cache_cap,
             experimental_no_snapshot_compaction: self.experimental_no_snapshot_compaction,
-            s3_snapshot_options: self.s3_snapshot_options.clone(),
         }
     }
 
     pub fn new(options: &IndexSchedulerOptions, auth_env: Env<WithoutTls>) -> Scheduler {
-        let IndexSchedulerOptions {
-            version_file_path,
-            auth_path: _,
-            tasks_path: _,
-            update_file_path: _,
-            indexes_path: _,
-            snapshots_path,
-            dumps_path,
-            cli_webhook_url: _,
-            cli_webhook_authorization: _,
-            task_db_size: _,
-            index_base_map_size: _,
-            enable_mdb_writemap: _,
-            index_growth_amount: _,
-            index_count: _,
-            indexer_config,
-            autobatching_enabled,
-            cleanup_enabled: _,
-            max_number_of_tasks: _,
-            max_number_of_batched_tasks,
-            batched_tasks_size_limit,
-            export_default_payload_size_bytes: _,
-            instance_features: _,
-            auto_upgrade: _,
-            embedding_cache_cap,
-            experimental_no_snapshot_compaction,
-        } = options;
-
         Scheduler {
             must_stop_processing: MustStopProcessing::default(),
             // we want to start the loop right away in case meilisearch was ctrl+Ced while processing things
             wake_up: Arc::new(SignalEvent::auto(true)),
-            autobatching_enabled: *autobatching_enabled,
+            autobatching_enabled: options.autobatching_enabled,
-            max_number_of_batched_tasks: *max_number_of_batched_tasks,
+            max_number_of_batched_tasks: options.max_number_of_batched_tasks,
-            batched_tasks_size_limit: *batched_tasks_size_limit,
+            batched_tasks_size_limit: options.batched_tasks_size_limit,
-            dumps_path: dumps_path.clone(),
+            dumps_path: options.dumps_path.clone(),
-            snapshots_path: snapshots_path.clone(),
+            snapshots_path: options.snapshots_path.clone(),
             auth_env,
-            version_file_path: version_file_path.clone(),
+            version_file_path: options.version_file_path.clone(),
-            embedding_cache_cap: *embedding_cache_cap,
+            embedding_cache_cap: options.embedding_cache_cap,
-            experimental_no_snapshot_compaction: *experimental_no_snapshot_compaction,
+            experimental_no_snapshot_compaction: options.experimental_no_snapshot_compaction,
-            s3_snapshot_options: indexer_config.s3_snapshot_options.clone(),
         }
     }
 }
@@ -183,8 +145,6 @@ impl IndexScheduler {
             self.breakpoint(crate::test_utils::Breakpoint::Start);
         }
 
-        let previous_processing_batch = self.processing_tasks.write().unwrap().stop_processing();
-
         if self.cleanup_enabled {
             let mut wtxn = self.env.write_txn()?;
             self.queue.cleanup_task_queue(&mut wtxn)?;
@@ -192,16 +152,11 @@ impl IndexScheduler {
         }
 
         let rtxn = self.env.read_txn().map_err(Error::HeedTransaction)?;
-        let (batch, mut processing_batch) = match self
+        let (batch, mut processing_batch) =
-            .create_next_batch(&rtxn, &previous_processing_batch.processing)
+            match self.create_next_batch(&rtxn).map_err(|e| Error::CreateBatch(Box::new(e)))? {
-            .map_err(|e| Error::CreateBatch(Box::new(e)))?
+                Some(batch) => batch,
-        {
+                None => return Ok(TickOutcome::WaitForSignal),
-            Some(batch) => batch,
+            };
-            None => {
-                *self.processing_tasks.write().unwrap() = previous_processing_batch;
-                return Ok(TickOutcome::WaitForSignal);
-            }
-        };
         let index_uid = batch.index_uid().map(ToOwned::to_owned);
         drop(rtxn);
 
@@ -231,12 +186,7 @@ impl IndexScheduler {
                 let handle = std::thread::Builder::new()
                     .name(String::from("batch-operation"))
                     .spawn_scoped(s, move || {
-                        cloned_index_scheduler.process_batch(
+                        cloned_index_scheduler.process_batch(batch, processing_batch, p)
-                            batch,
-                            processing_batch,
-                            p,
-                            &self.network(),
-                        )
                     })
                     .unwrap();
 
@@ -277,14 +227,7 @@ impl IndexScheduler {
         self.maybe_fail(crate::test_utils::FailureLocation::AcquiringWtxn)?;
 
         progress.update_progress(BatchProgress::WritingTasksToDisk);
-
         processing_batch.finished();
-        // whether the batch made progress.
-        // a batch make progress if it failed or if it contains at least one fully processed (or cancelled) task.
-        //
-        // if a batch did not make progress, it means that all of its tasks are waiting on the scheduler to make progress,
-        // and so we must wait for new tasks. Such a batch is not persisted to DB, and is resumed on the next tick.
-        let mut batch_made_progress = false;
         let mut stop_scheduler_forever = false;
         let mut wtxn = self.env.write_txn().map_err(Error::HeedTransaction)?;
         let mut canceled = RoaringBitmap::new();
@@ -305,11 +248,7 @@ impl IndexScheduler {
                 #[allow(unused_variables)]
                 for (i, mut task) in tasks.into_iter().enumerate() {
                     task_progress.fetch_add(1, Ordering::Relaxed);
-                    processing_batch.update_from_task(&task);
+                    processing_batch.update(&mut task);
-                    if !matches!(task.status, Status::Processing | Status::Enqueued) {
-                        batch_made_progress = true;
-                        processing_batch.finish_task(&mut task);
-                    }
                     if task.status == Status::Canceled {
                         canceled.insert(task.uid);
                         canceled_by = task.canceled_by;
@@ -376,9 +315,6 @@ impl IndexScheduler {
             }
             // In case of a failure we must get back and patch all the tasks with the error.
             Err(err) => {
-                // always persist failed batches
-                batch_made_progress = true;
-
                 #[cfg(test)]
                 self.breakpoint(crate::test_utils::Breakpoint::ProcessBatchFailed);
                 let (task_progress, task_progress_obj) = AtomicTaskStep::new(ids.len() as u32);
@@ -402,10 +338,7 @@ impl IndexScheduler {
                     task.status = Status::Failed;
                     task.error = Some(error.clone());
                     task.details = task.details.map(|d| d.to_failed());
-                    processing_batch.update_from_task(&task);
+                    processing_batch.update(&mut task);
-                    if !matches!(task.status, Status::Processing | Status::Enqueued) {
-                        processing_batch.finish_task(&mut task);
-                    }
 
                     #[cfg(test)]
                     self.maybe_fail(
@@ -428,12 +361,44 @@ impl IndexScheduler {
         let ProcessBatchInfo { congestion, pre_commit_dabases_sizes, post_commit_dabases_sizes } =
             process_batch_info;
 
-        processing_batch.write_stats(
+        processing_batch.stats.progress_trace =
-            &progress,
+            progress.accumulated_durations().into_iter().map(|(k, v)| (k, v.into())).collect();
-            congestion,
+        processing_batch.stats.write_channel_congestion = congestion.map(|congestion| {
-            pre_commit_dabases_sizes,
+            let mut congestion_info = serde_json::Map::new();
-            post_commit_dabases_sizes,
+            congestion_info.insert("attempts".into(), congestion.attempts.into());
-        );
+            congestion_info.insert("blocking_attempts".into(), congestion.blocking_attempts.into());
+            congestion_info.insert("blocking_ratio".into(), congestion.congestion_ratio().into());
+            congestion_info
+        });
+        processing_batch.stats.internal_database_sizes = pre_commit_dabases_sizes
+            .iter()
+            .flat_map(|(dbname, pre_size)| {
+                post_commit_dabases_sizes
+                    .get(dbname)
+                    .map(|post_size| {
+                        use std::cmp::Ordering::{Equal, Greater, Less};
+
+                        use byte_unit::Byte;
+                        use byte_unit::UnitType::Binary;
+
+                        let post = Byte::from_u64(*post_size as u64).get_appropriate_unit(Binary);
+                        let diff_size = post_size.abs_diff(*pre_size) as u64;
+                        let diff = Byte::from_u64(diff_size).get_appropriate_unit(Binary);
+                        let sign = match post_size.cmp(pre_size) {
+                            Equal => return None,
+                            Greater => "+",
+                            Less => "-",
+                        };
+
+                        Some((
+                            dbname.to_case(Case::Camel),
+                            format!("{post:#.2} ({sign}{diff:#.2})").into(),
+                        ))
+                    })
+                    .into_iter()
+                    .flatten()
+            })
+            .collect();
 
         if let Some(congestion) = congestion {
             tracing::debug!(
@@ -446,49 +411,46 @@ impl IndexScheduler {
 
         tracing::debug!("call trace: {:?}", progress.accumulated_durations());
 
-        if batch_made_progress {
+        self.queue.write_batch(&mut wtxn, processing_batch, &ids)?;
-            self.queue.write_batch(&mut wtxn, processing_batch, &ids)?;
-        }
 
         #[cfg(test)]
         self.maybe_fail(crate::test_utils::FailureLocation::CommittingWtxn)?;
 
         wtxn.commit().map_err(Error::HeedTransaction)?;
 
-        if batch_made_progress {
+        // We should stop processing AFTER everything is processed and written to disk otherwise, a batch (which only lives in RAM) may appear in the processing task
-            // We should stop processing AFTER everything is processed and written to disk otherwise, a batch (which only lives in RAM) may appear in the processing task
+        // and then become « not found » for some time until the commit everything is written and the final commit is made.
-            // and then become « not found » for some time until the commit everything is written and the final commit is made.
+        self.processing_tasks.write().unwrap().stop_processing();
-            self.processing_tasks.write().unwrap().stop_processing();
 
-            // Once the tasks are committed, we should delete all the update files associated ASAP to avoid leaking files in case of a restart
+        // Once the tasks are committed, we should delete all the update files associated ASAP to avoid leaking files in case of a restart
-            tracing::debug!("Deleting the update files");
+        tracing::debug!("Deleting the update files");
 
-            //We take one read transaction **per thread**. Then, every thread is going to pull out new IDs from the roaring bitmap with the help of an atomic shared index into the bitmap
+        //We take one read transaction **per thread**. Then, every thread is going to pull out new IDs from the roaring bitmap with the help of an atomic shared index into the bitmap
-            let idx = AtomicU32::new(0);
+        let idx = AtomicU32::new(0);
-            (0..current_num_threads()).into_par_iter().try_for_each(|_| -> Result<()> {
+        (0..current_num_threads()).into_par_iter().try_for_each(|_| -> Result<()> {
-                let rtxn = self.read_txn()?;
+            let rtxn = self.read_txn()?;
-                while let Some(id) = ids.select(idx.fetch_add(1, Ordering::Relaxed)) {
+            while let Some(id) = ids.select(idx.fetch_add(1, Ordering::Relaxed)) {
-                    let task = self
+                let task = self
-                        .queue
+                    .queue
-                        .tasks
+                    .tasks
-                        .get_task(&rtxn, id)
+                    .get_task(&rtxn, id)
-                        .map_err(|e| Error::UnrecoverableError(Box::new(e)))?
+                    .map_err(|e| Error::UnrecoverableError(Box::new(e)))?
-                        .ok_or(Error::CorruptedTaskQueue)?;
+                    .ok_or(Error::CorruptedTaskQueue)?;
-                    if let Err(e) = self.queue.delete_persisted_task_data(&task) {
+                if let Err(e) = self.queue.delete_persisted_task_data(&task) {
-                        tracing::error!(
+                    tracing::error!(
                         "Failure to delete the content files associated with task {}. Error: {e}",
                         task.uid
                     );
-                    }
                 }
-                Ok(())
+            }
-            })?;
+            Ok(())
+        })?;
 
-            self.notify_webhooks(ids);
+        self.notify_webhooks(ids);
-        }
 
         #[cfg(test)]
         self.breakpoint(crate::test_utils::Breakpoint::AfterProcessing);
+
         if stop_scheduler_forever {
             Ok(TickOutcome::StopProcessingForever)
         } else {

crates/index-scheduler/src/scheduler/process_batch.rs

@@ -1,28 +1,22 @@
 use std::collections::{BTreeSet, HashMap, HashSet};
-use std::fs::{remove_file, File};
-use std::io::{ErrorKind, Seek, SeekFrom};
 use std::panic::{catch_unwind, AssertUnwindSafe};
 use std::sync::atomic::Ordering;
 
-use byte_unit::Byte;
 use meilisearch_types::batches::{BatchEnqueuedAt, BatchId};
 use meilisearch_types::heed::{RoTxn, RwTxn};
-use meilisearch_types::milli::heed::CompactionOption;
 use meilisearch_types::milli::progress::{Progress, VariableNameStep};
 use meilisearch_types::milli::{self, ChannelCongestion};
-use meilisearch_types::network::Network;
 use meilisearch_types::tasks::{Details, IndexSwap, Kind, KindWithContent, Status, Task};
 use meilisearch_types::versioning::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH};
 use milli::update::Settings as MilliSettings;
 use roaring::RoaringBitmap;
-use tempfile::{PersistError, TempPath};
 use time::OffsetDateTime;
 
 use super::create_batch::Batch;
 use crate::processing::{
     AtomicBatchStep, AtomicTaskStep, CreateIndexProgress, DeleteIndexProgress, FinalizingIndexStep,
-    IndexCompaction, InnerSwappingTwoIndexes, SwappingTheIndexes, TaskCancelationProgress,
+    InnerSwappingTwoIndexes, SwappingTheIndexes, TaskCancelationProgress, TaskDeletionProgress,
-    TaskDeletionProgress, UpdateIndexProgress,
+    UpdateIndexProgress,
 };
 use crate::utils::{
     self, remove_n_tasks_datetime_earlier_than, remove_task_datetime, swap_index_uid_in_task,
@@ -30,9 +24,6 @@ use crate::utils::{
 };
 use crate::{Error, IndexScheduler, Result, TaskId};
 
-/// The name of the copy of the data.mdb file used during compaction.
-const DATA_MDB_COPY_NAME: &str = "data.mdb.cpy";
-
 #[derive(Debug, Default)]
 pub struct ProcessBatchInfo {
     /// The write channel congestion. None when unavailable: settings update.
@@ -56,7 +47,6 @@ impl IndexScheduler {
         batch: Batch,
         current_batch: &mut ProcessingBatch,
         progress: Progress,
-        network: &Network,
     ) -> Result<(Vec<Task>, ProcessBatchInfo)> {
         #[cfg(test)]
         {
@@ -178,7 +168,6 @@ impl IndexScheduler {
                     op,
                     &progress,
                     current_batch.embedder_stats.clone(),
-                    network,
                 )?;
 
                 {
@@ -238,7 +227,6 @@ impl IndexScheduler {
                     Batch::IndexUpdate { index_uid, primary_key, new_index_uid: None, task },
                     current_batch,
                     progress,
-                    network,
                 )
             }
             Batch::IndexUpdate { index_uid, primary_key, new_index_uid, mut task } => {
@@ -430,47 +418,6 @@ impl IndexScheduler {
                 task.status = Status::Succeeded;
                 Ok((vec![task], ProcessBatchInfo::default()))
             }
-            Batch::IndexCompaction { index_uid: _, mut task } => {
-                let KindWithContent::IndexCompaction { index_uid } = &task.kind else {
-                    unreachable!()
-                };
-
-                let rtxn = self.env.read_txn()?;
-                let ret = catch_unwind(AssertUnwindSafe(|| {
-                    self.apply_compaction(&rtxn, &progress, index_uid)
-                }));
-
-                let (pre_size, post_size) = match ret {
-                    Ok(Ok(stats)) => stats,
-                    Ok(Err(Error::AbortedTask)) => return Err(Error::AbortedTask),
-                    Ok(Err(e)) => return Err(e),
-                    Err(e) => {
-                        let msg = match e.downcast_ref::<&'static str>() {
-                            Some(s) => *s,
-                            None => match e.downcast_ref::<String>() {
-                                Some(s) => &s[..],
-                                None => "Box<dyn Any>",
-                            },
-                        };
-                        return Err(Error::Export(Box::new(Error::ProcessBatchPanicked(
-                            msg.to_string(),
-                        ))));
-                    }
-                };
-
-                task.status = Status::Succeeded;
-                if let Some(Details::IndexCompaction {
-                    index_uid: _,
-                    pre_compaction_size,
-                    post_compaction_size,
-                }) = task.details.as_mut()
-                {
-                    *pre_compaction_size = Some(Byte::from_u64(pre_size));
-                    *post_compaction_size = Some(Byte::from_u64(post_size));
-                }
-
-                Ok((vec![task], ProcessBatchInfo::default()))
-            }
             Batch::Export { mut task } => {
                 let KindWithContent::Export { url, api_key, payload_size, indexes } = &task.kind
                 else {
@@ -543,99 +490,9 @@ impl IndexScheduler {
 
                 Ok((tasks, ProcessBatchInfo::default()))
             }
-            Batch::NetworkIndexBatch { network_task, inner_batch } => {
-                self.process_network_index_batch(network_task, inner_batch, current_batch, progress)
-            }
-            Batch::NetworkReady { task } => self.process_network_ready(task, progress),
         }
     }
 
-    fn apply_compaction(
-        &self,
-        rtxn: &RoTxn,
-        progress: &Progress,
-        index_uid: &str,
-    ) -> Result<(u64, u64)> {
-        // 1. Verify that the index exists
-        if !self.index_mapper.index_exists(rtxn, index_uid)? {
-            return Err(Error::IndexNotFound(index_uid.to_owned()));
-        }
-
-        // 2. We retrieve the index and create a temporary file in the index directory
-        progress.update_progress(IndexCompaction::RetrieveTheIndex);
-        let index = self.index_mapper.index(rtxn, index_uid)?;
-
-        // the index operation can take a long time, so save this handle to make it available to the search for the duration of the tick
-        self.index_mapper
-            .set_currently_updating_index(Some((index_uid.to_string(), index.clone())));
-
-        progress.update_progress(IndexCompaction::CreateTemporaryFile);
-        let src_path = index.path().join("data.mdb");
-        let pre_size = std::fs::metadata(&src_path)?.len();
-
-        let dst_path = TempPath::from_path(index.path().join(DATA_MDB_COPY_NAME));
-        let file = File::create(&dst_path)?;
-        let mut file = tempfile::NamedTempFile::from_parts(file, dst_path);
-
-        // 3. We copy the index data to the temporary file
-        progress.update_progress(IndexCompaction::CopyAndCompactTheIndex);
-        index
-            .copy_to_file(file.as_file_mut(), CompactionOption::Enabled)
-            .map_err(|error| Error::Milli { error, index_uid: Some(index_uid.to_string()) })?;
-        // ...and reset the file position as specified in the documentation
-        file.seek(SeekFrom::Start(0))?;
-
-        // 4. We replace the index data file with the temporary file
-        progress.update_progress(IndexCompaction::PersistTheCompactedIndex);
-        match file.persist(src_path) {
-            Ok(file) => file.sync_all()?,
-            // TODO see if we have a _resource busy_ error and probably handle this by:
-            //      1. closing the index, 2. replacing and 3. reopening it
-            Err(PersistError { error, file: _ }) => return Err(Error::IoError(error)),
-        };
-
-        // 5. Prepare to close the index
-        progress.update_progress(IndexCompaction::CloseTheIndex);
-
-        // unmark that the index is the processing one so we don't keep a handle to it, preventing its closing
-        self.index_mapper.set_currently_updating_index(None);
-
-        self.index_mapper.close_index(rtxn, index_uid)?;
-        drop(index);
-
-        progress.update_progress(IndexCompaction::ReopenTheIndex);
-        // 6. Reopen the index
-        // The index will use the compacted data file when being reopened
-        let index = self.index_mapper.index(rtxn, index_uid)?;
-
-        // if the update processed successfully, we're going to store the new
-        // stats of the index. Since the tasks have already been processed and
-        // this is a non-critical operation. If it fails, we should not fail
-        // the entire batch.
-        let res = || -> Result<_> {
-            let mut wtxn = self.env.write_txn()?;
-            let index_rtxn = index.read_txn()?;
-            let stats = crate::index_mapper::IndexStats::new(&index, &index_rtxn)
-                .map_err(|e| Error::from_milli(e, Some(index_uid.to_string())))?;
-            self.index_mapper.store_stats_of(&mut wtxn, index_uid, &stats)?;
-            wtxn.commit()?;
-            Ok(stats.database_size)
-        }();
-
-        let post_size = match res {
-            Ok(post_size) => post_size,
-            Err(e) => {
-                tracing::error!(
-                    error = &e as &dyn std::error::Error,
-                    "Could not write the stats of the index"
-                );
-                0
-            }
-        };
-
-        Ok((pre_size, post_size))
-    }
-
     /// Swap the index `lhs` with the index `rhs`.
     fn apply_index_swap(
         &self,
@@ -923,10 +780,9 @@ impl IndexScheduler {
 
         let enqueued_tasks = &self.queue.tasks.get_status(rtxn, Status::Enqueued)?;
 
-        // 0. Check if any upgrade or compaction tasks were matched.
+        // 0. Check if any upgrade task was matched.
         //    If so, we cancel all the failed or enqueued upgrade tasks.
         let upgrade_tasks = &self.queue.tasks.get_kind(rtxn, Kind::UpgradeDatabase)?;
-        let compaction_tasks = &self.queue.tasks.get_kind(rtxn, Kind::IndexCompaction)?;
         let is_canceling_upgrade = !matched_tasks.is_disjoint(upgrade_tasks);
         if is_canceling_upgrade {
             let failed_tasks = self.queue.tasks.get_status(rtxn, Status::Failed)?;
@@ -991,33 +847,7 @@ impl IndexScheduler {
             }
         }
 
-        // 3. If we are cancelling a compaction task, remove the tempfiles after incomplete compactions
+        // 3. We now have a list of tasks to cancel, cancel them
-        for compaction_task in &tasks_to_cancel & compaction_tasks {
-            progress.update_progress(TaskCancelationProgress::CleaningCompactionLeftover);
-            let task = self.queue.tasks.get_task(rtxn, compaction_task)?.unwrap();
-            let Some(Details::IndexCompaction {
-                index_uid,
-                pre_compaction_size: _,
-                post_compaction_size: _,
-            }) = task.details
-            else {
-                unreachable!("wrong details for compaction task {compaction_task}")
-            };
-
-            let index_path = match self.index_mapper.index_mapping.get(rtxn, &index_uid)? {
-                Some(index_uuid) => self.index_mapper.index_path(index_uuid),
-                None => continue,
-            };
-
-            if let Err(e) = remove_file(index_path.join(DATA_MDB_COPY_NAME)) {
-                match e.kind() {
-                    ErrorKind::NotFound => (),
-                    _ => return Err(Error::IoError(e)),
-                }
-            }
-        }
-
-        // 4. We now have a list of tasks to cancel, cancel them
         let (task_progress, progress_obj) = AtomicTaskStep::new(tasks_to_cancel.len() as u32);
         progress.update_progress(progress_obj);
 

crates/index-scheduler/src/scheduler/process_export.rs

@@ -1,6 +1,5 @@
 use std::collections::BTreeMap;
 use std::io::{self, Write as _};
-use std::ops::ControlFlow;
 use std::sync::atomic;
 use std::time::Duration;
 
@@ -8,7 +7,6 @@ use backoff::ExponentialBackoff;
 use byte_unit::Byte;
 use flate2::write::GzEncoder;
 use flate2::Compression;
-use meilisearch_types::error::Code;
 use meilisearch_types::index_uid_pattern::IndexUidPattern;
 use meilisearch_types::milli::constants::RESERVED_VECTORS_FIELD_NAME;
 use meilisearch_types::milli::index::EmbeddingsWithMetadata;
@@ -17,10 +15,7 @@ use meilisearch_types::milli::update::{request_threads, Setting};
 use meilisearch_types::milli::vector::parsed_vectors::{ExplicitVectors, VectorOrArrayOfVectors};
 use meilisearch_types::milli::{self, obkv_to_json, Filter, InternalError};
 use meilisearch_types::settings::{self, SecretPolicy};
-use meilisearch_types::tasks::network::headers::SetHeader as _;
-use meilisearch_types::tasks::network::{headers, ImportData, ImportMetadata, Origin};
 use meilisearch_types::tasks::{DetailsExportIndexSettings, ExportIndexSettings};
-use roaring::RoaringBitmap;
 use serde::Deserialize;
 use ureq::{json, Response};
 
@@ -55,7 +50,6 @@ impl IndexScheduler {
         let agent = ureq::AgentBuilder::new().timeout(Duration::from_secs(5)).build();
         let must_stop_processing = self.scheduler.must_stop_processing.clone();
         for (i, (_pattern, uid, export_settings)) in indexes.iter().enumerate() {
-            let err = |err| Error::from_milli(err, Some(uid.to_string()));
             if must_stop_processing.get() {
                 return Err(Error::AbortedTask);
             }
@@ -67,473 +61,260 @@ impl IndexScheduler {
             ));
 
             let ExportIndexSettings { filter, override_settings } = export_settings;
-
             let index = self.index(uid)?;
             let index_rtxn = index.read_txn()?;
-            let filter = filter.as_ref().map(Filter::from_json).transpose().map_err(err)?.flatten();
+            let bearer = api_key.map(|api_key| format!("Bearer {api_key}"));
-            let filter_universe =
+
-                filter.map(|f| f.evaluate(&index_rtxn, &index)).transpose().map_err(err)?;
+            // First, check if the index already exists
-            let whole_universe =
+            let url = format!("{base_url}/indexes/{uid}");
-                index.documents_ids(&index_rtxn).map_err(milli::Error::from).map_err(err)?;
+            let response = retry(&must_stop_processing, || {
-            let universe = filter_universe.unwrap_or(whole_universe);
+                let mut request = agent.get(&url);
-            let target = TargetInstance { remote_name: None, base_url, api_key };
+                if let Some(bearer) = &bearer {
-            let ctx = ExportContext {
+                    request = request.set("Authorization", bearer);
-                index: &index,
+                }
-                index_rtxn: &index_rtxn,
+
-                universe: &universe,
+                request.send_bytes(Default::default()).map_err(into_backoff_error)
-                progress: &progress,
+            });
-                agent: &agent,
+            let index_exists = match response {
-                must_stop_processing: &must_stop_processing,
+                Ok(response) => response.status() == 200,
+                Err(Error::FromRemoteWhenExporting { code, .. }) if code == "index_not_found" => {
+                    false
+                }
+                Err(e) => return Err(e),
             };
-            let options = ExportOptions {
-                index_uid: uid,
-                payload_size,
-                override_settings: *override_settings,
-                export_mode: ExportMode::ExportRoute,
-            };
-            let total_documents = self.export_one_index(target, options, ctx)?;
 
-            output.insert(
+            let primary_key = index
-                IndexUidPattern::new_unchecked(uid.clone()),
+                .primary_key(&index_rtxn)
-                DetailsExportIndexSettings {
+                .map_err(|e| Error::from_milli(e.into(), Some(uid.to_string())))?;
-                    settings: (*export_settings).clone(),
-                    matched_documents: Some(total_documents),
-                },
-            );
-        }
 
-        Ok(output)
+            // Create the index
-    }
+            if !index_exists {
-
+                let url = format!("{base_url}/indexes");
-    pub(super) fn export_one_index(
+                retry(&must_stop_processing, || {
-        &self,
+                    let mut request = agent.post(&url);
-        target: TargetInstance<'_>,
+                    if let Some(bearer) = &bearer {
-        options: ExportOptions<'_>,
-        ctx: ExportContext<'_>,
-    ) -> Result<u64, Error> {
-        let err = |err| Error::from_milli(err, Some(options.index_uid.to_string()));
-        let total_index_documents = ctx.universe.len();
-        let task_network = options.task_network(total_index_documents);
-
-        let bearer = target.api_key.map(|api_key| format!("Bearer {api_key}"));
-        let url = format!(
-            "{base_url}/indexes/{index_uid}",
-            base_url = target.base_url,
-            index_uid = options.index_uid
-        );
-        let response = retry(ctx.must_stop_processing, || {
-            let mut request = ctx.agent.get(&url);
-            if let Some(bearer) = &bearer {
-                request = request.set("Authorization", bearer);
-            }
-
-            request.send_bytes(Default::default()).map_err(into_backoff_error)
-        });
-        let index_exists = match response {
-            Ok(response) => response.status() == 200,
-            Err(Error::FromRemoteWhenExporting { code, .. })
-                if code == Code::IndexNotFound.name() =>
-            {
-                false
-            }
-            Err(e) => return Err(e),
-        };
-        let primary_key =
-            ctx.index.primary_key(ctx.index_rtxn).map_err(milli::Error::from).map_err(err)?;
-        if !index_exists {
-            let url = format!("{base_url}/indexes", base_url = target.base_url);
-            let _ = handle_response(
-                target.remote_name,
-                retry(ctx.must_stop_processing, || {
-                    let mut request = ctx.agent.post(&url);
-
-                    if let Some((import_data, origin, metadata)) = &task_network {
-                        request = set_network_ureq_headers(request, import_data, origin, metadata);
-                    }
-
-                    if let Some(bearer) = bearer.as_ref() {
                         request = request.set("Authorization", bearer);
                     }
-                    let index_param =
+                    let index_param = json!({ "uid": uid, "primaryKey": primary_key });
-                        json!({ "uid": options.index_uid, "primaryKey": primary_key });
-
                     request.send_json(&index_param).map_err(into_backoff_error)
-                }),
+                })?;
-            )?;
+            }
-        }
+
-        if index_exists && options.override_settings {
+            // Patch the index primary key
-            let _ = handle_response(
+            if index_exists && *override_settings {
-                target.remote_name,
+                let url = format!("{base_url}/indexes/{uid}");
-                retry(ctx.must_stop_processing, || {
+                retry(&must_stop_processing, || {
-                    let mut request = ctx.agent.patch(&url);
+                    let mut request = agent.patch(&url);
-                    if let Some((import_data, origin, metadata)) = &task_network {
-                        request = set_network_ureq_headers(request, import_data, origin, metadata);
-                    }
                     if let Some(bearer) = &bearer {
                         request = request.set("Authorization", bearer);
                     }
                     let index_param = json!({ "primaryKey": primary_key });
                     request.send_json(&index_param).map_err(into_backoff_error)
-                }),
+                })?;
-            )?;
-        }
-        if !index_exists || options.override_settings {
-            let mut settings =
-                settings::settings(ctx.index, ctx.index_rtxn, SecretPolicy::RevealSecrets)
-                    .map_err(err)?;
-            // Remove the experimental chat setting if not enabled
-            if self.features().check_chat_completions("exporting chat settings").is_err() {
-                settings.chat = Setting::NotSet;
             }
-            // Retry logic for sending settings
-            let url = format!(
-                "{base_url}/indexes/{index_uid}/settings",
-                base_url = target.base_url,
-                index_uid = options.index_uid
-            );
-
-            let _ = handle_response(
-                target.remote_name,
-                retry(ctx.must_stop_processing, || {
-                    let mut request = ctx.agent.patch(&url);
-
-                    if let Some((import_data, origin, metadata)) = &task_network {
-                        request = set_network_ureq_headers(request, import_data, origin, metadata);
-                    }
 
+            // Send the index settings
+            if !index_exists || *override_settings {
+                let mut settings =
+                    settings::settings(&index, &index_rtxn, SecretPolicy::RevealSecrets)
+                        .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
+                // Remove the experimental chat setting if not enabled
+                if self.features().check_chat_completions("exporting chat settings").is_err() {
+                    settings.chat = Setting::NotSet;
+                }
+                // Retry logic for sending settings
+                let url = format!("{base_url}/indexes/{uid}/settings");
+                retry(&must_stop_processing, || {
+                    let mut request = agent.patch(&url);
                     if let Some(bearer) = bearer.as_ref() {
                         request = request.set("Authorization", bearer);
                     }
                     request.send_json(settings.clone()).map_err(into_backoff_error)
-                }),
+                })?;
-            )?;
-        }
-
-        let fields_ids_map = ctx.index.fields_ids_map(ctx.index_rtxn)?;
-        let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
-        let total_documents = ctx.universe.len() as u32;
-        let (step, progress_step) = AtomicDocumentStep::new(total_documents);
-        ctx.progress.update_progress(progress_step);
-
-        let limit = options
-            .payload_size
-            .map(|ps| ps.as_u64() as usize)
-            .unwrap_or(self.export_default_payload_size_bytes.as_u64() as usize);
-        let documents_url = format!(
-            "{base_url}/indexes/{index_uid}/documents",
-            base_url = target.base_url,
-            index_uid = options.index_uid
-        );
-
-        // no document to send, but we must still send a task when performing network balancing
-        if ctx.universe.is_empty() {
-            if let Some((import_data, network_change_origin, metadata)) = task_network {
-                let mut compressed_buffer = Vec::new();
-                // ignore control flow, we're returning anyway
-                let _ = send_buffer(
-                    b" ", // needs something otherwise meili complains about missing payload
-                    &mut compressed_buffer,
-                    ctx.must_stop_processing,
-                    ctx.agent,
-                    &documents_url,
-                    target.remote_name,
-                    bearer.as_deref(),
-                    Some(&(import_data, network_change_origin.clone(), metadata)),
-                    &err,
-                )?;
             }
-            return Ok(0);
-        }
 
-        let results = request_threads()
+            let filter = filter
-            .broadcast(|broadcast| {
+                .as_ref()
-                let mut task_network = options.task_network(total_index_documents);
+                .map(Filter::from_json)
+                .transpose()
+                .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?
+                .flatten();
 
-                let index_rtxn = ctx.index.read_txn().map_err(milli::Error::from).map_err(err)?;
+            let filter_universe = filter
+                .map(|f| f.evaluate(&index_rtxn, &index))
+                .transpose()
+                .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
+            let whole_universe = index
+                .documents_ids(&index_rtxn)
+                .map_err(|e| Error::from_milli(e.into(), Some(uid.to_string())))?;
+            let universe = filter_universe.unwrap_or(whole_universe);
 
-                let mut buffer = Vec::new();
+            let fields_ids_map = index.fields_ids_map(&index_rtxn)?;
-                let mut tmp_buffer = Vec::new();
+            let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
-                let mut compressed_buffer = Vec::new();
-                for (i, docid) in ctx.universe.iter().enumerate() {
-                    if i % broadcast.num_threads() != broadcast.index() {
-                        continue;
-                    }
-                    if let Some((import_data, _, metadata)) = &mut task_network {
-                        import_data.document_count += 1;
-                        metadata.task_key = Some(docid);
-                    }
 
-                    let document = ctx.index.document(&index_rtxn, docid).map_err(err)?;
+            // We don't need to keep this one alive as we will
+            // spawn many threads to process the documents
+            drop(index_rtxn);
 
-                    let mut document =
+            let total_documents = universe.len() as u32;
-                        obkv_to_json(&all_fields, &fields_ids_map, document).map_err(err)?;
+            let (step, progress_step) = AtomicDocumentStep::new(total_documents);
+            progress.update_progress(progress_step);
 
-                    // TODO definitely factorize this code
+            output.insert(
-                    'inject_vectors: {
+                IndexUidPattern::new_unchecked(uid.clone()),
-                        let embeddings = ctx.index.embeddings(&index_rtxn, docid).map_err(err)?;
+                DetailsExportIndexSettings {
+                    settings: (*export_settings).clone(),
+                    matched_documents: Some(total_documents as u64),
+                },
+            );
 
-                        if embeddings.is_empty() {
+            let limit = payload_size.map(|ps| ps.as_u64() as usize).unwrap_or(20 * 1024 * 1024); // defaults to 20 MiB
-                            break 'inject_vectors;
+            let documents_url = format!("{base_url}/indexes/{uid}/documents");
+
+            let results = request_threads()
+                .broadcast(|ctx| {
+                    let index_rtxn = index
+                        .read_txn()
+                        .map_err(|e| Error::from_milli(e.into(), Some(uid.to_string())))?;
+
+                    let mut buffer = Vec::new();
+                    let mut tmp_buffer = Vec::new();
+                    let mut compressed_buffer = Vec::new();
+                    for (i, docid) in universe.iter().enumerate() {
+                        if i % ctx.num_threads() != ctx.index() {
+                            continue;
                         }
 
-                        let vectors = document
+                        let document = index
-                            .entry(RESERVED_VECTORS_FIELD_NAME)
+                            .document(&index_rtxn, docid)
-                            .or_insert(serde_json::Value::Object(Default::default()));
+                            .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
 
-                        let serde_json::Value::Object(vectors) = vectors else {
+                        let mut document = obkv_to_json(&all_fields, &fields_ids_map, document)
-                            return Err(err(milli::Error::UserError(
+                            .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
-                                milli::UserError::InvalidVectorsMapType {
-                                    document_id: {
-                                        if let Ok(Some(Ok(index))) = ctx
-                                            .index
-                                            .external_id_of(&index_rtxn, std::iter::once(docid))
-                                            .map(|it| it.into_iter().next())
-                                        {
-                                            index
-                                        } else {
-                                            format!("internal docid={docid}")
-                                        }
-                                    },
-                                    value: vectors.clone(),
-                                },
-                            )));
-                        };
 
-                        for (
+                        // TODO definitely factorize this code
-                            embedder_name,
+                        'inject_vectors: {
-                            EmbeddingsWithMetadata { embeddings, regenerate, has_fragments },
+                            let embeddings = index
-                        ) in embeddings
+                                .embeddings(&index_rtxn, docid)
-                        {
+                                .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
-                            let embeddings = ExplicitVectors {
+
-                                embeddings: Some(VectorOrArrayOfVectors::from_array_of_vectors(
+                            if embeddings.is_empty() {
-                                    embeddings,
+                                break 'inject_vectors;
-                                )),
+                            }
-                                regenerate: regenerate &&
+
-                                // Meilisearch does not handle well dumps with fragments, because as the fragments
+                            let vectors = document
-                                // are marked as user-provided,
+                                .entry(RESERVED_VECTORS_FIELD_NAME)
-                                // all embeddings would be regenerated on any settings change or document update.
+                                .or_insert(serde_json::Value::Object(Default::default()));
-                                // To prevent this, we mark embeddings has non regenerate in this case.
+
-                                !has_fragments,
+                            let serde_json::Value::Object(vectors) = vectors else {
+                                return Err(Error::from_milli(
+                                    milli::Error::UserError(
+                                        milli::UserError::InvalidVectorsMapType {
+                                            document_id: {
+                                                if let Ok(Some(Ok(index))) = index
+                                                    .external_id_of(
+                                                        &index_rtxn,
+                                                        std::iter::once(docid),
+                                                    )
+                                                    .map(|it| it.into_iter().next())
+                                                {
+                                                    index
+                                                } else {
+                                                    format!("internal docid={docid}")
+                                                }
+                                            },
+                                            value: vectors.clone(),
+                                        },
+                                    ),
+                                    Some(uid.to_string()),
+                                ));
                             };
-                            vectors
+
-                                .insert(embedder_name, serde_json::to_value(embeddings).unwrap());
+                            for (
+                                embedder_name,
+                                EmbeddingsWithMetadata { embeddings, regenerate, has_fragments },
+                            ) in embeddings
+                            {
+                                let embeddings = ExplicitVectors {
+                                    embeddings: Some(
+                                        VectorOrArrayOfVectors::from_array_of_vectors(embeddings),
+                                    ),
+                                    regenerate: regenerate &&
+                                    // Meilisearch does not handle well dumps with fragments, because as the fragments
+                                    // are marked as user-provided,
+                                    // all embeddings would be regenerated on any settings change or document update.
+                                    // To prevent this, we mark embeddings has non regenerate in this case.
+                                    !has_fragments,
+                                };
+                                vectors.insert(
+                                    embedder_name,
+                                    serde_json::to_value(embeddings).unwrap(),
+                                );
+                            }
+                        }
+
+                        tmp_buffer.clear();
+                        serde_json::to_writer(&mut tmp_buffer, &document)
+                            .map_err(milli::InternalError::from)
+                            .map_err(|e| Error::from_milli(e.into(), Some(uid.to_string())))?;
+
+                        // Make sure we put at least one document in the buffer even
+                        // though we might go above the buffer limit before sending
+                        if !buffer.is_empty() && buffer.len() + tmp_buffer.len() > limit {
+                            // We compress the documents before sending them
+                            let mut encoder =
+                                GzEncoder::new(&mut compressed_buffer, Compression::default());
+                            encoder
+                                .write_all(&buffer)
+                                .map_err(|e| Error::from_milli(e.into(), Some(uid.clone())))?;
+                            encoder
+                                .finish()
+                                .map_err(|e| Error::from_milli(e.into(), Some(uid.clone())))?;
+
+                            retry(&must_stop_processing, || {
+                                let mut request = agent.post(&documents_url);
+                                request = request.set("Content-Type", "application/x-ndjson");
+                                request = request.set("Content-Encoding", "gzip");
+                                if let Some(bearer) = &bearer {
+                                    request = request.set("Authorization", bearer);
+                                }
+                                request.send_bytes(&compressed_buffer).map_err(into_backoff_error)
+                            })?;
+                            buffer.clear();
+                            compressed_buffer.clear();
+                        }
+                        buffer.extend_from_slice(&tmp_buffer);
+
+                        if i > 0 && i % 100 == 0 {
+                            step.fetch_add(100, atomic::Ordering::Relaxed);
                         }
                     }
 
-                    tmp_buffer.clear();
+                    retry(&must_stop_processing, || {
-                    serde_json::to_writer(&mut tmp_buffer, &document)
+                        let mut request = agent.post(&documents_url);
-                        .map_err(milli::InternalError::from)
+                        request = request.set("Content-Type", "application/x-ndjson");
-                        .map_err(milli::Error::from)
+                        if let Some(bearer) = &bearer {
-                        .map_err(err)?;
+                            request = request.set("Authorization", bearer);
-
-                    // Make sure we put at least one document in the buffer even
-                    // though we might go above the buffer limit before sending
-                    if !buffer.is_empty() && buffer.len() + tmp_buffer.len() > limit {
-                        let control_flow = send_buffer(
-                            &buffer,
-                            &mut compressed_buffer,
-                            ctx.must_stop_processing,
-                            ctx.agent,
-                            &documents_url,
-                            target.remote_name,
-                            bearer.as_deref(),
-                            task_network.as_ref(),
-                            &err,
-                        )?;
-                        buffer.clear();
-                        compressed_buffer.clear();
-                        if let Some((import_data, _, metadata)) = &mut task_network {
-                            import_data.document_count = 0;
-                            metadata.task_key = None;
                         }
-                        if control_flow.is_break() {
+                        request.send_bytes(&buffer).map_err(into_backoff_error)
-                            return Ok(());
+                    })?;
-                        }
-                    }
-                    buffer.extend_from_slice(&tmp_buffer);
 
-                    if i > 0 && i % 100 == 0 {
+                    Ok(())
-                        step.fetch_add(100, atomic::Ordering::Relaxed);
+                })
-                    }
+                .map_err(|e| {
-                }
+                    Error::from_milli(
+                        milli::Error::InternalError(InternalError::PanicInThreadPool(e)),
+                        Some(uid.to_string()),
+                    )
+                })?;
+            for result in results {
+                result?;
+            }
 
-                // send the last buffered documents if any
+            step.store(total_documents, atomic::Ordering::Relaxed);
-                if !buffer.is_empty() {
-                    // ignore control flow here
-                    let _ = send_buffer(
-                        &buffer,
-                        &mut compressed_buffer,
-                        ctx.must_stop_processing,
-                        ctx.agent,
-                        &documents_url,
-                        target.remote_name,
-                        bearer.as_deref(),
-                        task_network.as_ref(),
-                        &err,
-                    )?;
-                }
-
-                Ok(())
-            })
-            .map_err(|e| err(milli::Error::InternalError(InternalError::PanicInThreadPool(e))))?;
-        for result in results {
-            result?;
-        }
-        step.store(total_documents, atomic::Ordering::Relaxed);
-        Ok(total_documents as u64)
-    }
-
-    #[cfg(feature = "enterprise")] // only used in enterprise edition for now
-    pub(super) fn export_no_index(
-        &self,
-        target: TargetInstance<'_>,
-        export_old_remote_name: &str,
-        network_change_origin: &Origin,
-        agent: &ureq::Agent,
-        must_stop_processing: &MustStopProcessing,
-    ) -> Result<(), Error> {
-        let bearer = target.api_key.map(|api_key| format!("Bearer {api_key}"));
-        let url = format!("{base_url}/network", base_url = target.base_url,);
-
-        {
-            let _ = handle_response(
-                target.remote_name,
-                retry(must_stop_processing, || {
-                    let request = agent.patch(&url);
-                    let mut request = set_network_ureq_headers(
-                        request,
-                        &ImportData {
-                            remote_name: export_old_remote_name.to_string(),
-                            index_name: None,
-                            document_count: 0,
-                        },
-                        network_change_origin,
-                        &ImportMetadata {
-                            index_count: 0,
-                            task_key: None,
-                            total_index_documents: 0,
-                        },
-                    );
-                    request = request.set("Content-Type", "application/json");
-                    if let Some(bearer) = &bearer {
-                        request = request.set("Authorization", bearer);
-                    }
-                    request
-                        .send_json(
-                            // empty payload that will be disregarded
-                            serde_json::Value::Object(Default::default()),
-                        )
-                        .map_err(into_backoff_error)
-                }),
-            )?;
         }
 
-        Ok(())
+        Ok(output)
-    }
-}
-
-fn set_network_ureq_headers(
-    request: ureq::Request,
-    import_data: &ImportData,
-    origin: &Origin,
-    metadata: &ImportMetadata,
-) -> ureq::Request {
-    let request = RequestWrapper(request);
-
-    let ImportMetadata { index_count, task_key, total_index_documents } = metadata;
-    let Origin { remote_name: origin_remote, task_uid, network_version } = origin;
-    let ImportData { remote_name: import_remote, index_name, document_count } = import_data;
-
-    let request = request
-        .set_origin_remote(origin_remote)
-        .set_origin_task_uid(*task_uid)
-        .set_origin_network_version(*network_version)
-        .set_import_remote(import_remote)
-        .set_import_docs(*document_count)
-        .set_import_index_count(*index_count)
-        .set_import_index_docs(*total_index_documents);
-
-    let request = if let Some(index_name) = index_name.as_deref() {
-        request.set_import_index(index_name)
-    } else {
-        request
-    };
-    let RequestWrapper(request) = if let Some(task_key) = task_key {
-        request.set_import_task_key(*task_key)
-    } else {
-        request
-    };
-
-    request
-}
-
-struct RequestWrapper(ureq::Request);
-impl headers::SetHeader for RequestWrapper {
-    fn set_header(self, name: &str, value: &str) -> Self {
-        Self(self.0.set(name, value))
-    }
-}
-
-#[allow(clippy::too_many_arguments)]
-fn send_buffer<'a>(
-    buffer: &'a [u8],
-    mut compressed_buffer: &'a mut Vec<u8>,
-    must_stop_processing: &MustStopProcessing,
-    agent: &ureq::Agent,
-    documents_url: &'a str,
-    remote_name: Option<&str>,
-    bearer: Option<&'a str>,
-    task_network: Option<&(ImportData, Origin, ImportMetadata)>,
-    err: &'a impl Fn(milli::Error) -> crate::Error,
-) -> Result<ControlFlow<(), ()>> {
-    // We compress the documents before sending them
-    let mut encoder: GzEncoder<&mut &mut Vec<u8>> =
-        GzEncoder::new(&mut compressed_buffer, Compression::default());
-    encoder.write_all(buffer).map_err(milli::Error::from).map_err(err)?;
-    encoder.finish().map_err(milli::Error::from).map_err(err)?;
-
-    let res = retry(must_stop_processing, || {
-        let mut request = agent.post(documents_url);
-        request = request.set("Content-Type", "application/x-ndjson");
-        request = request.set("Content-Encoding", "gzip");
-        if let Some(bearer) = bearer {
-            request = request.set("Authorization", bearer);
-        }
-        if let Some((import_data, origin, metadata)) = task_network {
-            request = set_network_ureq_headers(request, import_data, origin, metadata);
-        }
-        request.send_bytes(compressed_buffer).map_err(into_backoff_error)
-    });
-
-    handle_response(remote_name, res)
-}
-
-fn handle_response(remote_name: Option<&str>, res: Result<Response>) -> Result<ControlFlow<()>> {
-    let remote_name = remote_name.unwrap_or("unnamed");
-    match res {
-        Ok(_response) => Ok(ControlFlow::Continue(())),
-        Err(Error::FromRemoteWhenExporting { code, .. })
-            if code == Code::ImportTaskAlreadyReceived.name() =>
-        {
-            Ok(ControlFlow::Continue(()))
-        }
-        Err(Error::FromRemoteWhenExporting { code, message, .. })
-            if code == Code::ImportTaskUnknownRemote.name() =>
-        {
-            tracing::warn!("remote `{remote_name}` answered with: {message}");
-            Ok(ControlFlow::Break(()))
-        }
-        // note: there has already been many attempts to get this due to exponential backoff
-        Err(Error::FromRemoteWhenExporting { code, message, .. })
-            if code == Code::ImportTaskWithoutNetworkTask.name() =>
-        {
-            tracing::warn!("remote `{remote_name}` answered with: {message}");
-            Ok(ControlFlow::Break(()))
-        }
-        Err(e) => {
-            tracing::warn!("error while exporting: {e}");
-            Err(e)
-        }
     }
 }
 
@@ -593,65 +374,4 @@ fn ureq_error_into_error(error: ureq::Error) -> Error {
     }
 }
 
-// export_one_index arguments
-pub(super) struct TargetInstance<'a> {
-    pub(super) remote_name: Option<&'a str>,
-    pub(super) base_url: &'a str,
-    pub(super) api_key: Option<&'a str>,
-}
-
-pub(super) struct ExportOptions<'a> {
-    pub(super) index_uid: &'a str,
-    pub(super) payload_size: Option<&'a Byte>,
-    pub(super) override_settings: bool,
-    pub(super) export_mode: ExportMode<'a>,
-}
-
-impl ExportOptions<'_> {
-    fn task_network(
-        &self,
-        total_index_documents: u64,
-    ) -> Option<(ImportData, Origin, ImportMetadata)> {
-        if let ExportMode::NetworkBalancing {
-            index_count,
-            export_old_remote_name,
-            network_change_origin,
-        } = self.export_mode
-        {
-            Some((
-                ImportData {
-                    remote_name: export_old_remote_name.to_string(),
-                    index_name: Some(self.index_uid.to_string()),
-                    document_count: 0,
-                },
-                network_change_origin.clone(),
-                ImportMetadata { index_count, task_key: None, total_index_documents },
-            ))
-        } else {
-            None
-        }
-    }
-}
-
-pub(super) struct ExportContext<'a> {
-    pub(super) index: &'a meilisearch_types::milli::Index,
-    pub(super) index_rtxn: &'a milli::heed::RoTxn<'a>,
-    pub(super) universe: &'a RoaringBitmap,
-    pub(super) progress: &'a Progress,
-    pub(super) agent: &'a ureq::Agent,
-    pub(super) must_stop_processing: &'a MustStopProcessing,
-}
-
-pub(super) enum ExportMode<'a> {
-    ExportRoute,
-    #[cfg_attr(not(feature = "enterprise"), allow(dead_code))]
-    NetworkBalancing {
-        index_count: u64,
-
-        export_old_remote_name: &'a str,
-        network_change_origin: &'a Origin,
-    },
-}
-
-// progress related
 enum ExportIndex {}

crates/index-scheduler/src/scheduler/process_index_operation.rs

@@ -8,7 +8,6 @@ use meilisearch_types::milli::progress::{EmbedderStats, Progress};
 use meilisearch_types::milli::update::new::indexer::{self, UpdateByFunction};
 use meilisearch_types::milli::update::DocumentAdditionResult;
 use meilisearch_types::milli::{self, ChannelCongestion, Filter};
-use meilisearch_types::network::Network;
 use meilisearch_types::settings::apply_settings_to_builder;
 use meilisearch_types::tasks::{Details, KindWithContent, Status, Task};
 use meilisearch_types::Index;
@@ -37,7 +36,6 @@ impl IndexScheduler {
         operation: IndexOperation,
         progress: &Progress,
         embedder_stats: Arc<EmbedderStats>,
-        network: &Network,
     ) -> Result<(Vec<Task>, Option<ChannelCongestion>)> {
         let indexer_alloc = Bump::new();
         let started_processing_at = std::time::Instant::now();
@@ -69,6 +67,8 @@ impl IndexScheduler {
             IndexOperation::DocumentOperation { index_uid, primary_key, operations, mut tasks } => {
                 progress.update_progress(DocumentOperationProgress::RetrievingConfig);
 
+                let network = self.network();
+
                 let shards = network.shards();
 
                 // TODO: at some point, for better efficiency we might want to reuse the bumpalo for successive batches.
@@ -504,7 +504,6 @@ impl IndexScheduler {
                     },
                     progress,
                     embedder_stats.clone(),
-                    network,
                 )?;
 
                 let (settings_tasks, _congestion) = self.apply_index_operation(
@@ -513,7 +512,6 @@ impl IndexScheduler {
                     IndexOperation::Settings { index_uid, settings, tasks: settings_tasks },
                     progress,
                     embedder_stats,
-                    network,
                 )?;
 
                 let mut tasks = settings_tasks;

crates/index-scheduler/src/scheduler/process_snapshot_creation.rs

@@ -12,8 +12,6 @@ use crate::processing::{AtomicUpdateFileStep, SnapshotCreationProgress};
 use crate::queue::TaskQueue;
 use crate::{Error, IndexScheduler, Result};
 
-const UPDATE_FILES_DIR_NAME: &str = "update_files";
-
 /// # Safety
 ///
 /// See [`EnvOpenOptions::open`].
@@ -80,32 +78,10 @@ impl IndexScheduler {
     pub(super) fn process_snapshot(
         &self,
         progress: Progress,
-        tasks: Vec<Task>,
+        mut tasks: Vec<Task>,
     ) -> Result<Vec<Task>> {
         progress.update_progress(SnapshotCreationProgress::StartTheSnapshotCreation);
 
-        match self.scheduler.s3_snapshot_options.clone() {
-            Some(options) => {
-                #[cfg(not(unix))]
-                {
-                    let _ = options;
-                    panic!("Non-unix platform does not support S3 snapshotting");
-                }
-                #[cfg(unix)]
-                self.runtime
-                    .as_ref()
-                    .expect("Runtime not initialized")
-                    .block_on(self.process_snapshot_to_s3(progress, options, tasks))
-            }
-            None => self.process_snapshots_to_disk(progress, tasks),
-        }
-    }
-
-    fn process_snapshots_to_disk(
-        &self,
-        progress: Progress,
-        mut tasks: Vec<Task>,
-    ) -> Result<Vec<Task>, Error> {
         fs::create_dir_all(&self.scheduler.snapshots_path)?;
         let temp_snapshot_dir = tempfile::tempdir()?;
 
@@ -152,7 +128,7 @@ impl IndexScheduler {
         let rtxn = self.env.read_txn()?;
 
         // 2.4 Create the update files directory
-        let update_files_dir = temp_snapshot_dir.path().join(UPDATE_FILES_DIR_NAME);
+        let update_files_dir = temp_snapshot_dir.path().join("update_files");
         fs::create_dir_all(&update_files_dir)?;
 
         // 2.5 Only copy the update files of the enqueued tasks
@@ -164,7 +140,7 @@ impl IndexScheduler {
             let task =
                 self.queue.tasks.get_task(&rtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?;
             if let Some(content_uuid) = task.content_uuid() {
-                let src = self.queue.file_store.update_path(content_uuid);
+                let src = self.queue.file_store.get_update_path(content_uuid);
                 let dst = update_files_dir.join(content_uuid.to_string());
                 fs::copy(src, dst)?;
             }
@@ -230,407 +206,4 @@ impl IndexScheduler {
 
         Ok(tasks)
     }
-
-    #[cfg(unix)]
-    pub(super) async fn process_snapshot_to_s3(
-        &self,
-        progress: Progress,
-        opts: meilisearch_types::milli::update::S3SnapshotOptions,
-        mut tasks: Vec<Task>,
-    ) -> Result<Vec<Task>> {
-        use meilisearch_types::milli::update::S3SnapshotOptions;
-
-        let S3SnapshotOptions {
-            s3_bucket_url,
-            s3_bucket_region,
-            s3_bucket_name,
-            s3_snapshot_prefix,
-            s3_access_key,
-            s3_secret_key,
-            s3_max_in_flight_parts,
-            s3_compression_level: level,
-            s3_signature_duration,
-            s3_multipart_part_size,
-        } = opts;
-
-        let must_stop_processing = self.scheduler.must_stop_processing.clone();
-        let retry_backoff = backoff::ExponentialBackoff::default();
-        let db_name = {
-            let mut base_path = self.env.path().to_owned();
-            base_path.pop();
-            base_path.file_name().and_then(OsStr::to_str).unwrap_or("data.ms").to_string()
-        };
-
-        let (reader, writer) = std::io::pipe()?;
-        let uploader_task = tokio::spawn(multipart_stream_to_s3(
-            s3_bucket_url,
-            s3_bucket_region,
-            s3_bucket_name,
-            s3_snapshot_prefix,
-            s3_access_key,
-            s3_secret_key,
-            s3_max_in_flight_parts,
-            s3_signature_duration,
-            s3_multipart_part_size,
-            must_stop_processing,
-            retry_backoff,
-            db_name,
-            reader,
-        ));
-
-        let index_scheduler = IndexScheduler::private_clone(self);
-        let builder_task = tokio::task::spawn_blocking(move || {
-            stream_tarball_into_pipe(progress, level, writer, index_scheduler)
-        });
-
-        let (uploader_result, builder_result) = tokio::join!(uploader_task, builder_task);
-
-        // Check uploader result first to early return on task abortion.
-        // safety: JoinHandle can return an error if the task was aborted, cancelled, or panicked.
-        uploader_result.unwrap()?;
-        builder_result.unwrap()?;
-
-        for task in &mut tasks {
-            task.status = Status::Succeeded;
-        }
-
-        Ok(tasks)
-    }
-}
-
-/// Streams a tarball of the database content into a pipe.
-#[cfg(unix)]
-fn stream_tarball_into_pipe(
-    progress: Progress,
-    level: u32,
-    writer: std::io::PipeWriter,
-    index_scheduler: IndexScheduler,
-) -> std::result::Result<(), Error> {
-    use std::io::Write as _;
-    use std::path::Path;
-
-    let writer = flate2::write::GzEncoder::new(writer, flate2::Compression::new(level));
-    let mut tarball = tar::Builder::new(writer);
-
-    // 1. Snapshot the version file
-    tarball
-        .append_path_with_name(&index_scheduler.scheduler.version_file_path, VERSION_FILE_NAME)?;
-
-    // 2. Snapshot the index scheduler LMDB env
-    progress.update_progress(SnapshotCreationProgress::SnapshotTheIndexScheduler);
-    let tasks_env_file = index_scheduler.env.try_clone_inner_file()?;
-    let path = Path::new("tasks").join("data.mdb");
-    append_file_to_tarball(&mut tarball, path, tasks_env_file)?;
-
-    // 2.3 Create a read transaction on the index-scheduler
-    let rtxn = index_scheduler.env.read_txn()?;
-
-    // 2.4 Create the update files directory
-    //     And only copy the update files of the enqueued tasks
-    progress.update_progress(SnapshotCreationProgress::SnapshotTheUpdateFiles);
-    let enqueued = index_scheduler.queue.tasks.get_status(&rtxn, Status::Enqueued)?;
-    let (atomic, update_file_progress) = AtomicUpdateFileStep::new(enqueued.len() as u32);
-    progress.update_progress(update_file_progress);
-
-    // We create the update_files directory so that it
-    // always exists even if there are no update files
-    let update_files_dir = Path::new(UPDATE_FILES_DIR_NAME);
-    let src_update_files_dir = {
-        let mut path = index_scheduler.env.path().to_path_buf();
-        path.pop();
-        path.join(UPDATE_FILES_DIR_NAME)
-    };
-    tarball.append_dir(update_files_dir, src_update_files_dir)?;
-
-    for task_id in enqueued {
-        let task = index_scheduler
-            .queue
-            .tasks
-            .get_task(&rtxn, task_id)?
-            .ok_or(Error::CorruptedTaskQueue)?;
-        if let Some(content_uuid) = task.content_uuid() {
-            use std::fs::File;
-
-            let src = index_scheduler.queue.file_store.update_path(content_uuid);
-            let mut update_file = File::open(src)?;
-            let path = update_files_dir.join(content_uuid.to_string());
-            tarball.append_file(path, &mut update_file)?;
-        }
-        atomic.fetch_add(1, Ordering::Relaxed);
-    }
-
-    // 3. Snapshot every indexes
-    progress.update_progress(SnapshotCreationProgress::SnapshotTheIndexes);
-    let index_mapping = index_scheduler.index_mapper.index_mapping;
-    let nb_indexes = index_mapping.len(&rtxn)? as u32;
-    let indexes_dir = Path::new("indexes");
-    let indexes_references: Vec<_> = index_scheduler
-        .index_mapper
-        .index_mapping
-        .iter(&rtxn)?
-        .map(|res| res.map_err(Error::from).map(|(name, uuid)| (name.to_string(), uuid)))
-        .collect::<Result<_, Error>>()?;
-
-    // It's prettier to use a for loop instead of the IndexMapper::try_for_each_index
-    // method, especially when we need to access the UUID, local path and index number.
-    for (i, (name, uuid)) in indexes_references.into_iter().enumerate() {
-        progress.update_progress(VariableNameStep::<SnapshotCreationProgress>::new(
-            &name, i as u32, nb_indexes,
-        ));
-        let path = indexes_dir.join(uuid.to_string()).join("data.mdb");
-        let index = index_scheduler.index_mapper.index(&rtxn, &name)?;
-        let index_file = index.try_clone_inner_file()?;
-        tracing::trace!("Appending index file for {name} in {}", path.display());
-        append_file_to_tarball(&mut tarball, path, index_file)?;
-    }
-
-    drop(rtxn);
-
-    // 4. Snapshot the auth LMDB env
-    progress.update_progress(SnapshotCreationProgress::SnapshotTheApiKeys);
-    let auth_env_file = index_scheduler.scheduler.auth_env.try_clone_inner_file()?;
-    let path = Path::new("auth").join("data.mdb");
-    append_file_to_tarball(&mut tarball, path, auth_env_file)?;
-
-    let mut gzencoder = tarball.into_inner()?;
-    gzencoder.flush()?;
-    gzencoder.try_finish()?;
-    let mut writer = gzencoder.finish()?;
-    writer.flush()?;
-
-    Result::<_, Error>::Ok(())
-}
-
-#[cfg(unix)]
-fn append_file_to_tarball<W, P>(
-    tarball: &mut tar::Builder<W>,
-    path: P,
-    mut auth_env_file: fs::File,
-) -> Result<(), Error>
-where
-    W: std::io::Write,
-    P: AsRef<std::path::Path>,
-{
-    use std::io::{Seek as _, SeekFrom};
-
-    // Note: A previous snapshot operation may have left the cursor
-    //       at the end of the file so we need to seek to the start.
-    auth_env_file.seek(SeekFrom::Start(0))?;
-    tarball.append_file(path, &mut auth_env_file)?;
-    Ok(())
-}
-
-/// Streams the content read from the given reader to S3.
-#[cfg(unix)]
-#[allow(clippy::too_many_arguments)]
-async fn multipart_stream_to_s3(
-    s3_bucket_url: String,
-    s3_bucket_region: String,
-    s3_bucket_name: String,
-    s3_snapshot_prefix: String,
-    s3_access_key: String,
-    s3_secret_key: String,
-    s3_max_in_flight_parts: std::num::NonZero<usize>,
-    s3_signature_duration: std::time::Duration,
-    s3_multipart_part_size: u64,
-    must_stop_processing: super::MustStopProcessing,
-    retry_backoff: backoff::exponential::ExponentialBackoff<backoff::SystemClock>,
-    db_name: String,
-    reader: std::io::PipeReader,
-) -> Result<(), Error> {
-    use std::collections::VecDeque;
-    use std::io;
-    use std::os::fd::OwnedFd;
-    use std::path::PathBuf;
-
-    use bytes::{Bytes, BytesMut};
-    use reqwest::{Client, Response};
-    use rusty_s3::actions::CreateMultipartUpload;
-    use rusty_s3::{Bucket, BucketError, Credentials, S3Action as _, UrlStyle};
-    use tokio::task::JoinHandle;
-
-    let reader = OwnedFd::from(reader);
-    let reader = tokio::net::unix::pipe::Receiver::from_owned_fd(reader)?;
-    let s3_snapshot_prefix = PathBuf::from(s3_snapshot_prefix);
-    let url =
-        s3_bucket_url.parse().map_err(BucketError::ParseError).map_err(Error::S3BucketError)?;
-    let bucket = Bucket::new(url, UrlStyle::Path, s3_bucket_name, s3_bucket_region)
-        .map_err(Error::S3BucketError)?;
-    let credential = Credentials::new(s3_access_key, s3_secret_key);
-
-    // Note for the future (rust 1.91+): use with_added_extension, it's prettier
-    let object_path = s3_snapshot_prefix.join(format!("{db_name}.snapshot"));
-    // Note: It doesn't work on Windows and if a port to this platform is needed,
-    //       use the slash-path crate or similar to get the correct path separator.
-    let object = object_path.display().to_string();
-
-    let action = bucket.create_multipart_upload(Some(&credential), &object);
-    let url = action.sign(s3_signature_duration);
-
-    let client = Client::new();
-    let resp = client.post(url).send().await.map_err(Error::S3HttpError)?;
-    let status = resp.status();
-
-    let body = match resp.error_for_status_ref() {
-        Ok(_) => resp.text().await.map_err(Error::S3HttpError)?,
-        Err(_) => {
-            return Err(Error::S3Error { status, body: resp.text().await.unwrap_or_default() })
-        }
-    };
-
-    let multipart =
-        CreateMultipartUpload::parse_response(&body).map_err(|e| Error::S3XmlError(Box::new(e)))?;
-    tracing::debug!("Starting the upload of the snapshot to {object}");
-
-    // We use this bumpalo for etags strings.
-    let bump = bumpalo::Bump::new();
-    let mut etags = Vec::<&str>::new();
-    let mut in_flight = VecDeque::<(JoinHandle<reqwest::Result<Response>>, Bytes)>::with_capacity(
-        s3_max_in_flight_parts.get(),
-    );
-
-    // Part numbers start at 1 and cannot be larger than 10k
-    for part_number in 1u16.. {
-        if must_stop_processing.get() {
-            return Err(Error::AbortedTask);
-        }
-
-        let part_upload =
-            bucket.upload_part(Some(&credential), &object, part_number, multipart.upload_id());
-        let url = part_upload.sign(s3_signature_duration);
-
-        // Wait for a buffer to be ready if there are in-flight parts that landed
-        let mut buffer = if in_flight.len() >= s3_max_in_flight_parts.get() {
-            let (handle, buffer) = in_flight.pop_front().expect("At least one in flight request");
-            let resp = join_and_map_error(handle).await?;
-            extract_and_append_etag(&bump, &mut etags, resp.headers())?;
-
-            let mut buffer = match buffer.try_into_mut() {
-                Ok(buffer) => buffer,
-                Err(_) => unreachable!("All bytes references were consumed in the task"),
-            };
-            buffer.clear();
-            buffer
-        } else {
-            BytesMut::with_capacity(s3_multipart_part_size as usize)
-        };
-
-        // If we successfully read enough bytes,
-        // we can continue and send the buffer/part
-        while buffer.len() < (s3_multipart_part_size as usize / 2) {
-            // Wait for the pipe to be readable
-
-            reader.readable().await?;
-
-            match reader.try_read_buf(&mut buffer) {
-                Ok(0) => break,
-                // We read some bytes but maybe not enough
-                Ok(_) => continue,
-                // The readiness event is a false positive.
-                Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => continue,
-                Err(e) => return Err(e.into()),
-            }
-        }
-
-        if buffer.is_empty() {
-            // Break the loop if the buffer is
-            // empty after we tried to read bytes
-            break;
-        }
-
-        let body = buffer.freeze();
-        tracing::trace!("Sending part {part_number}");
-        let task = tokio::spawn({
-            let client = client.clone();
-            let body = body.clone();
-            backoff::future::retry(retry_backoff.clone(), move || {
-                let client = client.clone();
-                let url = url.clone();
-                let body = body.clone();
-                async move {
-                    match client.put(url).body(body).send().await {
-                        Ok(resp) if resp.status().is_client_error() => {
-                            resp.error_for_status().map_err(backoff::Error::Permanent)
-                        }
-                        Ok(resp) => Ok(resp),
-                        Err(e) => Err(backoff::Error::transient(e)),
-                    }
-                }
-            })
-        });
-        in_flight.push_back((task, body));
-    }
-
-    for (handle, _buffer) in in_flight {
-        let resp = join_and_map_error(handle).await?;
-        extract_and_append_etag(&bump, &mut etags, resp.headers())?;
-    }
-
-    tracing::debug!("Finalizing the multipart upload");
-
-    let action = bucket.complete_multipart_upload(
-        Some(&credential),
-        &object,
-        multipart.upload_id(),
-        etags.iter().map(AsRef::as_ref),
-    );
-    let url = action.sign(s3_signature_duration);
-    let body = action.body();
-    let resp = backoff::future::retry(retry_backoff, move || {
-        let client = client.clone();
-        let url = url.clone();
-        let body = body.clone();
-        async move {
-            match client.post(url).body(body).send().await {
-                Ok(resp) if resp.status().is_client_error() => {
-                    Err(backoff::Error::Permanent(Error::S3Error {
-                        status: resp.status(),
-                        body: resp.text().await.unwrap_or_default(),
-                    }))
-                }
-                Ok(resp) => Ok(resp),
-                Err(e) => Err(backoff::Error::transient(Error::S3HttpError(e))),
-            }
-        }
-    })
-    .await?;
-
-    let status = resp.status();
-    let body = resp.text().await.map_err(|e| Error::S3Error { status, body: e.to_string() })?;
-    if status.is_success() {
-        Ok(())
-    } else {
-        Err(Error::S3Error { status, body })
-    }
-}
-
-#[cfg(unix)]
-async fn join_and_map_error(
-    join_handle: tokio::task::JoinHandle<Result<reqwest::Response, reqwest::Error>>,
-) -> Result<reqwest::Response> {
-    // safety: Panic happens if the task (JoinHandle) was aborted, cancelled, or panicked
-    let request = join_handle.await.unwrap();
-    let resp = request.map_err(Error::S3HttpError)?;
-    match resp.error_for_status_ref() {
-        Ok(_) => Ok(resp),
-        Err(_) => Err(Error::S3Error {
-            status: resp.status(),
-            body: resp.text().await.unwrap_or_default(),
-        }),
-    }
-}
-
-#[cfg(unix)]
-fn extract_and_append_etag<'b>(
-    bump: &'b bumpalo::Bump,
-    etags: &mut Vec<&'b str>,
-    headers: &reqwest::header::HeaderMap,
-) -> Result<()> {
-    use reqwest::header::ETAG;
-
-    let etag = headers.get(ETAG).ok_or_else(|| Error::S3XmlError("Missing ETag header".into()))?;
-    let etag = etag.to_str().map_err(|e| Error::S3XmlError(Box::new(e)))?;
-    etags.push(bump.alloc_str(etag));
-
-    Ok(())
 }

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: [current version] }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 22, 2) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
 3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
@@ -57,7 +57,7 @@ girafo: { number_of_documents: 0, field_distribution: {} }
 [timestamp] [4,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"[current version]"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.22.2"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", }
 2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", }
 3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", }

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: [current version] }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 22, 2) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,]

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: [current version] }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 22, 2) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 ----------------------------------------------------------------------
 ### Status:

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: [current version] }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 22, 2) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 ----------------------------------------------------------------------
 ### Status:
@@ -37,7 +37,7 @@ catto [1,]
 [timestamp] [0,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"[current version]"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.22.2"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: [current version] }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 22, 2) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
 ----------------------------------------------------------------------
@@ -40,7 +40,7 @@ doggo [2,]
 [timestamp] [0,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"[current version]"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.22.2"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: [current version] }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 22, 2) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
 3 {uid: 3, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
@@ -43,7 +43,7 @@ doggo [2,3,]
 [timestamp] [0,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"[current version]"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.22.2"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]

crates/index-scheduler/src/scheduler/test.rs

@@ -722,7 +722,7 @@ fn basic_get_stats() {
     let kind = index_creation_task("whalo", "fish");
     let _task = index_scheduler.register(kind, None, false).unwrap();
 
-    snapshot!(json_string!(index_scheduler.get_stats().unwrap()), @r###"
+    snapshot!(json_string!(index_scheduler.get_stats().unwrap()), @r#"
     {
       "indexes": {
         "catto": 1,
@@ -742,12 +742,10 @@ fn basic_get_stats() {
         "documentEdition": 0,
         "dumpCreation": 0,
         "export": 0,
-        "indexCompaction": 0,
         "indexCreation": 3,
         "indexDeletion": 0,
         "indexSwap": 0,
         "indexUpdate": 0,
-        "networkTopologyChange": 0,
         "settingsUpdate": 0,
         "snapshotCreation": 0,
         "taskCancelation": 0,
@@ -755,10 +753,10 @@ fn basic_get_stats() {
         "upgradeDatabase": 0
       }
     }
-    "###);
+    "#);
 
     handle.advance_till([Start, BatchCreated]);
-    snapshot!(json_string!(index_scheduler.get_stats().unwrap()), @r###"
+    snapshot!(json_string!(index_scheduler.get_stats().unwrap()), @r#"
     {
       "indexes": {
         "catto": 1,
@@ -778,12 +776,10 @@ fn basic_get_stats() {
         "documentEdition": 0,
         "dumpCreation": 0,
         "export": 0,
-        "indexCompaction": 0,
         "indexCreation": 3,
         "indexDeletion": 0,
         "indexSwap": 0,
         "indexUpdate": 0,
-        "networkTopologyChange": 0,
         "settingsUpdate": 0,
         "snapshotCreation": 0,
         "taskCancelation": 0,
@@ -791,7 +787,7 @@ fn basic_get_stats() {
         "upgradeDatabase": 0
       }
     }
-    "###);
+    "#);
 
     handle.advance_till([
         InsideProcessBatch,
@@ -801,7 +797,7 @@ fn basic_get_stats() {
         Start,
         BatchCreated,
     ]);
-    snapshot!(json_string!(index_scheduler.get_stats().unwrap()), @r###"
+    snapshot!(json_string!(index_scheduler.get_stats().unwrap()), @r#"
     {
       "indexes": {
         "catto": 1,
@@ -821,12 +817,10 @@ fn basic_get_stats() {
         "documentEdition": 0,
         "dumpCreation": 0,
         "export": 0,
-        "indexCompaction": 0,
         "indexCreation": 3,
         "indexDeletion": 0,
         "indexSwap": 0,
         "indexUpdate": 0,
-        "networkTopologyChange": 0,
         "settingsUpdate": 0,
         "snapshotCreation": 0,
         "taskCancelation": 0,
@@ -834,7 +828,7 @@ fn basic_get_stats() {
         "upgradeDatabase": 0
       }
     }
-    "###);
+    "#);
 
     // now we make one more batch, the started_at field of the new tasks will be past `second_start_time`
     handle.advance_till([
@@ -845,7 +839,7 @@ fn basic_get_stats() {
         Start,
         BatchCreated,
     ]);
-    snapshot!(json_string!(index_scheduler.get_stats().unwrap()), @r###"
+    snapshot!(json_string!(index_scheduler.get_stats().unwrap()), @r#"
     {
       "indexes": {
         "catto": 1,
@@ -865,12 +859,10 @@ fn basic_get_stats() {
         "documentEdition": 0,
         "dumpCreation": 0,
         "export": 0,
-        "indexCompaction": 0,
         "indexCreation": 3,
         "indexDeletion": 0,
         "indexSwap": 0,
         "indexUpdate": 0,
-        "networkTopologyChange": 0,
         "settingsUpdate": 0,
         "snapshotCreation": 0,
         "taskCancelation": 0,
@@ -878,7 +870,7 @@ fn basic_get_stats() {
         "upgradeDatabase": 0
       }
     }
-    "###);
+    "#);
 }
 
 #[test]

crates/index-scheduler/src/test_utils.rs

@@ -112,7 +112,6 @@ impl IndexScheduler {
             max_number_of_batched_tasks: usize::MAX,
             batched_tasks_size_limit: u64::MAX,
             instance_features: Default::default(),
-            export_default_payload_size_bytes: byte_unit::Byte::parse_str("20MiB", false).unwrap(),
             auto_upgrade: true, // Don't cost much and will ensure the happy path works
             embedding_cache_cap: 10,
             experimental_no_snapshot_compaction: false,
@@ -127,7 +126,7 @@ impl IndexScheduler {
         std::fs::create_dir_all(&options.auth_path).unwrap();
         let auth_env = open_auth_store_env(&options.auth_path).unwrap();
         let index_scheduler =
-            Self::new_test(options, auth_env, version, None, sender, planned_failures).unwrap();
+            Self::new(options, auth_env, version, sender, planned_failures).unwrap();
 
         // To be 100% consistent between all test we're going to start the scheduler right now
         // and ensure it's in the expected starting state.

crates/index-scheduler/src/upgrade/mod.rs

@@ -1,93 +1,85 @@
 use anyhow::bail;
 use meilisearch_types::heed::{Env, RwTxn, WithoutTls};
 use meilisearch_types::tasks::{Details, KindWithContent, Status, Task};
-use meilisearch_types::versioning;
+use meilisearch_types::versioning::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH};
 use time::OffsetDateTime;
 use tracing::info;
 
 use crate::queue::TaskQueue;
 use crate::versioning::Versioning;
 
-mod v1_29;
-mod v1_30;
 trait UpgradeIndexScheduler {
-    fn upgrade(&self, env: &Env<WithoutTls>, wtxn: &mut RwTxn) -> anyhow::Result<()>;
+    fn upgrade(
-    /// Whether the migration should be applied, depending on the initial version of the index scheduler before
+        &self,
-    /// any migration was applied
+        env: &Env<WithoutTls>,
-    fn must_upgrade(&self, initial_version: (u32, u32, u32)) -> bool;
+        wtxn: &mut RwTxn,
-    /// A progress-centric description of the migration
+        original: (u32, u32, u32),
-    fn description(&self) -> &'static str;
+    ) -> anyhow::Result<()>;
+    fn target_version(&self) -> (u32, u32, u32);
 }
 
-/// Upgrade the index scheduler to the binary version.
-///
-/// # Warning
-///
-/// The current implementation uses a single wtxn to the index scheduler for the whole duration of the upgrade.
-/// If migrations start taking take a long time, it might prevent tasks from being registered.
-/// If this issue manifests, then it can be mitigated by adding a `fn target_version` to `UpgradeIndexScheduler`,
-/// to be able to write intermediate versions and drop the wtxn between applying migrations.
 pub fn upgrade_index_scheduler(
     env: &Env<WithoutTls>,
     versioning: &Versioning,
-    initial_version: (u32, u32, u32),
+    from: (u32, u32, u32),
+    to: (u32, u32, u32),
 ) -> anyhow::Result<()> {
-    let target_major: u32 = versioning::VERSION_MAJOR;
+    let current_major = to.0;
-    let target_minor: u32 = versioning::VERSION_MINOR;
+    let current_minor = to.1;
-    let target_patch: u32 = versioning::VERSION_PATCH;
+    let current_patch = to.2;
-    let target_version = (target_major, target_minor, target_patch);
-
-    if initial_version == target_version {
-        return Ok(());
-    }
 
     let upgrade_functions: &[&dyn UpgradeIndexScheduler] = &[
-        // List all upgrade functions to apply in order here.
+        // This is the last upgrade function, it will be called when the index is up to date.
-        &v1_30::MigrateNetwork,
+        // any other upgrade function should be added before this one.
+        &ToCurrentNoOp {},
     ];
 
-    let (initial_major, initial_minor, initial_patch) = initial_version;
+    let start = match from {
-
+        (1, 12, _) => 0,
-    if initial_version > target_version {
+        (1, 13, _) => 0,
-        bail!(
+        (1, 14, _) => 0,
-                "Database version {initial_major}.{initial_minor}.{initial_patch} is higher than the Meilisearch version {target_major}.{target_minor}.{target_patch}. Downgrade is not supported",
+        (1, 15, _) => 0,
+        (1, 16, _) => 0,
+        (1, 17, _) => 0,
+        (1, 18, _) => 0,
+        (1, 19, _) => 0,
+        (1, 20, _) => 0,
+        (1, 21, _) => 0,
+        (1, 22, _) => 0,
+        (major, minor, patch) => {
+            if major > current_major
+                || (major == current_major && minor > current_minor)
+                || (major == current_major && minor == current_minor && patch > current_patch)
+            {
+                bail!(
+                "Database version {major}.{minor}.{patch} is higher than the Meilisearch version {current_major}.{current_minor}.{current_patch}. Downgrade is not supported",
+                );
+            } else if major < 1 || (major == current_major && minor < 12) {
+                bail!(
+                "Database version {major}.{minor}.{patch} is too old for the experimental dumpless upgrade feature. Please generate a dump using the v{major}.{minor}.{patch} and import it in the v{current_major}.{current_minor}.{current_patch}",
             );
-    }
+            } else {
-
+                bail!("Unknown database version: v{major}.{minor}.{patch}");
-    if initial_version < (1, 12, 0) {
+            }
-        bail!(
+        }
-                "Database version {initial_major}.{initial_minor}.{initial_patch} is too old for the experimental dumpless upgrade feature. Please generate a dump using the v{initial_major}.{initial_minor}.{initial_patch} and import it in the v{target_major}.{target_minor}.{target_patch}",
+    };
-            );
-    }
 
     info!("Upgrading the task queue");
-    let mut wtxn = env.write_txn()?;
+    let mut local_from = from;
-    let migration_count = upgrade_functions.len();
+    for upgrade in upgrade_functions[start..].iter() {
-    for (migration_index, upgrade) in upgrade_functions.iter().enumerate() {
+        let target = upgrade.target_version();
-        if upgrade.must_upgrade(initial_version) {
+        info!(
-            info!(
+            "Upgrading from v{}.{}.{} to v{}.{}.{}",
-                "[{migration_index}/{migration_count}]Applying migration: {}",
+            local_from.0, local_from.1, local_from.2, target.0, target.1, target.2
-                upgrade.description()
+        );
-            );
+        let mut wtxn = env.write_txn()?;
-
+        upgrade.upgrade(env, &mut wtxn, local_from)?;
-            upgrade.upgrade(env, &mut wtxn)?;
+        versioning.set_version(&mut wtxn, target)?;
-
+        wtxn.commit()?;
-            info!(
+        local_from = target;
-                "[{}/{migration_count}]Migration applied: {}",
-                migration_index + 1,
-                upgrade.description()
-            )
-        } else {
-            info!(
-                "[{migration_index}/{migration_count}]Skipping unnecessary migration: {}",
-                upgrade.description()
-            )
-        }
     }
 
-    versioning.set_version(&mut wtxn, target_version)?;
+    let mut wtxn = env.write_txn()?;
-    info!("Task queue upgraded, spawning the upgrade database task");
-
     let queue = TaskQueue::new(env, &mut wtxn)?;
     let uid = queue.next_task_id(&wtxn)?;
     queue.register(
@@ -100,14 +92,31 @@ pub fn upgrade_index_scheduler(
             finished_at: None,
             error: None,
             canceled_by: None,
-            details: Some(Details::UpgradeDatabase { from: initial_version, to: target_version }),
+            details: Some(Details::UpgradeDatabase { from, to }),
             status: Status::Enqueued,
-            kind: KindWithContent::UpgradeDatabase { from: initial_version },
+            kind: KindWithContent::UpgradeDatabase { from },
             network: None,
-            custom_metadata: None,
         },
     )?;
     wtxn.commit()?;
 
     Ok(())
 }
+
+#[allow(non_camel_case_types)]
+struct ToCurrentNoOp {}
+
+impl UpgradeIndexScheduler for ToCurrentNoOp {
+    fn upgrade(
+        &self,
+        _env: &Env<WithoutTls>,
+        _wtxn: &mut RwTxn,
+        _original: (u32, u32, u32),
+    ) -> anyhow::Result<()> {
+        Ok(())
+    }
+
+    fn target_version(&self) -> (u32, u32, u32) {
+        (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH)
+    }
+}

crates/index-scheduler/src/upgrade/v1_29.rs

@@ -1,47 +0,0 @@
-use std::collections::BTreeMap;
-
-use meilisearch_types::heed::types::{SerdeJson, Str};
-use meilisearch_types::heed::{Env, RoTxn, WithoutTls};
-use serde::{Deserialize, Serialize};
-
-use crate::Result;
-
-/// Database const names for the `FeatureData`.
-mod db_name {
-    pub const EXPERIMENTAL_FEATURES: &str = "experimental-features";
-}
-
-mod db_keys {
-    pub const NETWORK: &str = "network";
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)]
-#[serde(rename_all = "camelCase")]
-pub struct Network {
-    #[serde(default, rename = "self")]
-    pub local: Option<String>,
-    #[serde(default)]
-    pub remotes: BTreeMap<String, Remote>,
-    #[serde(default)]
-    pub sharding: bool,
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
-#[serde(rename_all = "camelCase")]
-pub struct Remote {
-    pub url: String,
-    #[serde(default)]
-    pub search_api_key: Option<String>,
-    #[serde(default)]
-    pub write_api_key: Option<String>,
-}
-
-pub fn get_network(env: &Env<WithoutTls>, rtxn: &RoTxn) -> Result<Option<Network>> {
-    let Some(network_db) =
-        env.open_database::<Str, SerdeJson<Network>>(rtxn, Some(db_name::EXPERIMENTAL_FEATURES))?
-    else {
-        return Ok(None);
-    };
-
-    Ok(network_db.get(rtxn, db_keys::NETWORK)?)
-}

crates/index-scheduler/src/upgrade/v1_30.rs

@@ -1,82 +0,0 @@
-use std::collections::BTreeMap;
-
-use meilisearch_types::heed::types::{SerdeJson, Str};
-use meilisearch_types::heed::{Env, RwTxn, WithoutTls};
-use serde::{Deserialize, Serialize};
-use uuid::Uuid;
-
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)]
-#[serde(rename_all = "camelCase")]
-pub struct Network {
-    #[serde(default, rename = "self")]
-    pub local: Option<String>,
-    #[serde(default)]
-    pub remotes: BTreeMap<String, Remote>,
-    #[serde(default)]
-    pub leader: Option<String>,
-    #[serde(default)]
-    pub version: Uuid,
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
-#[serde(rename_all = "camelCase")]
-pub struct Remote {
-    pub url: String,
-    #[serde(default)]
-    pub search_api_key: Option<String>,
-    #[serde(default)]
-    pub write_api_key: Option<String>,
-}
-
-use super::v1_29;
-use crate::Result;
-
-/// Database const names for the `FeatureData`.
-mod db_name {
-    pub const EXPERIMENTAL_FEATURES: &str = "experimental-features";
-}
-
-mod db_keys {
-    pub const NETWORK: &str = "network";
-}
-
-pub struct MigrateNetwork;
-
-impl super::UpgradeIndexScheduler for MigrateNetwork {
-    fn upgrade(&self, env: &Env<WithoutTls>, wtxn: &mut RwTxn) -> anyhow::Result<()> {
-        let Some(v1_29::Network { local, remotes, sharding }) = v1_29::get_network(env, wtxn)?
-        else {
-            return Ok(());
-        };
-
-        let leader = if sharding { remotes.keys().next().cloned() } else { None };
-
-        let remotes = remotes
-            .into_iter()
-            .map(|(name, v1_29::Remote { url, search_api_key, write_api_key })| {
-                (name, Remote { url, search_api_key, write_api_key })
-            })
-            .collect();
-
-        let network = Network { local, remotes, leader, version: Uuid::nil() };
-
-        set_network(env, wtxn, &network)?;
-        Ok(())
-    }
-
-    fn must_upgrade(&self, initial_version: (u32, u32, u32)) -> bool {
-        initial_version < (1, 30, 0)
-    }
-
-    fn description(&self) -> &'static str {
-        "updating the network struct"
-    }
-}
-
-fn set_network(env: &Env<WithoutTls>, wtxn: &mut RwTxn<'_>, network: &Network) -> Result<()> {
-    let network_db =
-        env.create_database::<Str, SerdeJson<Network>>(wtxn, Some(db_name::EXPERIMENTAL_FEATURES))?;
-
-    network_db.put(wtxn, db_keys::NETWORK, network)?;
-    Ok(())
-}

crates/index-scheduler/src/utils.rs

@@ -4,11 +4,9 @@ use std::collections::{BTreeSet, HashSet};
 use std::ops::Bound;
 use std::sync::Arc;
 
-use convert_case::{Case, Casing as _};
 use meilisearch_types::batches::{Batch, BatchEnqueuedAt, BatchId, BatchStats};
 use meilisearch_types::heed::{Database, RoTxn, RwTxn};
-use meilisearch_types::milli::progress::Progress;
+use meilisearch_types::milli::CboRoaringBitmapCodec;
-use meilisearch_types::milli::{CboRoaringBitmapCodec, ChannelCongestion};
 use meilisearch_types::task_view::DetailsView;
 use meilisearch_types::tasks::{
     BatchStopReason, Details, IndexSwap, Kind, KindWithContent, Status,
@@ -121,8 +119,17 @@ impl ProcessingBatch {
         self.stats.total_nb_tasks = 0;
     }
 
-    /// Update batch task from a processed task
+    /// Update the timestamp of the tasks and the inner structure of this structure.
-    pub fn update_from_task(&mut self, task: &Task) {
+    pub fn update(&mut self, task: &mut Task) {
+        // We must re-set this value in case we're dealing with a task that has been added between
+        // the `processing` and `finished` state
+        // We must re-set this value in case we're dealing with a task that has been added between
+        // the `processing` and `finished` state or that failed.
+        task.batch_uid = Some(self.uid);
+        // Same
+        task.started_at = Some(self.started_at);
+        task.finished_at = self.finished_at;
+
         self.statuses.insert(task.status);
 
         // Craft an aggregation of the details of all the tasks encountered in this batch.
@@ -137,63 +144,6 @@ impl ProcessingBatch {
         }
     }
 
-    /// Update the timestamp of the tasks after they're done
-    pub fn finish_task(&self, task: &mut Task) {
-        // We must re-set this value in case we're dealing with a task that has been added between
-        // the `processing` and `finished` state or that failed.
-        task.batch_uid = Some(self.uid);
-        // Same
-        task.started_at = Some(self.started_at);
-        task.finished_at = self.finished_at;
-    }
-
-    pub fn write_stats(
-        &mut self,
-        progress: &Progress,
-        congestion: Option<ChannelCongestion>,
-        pre_commit_dabases_sizes: indexmap::IndexMap<&'static str, usize>,
-        post_commit_dabases_sizes: indexmap::IndexMap<&'static str, usize>,
-    ) {
-        self.stats.progress_trace =
-            progress.accumulated_durations().into_iter().map(|(k, v)| (k, v.into())).collect();
-        self.stats.write_channel_congestion = congestion.map(|congestion| {
-            let mut congestion_info = serde_json::Map::new();
-            congestion_info.insert("attempts".into(), congestion.attempts.into());
-            congestion_info.insert("blocking_attempts".into(), congestion.blocking_attempts.into());
-            congestion_info.insert("blocking_ratio".into(), congestion.congestion_ratio().into());
-            congestion_info
-        });
-        self.stats.internal_database_sizes = pre_commit_dabases_sizes
-            .iter()
-            .flat_map(|(dbname, pre_size)| {
-                post_commit_dabases_sizes
-                    .get(dbname)
-                    .map(|post_size| {
-                        use std::cmp::Ordering::{Equal, Greater, Less};
-
-                        use byte_unit::Byte;
-                        use byte_unit::UnitType::Binary;
-
-                        let post = Byte::from_u64(*post_size as u64).get_appropriate_unit(Binary);
-                        let diff_size = post_size.abs_diff(*pre_size) as u64;
-                        let diff = Byte::from_u64(diff_size).get_appropriate_unit(Binary);
-                        let sign = match post_size.cmp(pre_size) {
-                            Equal => return None,
-                            Greater => "+",
-                            Less => "-",
-                        };
-
-                        Some((
-                            dbname.to_case(Case::Camel),
-                            format!("{post:#.2} ({sign}{diff:#.2})").into(),
-                        ))
-                    })
-                    .into_iter()
-                    .flatten()
-            })
-            .collect();
-    }
-
     pub fn to_batch(&self) -> Batch {
         Batch {
             uid: self.uid,
@@ -306,15 +256,14 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) {
     use KindWithContent as K;
     let mut index_uids = vec![];
     match &mut task.kind {
-        K::DocumentAdditionOrUpdate { index_uid, .. }
+        K::DocumentAdditionOrUpdate { index_uid, .. } => index_uids.push(index_uid),
-        | K::DocumentEdition { index_uid, .. }
+        K::DocumentEdition { index_uid, .. } => index_uids.push(index_uid),
-        | K::DocumentDeletion { index_uid, .. }
+        K::DocumentDeletion { index_uid, .. } => index_uids.push(index_uid),
-        | K::DocumentDeletionByFilter { index_uid, .. }
+        K::DocumentDeletionByFilter { index_uid, .. } => index_uids.push(index_uid),
-        | K::DocumentClear { index_uid }
+        K::DocumentClear { index_uid } => index_uids.push(index_uid),
-        | K::SettingsUpdate { index_uid, .. }
+        K::SettingsUpdate { index_uid, .. } => index_uids.push(index_uid),
-        | K::IndexDeletion { index_uid }
+        K::IndexDeletion { index_uid } => index_uids.push(index_uid),
-        | K::IndexCreation { index_uid, .. }
+        K::IndexCreation { index_uid, .. } => index_uids.push(index_uid),
-        | K::IndexCompaction { index_uid, .. } => index_uids.push(index_uid),
         K::IndexUpdate { index_uid, new_index_uid, .. } => {
             index_uids.push(index_uid);
             if let Some(new_uid) = new_index_uid {
@@ -336,7 +285,6 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) {
         | K::DumpCreation { .. }
         | K::Export { .. }
         | K::UpgradeDatabase { .. }
-        | K::NetworkTopologyChange(_)
         | K::SnapshotCreation => (),
     };
     if let Some(Details::IndexSwap { swaps }) = &mut task.details {
@@ -430,7 +378,6 @@ impl crate::IndexScheduler {
                 status,
                 kind,
                 network: _,
-                custom_metadata: _,
             } = task;
             assert_eq!(uid, task.uid);
             if task.status != Status::Enqueued {
@@ -671,16 +618,6 @@ impl crate::IndexScheduler {
                     Details::UpgradeDatabase { from: _, to: _ } => {
                         assert_eq!(kind.as_kind(), Kind::UpgradeDatabase);
                     }
-                    Details::IndexCompaction {
-                        index_uid: _,
-                        pre_compaction_size: _,
-                        post_compaction_size: _,
-                    } => {
-                        assert_eq!(kind.as_kind(), Kind::IndexCompaction);
-                    }
-                    Details::NetworkTopologyChange { moved_documents: _, message: _ } => {
-                        assert_eq!(kind.as_kind(), Kind::NetworkTopologyChange);
-                    }
                 }
             }
 

crates/index-scheduler/src/versioning.rs

@@ -64,7 +64,14 @@ impl Versioning {
         };
         wtxn.commit()?;
 
-        upgrade_index_scheduler(env, &this, from)?;
+        let bin_major: u32 = versioning::VERSION_MAJOR;
+        let bin_minor: u32 = versioning::VERSION_MINOR;
+        let bin_patch: u32 = versioning::VERSION_PATCH;
+        let to = (bin_major, bin_minor, bin_patch);
+
+        if from != to {
+            upgrade_index_scheduler(env, &this, from, to)?;
+        }
 
         // Once we reach this point it means the upgrade process, if there was one is entirely finished
         // we can safely say we reached the latest version of the index scheduler

crates/json-depth-checker/Cargo.toml

@@ -15,7 +15,7 @@ license.workspace = true
 serde_json = "1.0"
 
 [dev-dependencies]
-criterion = "0.7.0"
+criterion = "0.6.0"
 
 [[bench]]
 name = "depth"

crates/meili-snap/Cargo.toml

@@ -13,7 +13,7 @@ license.workspace = true
 [dependencies]
 # fixed version due to format breakages in v1.40
 insta = { version = "=1.39.0", features = ["json", "redactions"] }
-md5 = "0.8.0"
+md5 = "0.7.0"
 once_cell = "1.21"
-regex-lite = "0.1.8"
+regex-lite = "0.1.6"
-uuid = { version = "1.18.1", features = ["v4"] }
+uuid = { version = "1.17.0", features = ["v4"] }

crates/meilisearch-auth/Cargo.toml

@@ -12,15 +12,15 @@ license.workspace = true
 
 [dependencies]
 base64 = "0.22.1"
-enum-iterator = "2.3.0"
+enum-iterator = "2.1.0"
 hmac = "0.12.1"
 maplit = "1.0.2"
 meilisearch-types = { path = "../meilisearch-types" }
 rand = "0.8.5"
 roaring = { version = "0.10.12", features = ["serde"] }
-serde = { version = "1.0.228", features = ["derive"] }
+serde = { version = "1.0.219", features = ["derive"] }
-serde_json = { version = "1.0.145", features = ["preserve_order"] }
+serde_json = { version = "1.0.140", features = ["preserve_order"] }
 sha2 = "0.10.9"
-thiserror = "2.0.17"
+thiserror = "2.0.12"
-time = { version = "0.3.44", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+time = { version = "0.3.41", features = ["serde-well-known", "formatting", "parsing", "macros"] }
-uuid = { version = "1.18.1", features = ["serde", "v4"] }
+uuid = { version = "1.17.0", features = ["serde", "v4"] }

crates/meilisearch-auth/src/store.rs

@@ -109,7 +109,6 @@ impl HeedAuthStore {
                             Action::IndexesGet,
                             Action::IndexesUpdate,
                             Action::IndexesSwap,
-                            Action::IndexesCompact,
                         ]
                         .iter(),
                     );

crates/meilisearch-types/Cargo.toml

@@ -11,41 +11,38 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-actix-web = { version = "4.12.0", default-features = false }
+actix-web = { version = "4.11.0", default-features = false }
-anyhow = "1.0.100"
+anyhow = "1.0.98"
-base64 = "0.22.1"
+bumpalo = "3.18.1"
-bumpalo = "3.19.0"
 bumparaw-collections = "0.1.4"
 byte-unit = { version = "5.1.6", features = ["serde"] }
-convert_case = "0.9.0"
+convert_case = "0.8.0"
-csv = "1.4.0"
+csv = "1.3.1"
-deserr = { version = "0.6.4", features = ["actix-web"] }
+deserr = { version = "0.6.3", features = ["actix-web"] }
 either = { version = "1.15.0", features = ["serde"] }
-enum-iterator = "2.3.0"
+enum-iterator = "2.1.0"
 file-store = { path = "../file-store" }
-flate2 = "1.1.5"
+flate2 = "1.1.2"
 fst = "0.4.7"
-itertools = "0.14.0"
+memmap2 = "0.9.7"
-memmap2 = "0.9.9"
 milli = { path = "../milli" }
 roaring = { version = "0.10.12", features = ["serde"] }
 rustc-hash = "2.1.1"
-serde = { version = "1.0.228", features = ["derive"] }
+serde = { version = "1.0.219", features = ["derive"] }
 serde-cs = "0.2.4"
-serde_json = { version = "1.0.145", features = ["preserve_order"] }
+serde_json = { version = "1.0.140", features = ["preserve_order"] }
 tar = "0.4.44"
-tempfile = "3.23.0"
+tempfile = "3.20.0"
-thiserror = "2.0.17"
+thiserror = "2.0.12"
-time = { version = "0.3.44", features = [
+time = { version = "0.3.41", features = [
     "serde-well-known",
     "formatting",
     "parsing",
     "macros",
 ] }
-tokio = "1.48"
+tokio = "1.45"
-urlencoding = "2.1.3"
 utoipa = { version = "5.4.0", features = ["macros"] }
-uuid = { version = "1.18.1", features = ["serde", "v4"] }
+uuid = { version = "1.17.0", features = ["serde", "v4"] }
 
 [dev-dependencies]
 # fixed version due to format breakages in v1.40
@@ -59,9 +56,6 @@ all-tokenizations = ["milli/all-tokenizations"]
 # chinese specialized tokenization
 chinese = ["milli/chinese"]
 chinese-pinyin = ["milli/chinese-pinyin"]
-
-enterprise = ["milli/enterprise"]
-
 # hebrew specialized tokenization
 hebrew = ["milli/hebrew"]
 # japanese specialized tokenization

crates/meilisearch-types/src/community_edition.rs

@@ -1,16 +0,0 @@
-pub mod network {
-    use milli::update::new::indexer::current_edition::sharding::Shards;
-
-    use crate::network::Network;
-
-    impl Network {
-        pub fn shards(&self) -> Option<Shards> {
-            None
-        }
-
-        pub fn sharding(&self) -> bool {
-            // always false in CE
-            false
-        }
-    }
-}

crates/meilisearch-types/src/enterprise_edition/network.rs

@@ -3,23 +3,45 @@
 // Use of this source code is governed by the Business Source License 1.1,
 // as found in the LICENSE-EE file or at <https://mariadb.com/bsl11>
 
-use milli::update::new::indexer::enterprise_edition::sharding::Shards;
+use std::collections::BTreeMap;
 
-use crate::network::Network;
+use milli::update::new::indexer::enterprise_edition::sharding::Shards;
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct Network {
+    #[serde(default, rename = "self")]
+    pub local: Option<String>,
+    #[serde(default)]
+    pub remotes: BTreeMap<String, Remote>,
+    #[serde(default)]
+    pub sharding: bool,
+}
 
 impl Network {
     pub fn shards(&self) -> Option<Shards> {
-        if self.sharding() {
+        if self.sharding {
-            Some(Shards::from_remotes_local(
+            let this = self.local.as_deref().expect("Inconsistent `sharding` and `self`");
-                self.remotes.keys().map(String::as_str),
+            let others = self
-                self.local.as_deref(),
+                .remotes
-            ))
+                .keys()
+                .filter(|name| name.as_str() != this)
+                .map(|name| name.to_owned())
+                .collect();
+            Some(Shards { own: vec![this.to_owned()], others })
         } else {
             None
         }
     }
-
+}
-    pub fn sharding(&self) -> bool {
+
-        self.leader.is_some()
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
-    }
+#[serde(rename_all = "camelCase")]
+pub struct Remote {
+    pub url: String,
+    #[serde(default)]
+    pub search_api_key: Option<String>,
+    #[serde(default)]
+    pub write_api_key: Option<String>,
 }

crates/meilisearch-types/src/error.rs

@@ -156,7 +156,7 @@ macro_rules! make_error_codes {
             }
 
             /// return error name, used as error code
-            pub fn name(&self) -> String {
+            fn name(&self) -> String {
                 match self {
                     $(
                         Code::$code_ident => stringify!($code_ident).to_case(convert_case::Case::Snake)
@@ -214,9 +214,6 @@ ImmutableApiKeyUid                             , InvalidRequest       , BAD_REQU
 ImmutableApiKeyUpdatedAt                       , InvalidRequest       , BAD_REQUEST;
 ImmutableIndexCreatedAt                        , InvalidRequest       , BAD_REQUEST;
 ImmutableIndexUpdatedAt                        , InvalidRequest       , BAD_REQUEST;
-ImportTaskAlreadyReceived                      , InvalidRequest       , PRECONDITION_FAILED;
-ImportTaskUnknownRemote                        , InvalidRequest       , PRECONDITION_FAILED;
-ImportTaskWithoutNetworkTask                   , InvalidRequest       , SERVICE_UNAVAILABLE;
 IndexAlreadyExists                             , InvalidRequest       , CONFLICT ;
 IndexCreationFailed                            , Internal             , INTERNAL_SERVER_ERROR;
 IndexNotFound                                  , InvalidRequest       , NOT_FOUND;
@@ -257,12 +254,10 @@ InvalidSearchHybridQuery                       , InvalidRequest       , BAD_REQU
 InvalidIndexLimit                              , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexOffset                             , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexPrimaryKey                         , InvalidRequest       , BAD_REQUEST ;
-InvalidIndexCustomMetadata                     , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexUid                                , InvalidRequest       , BAD_REQUEST ;
 InvalidMultiSearchFacets                       , InvalidRequest       , BAD_REQUEST ;
 InvalidMultiSearchFacetsByIndex                , InvalidRequest       , BAD_REQUEST ;
 InvalidMultiSearchFacetOrder                   , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchQueryPersonalization         , InvalidRequest       , BAD_REQUEST ;
 InvalidMultiSearchFederated                    , InvalidRequest       , BAD_REQUEST ;
 InvalidMultiSearchFederationOptions            , InvalidRequest       , BAD_REQUEST ;
 InvalidMultiSearchMaxValuesPerFacet            , InvalidRequest       , BAD_REQUEST ;
@@ -273,9 +268,9 @@ InvalidMultiSearchQueryRankingRules            , InvalidRequest       , BAD_REQU
 InvalidMultiSearchQueryPosition                , InvalidRequest       , BAD_REQUEST ;
 InvalidMultiSearchRemote                       , InvalidRequest       , BAD_REQUEST ;
 InvalidMultiSearchWeight                       , InvalidRequest       , BAD_REQUEST ;
-InvalidNetworkLeader                           , InvalidRequest       , BAD_REQUEST ;
 InvalidNetworkRemotes                          , InvalidRequest       , BAD_REQUEST ;
 InvalidNetworkSelf                             , InvalidRequest       , BAD_REQUEST ;
+InvalidNetworkSharding                         , InvalidRequest       , BAD_REQUEST ;
 InvalidNetworkSearchApiKey                     , InvalidRequest       , BAD_REQUEST ;
 InvalidNetworkWriteApiKey                      , InvalidRequest       , BAD_REQUEST ;
 InvalidNetworkUrl                              , InvalidRequest       , BAD_REQUEST ;
@@ -320,8 +315,6 @@ InvalidSearchShowRankingScoreDetails           , InvalidRequest       , BAD_REQU
 InvalidSimilarShowRankingScoreDetails          , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchSort                              , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchDistinct                          , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchPersonalize                       , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchPersonalizeUserContext            , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchMediaAndVector                    , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsDisplayedAttributes             , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsDistinctAttribute               , InvalidRequest       , BAD_REQUEST ;
@@ -380,9 +373,7 @@ MissingPayload                                 , InvalidRequest       , BAD_REQU
 MissingSearchHybrid                            , InvalidRequest       , BAD_REQUEST ;
 MissingSwapIndexes                             , InvalidRequest       , BAD_REQUEST ;
 MissingTaskFilters                             , InvalidRequest       , BAD_REQUEST ;
-NetworkVersionMismatch                         , InvalidRequest       , PRECONDITION_FAILED ;
 NoSpaceLeftOnDevice                            , System               , UNPROCESSABLE_ENTITY;
-NotLeader                                      , InvalidRequest       , BAD_REQUEST ;
 PayloadTooLarge                                , InvalidRequest       , PAYLOAD_TOO_LARGE ;
 RemoteBadResponse                              , System               , BAD_GATEWAY ;
 RemoteBadRequest                               , InvalidRequest       , BAD_REQUEST ;
@@ -396,15 +387,9 @@ TaskFileNotFound                               , InvalidRequest       , NOT_FOUN
 BatchNotFound                                  , InvalidRequest       , NOT_FOUND ;
 TooManyOpenFiles                               , System               , UNPROCESSABLE_ENTITY ;
 TooManyVectors                                 , InvalidRequest       , BAD_REQUEST ;
-UnexpectedNetworkPreviousRemotes               , InvalidRequest       , BAD_REQUEST ;
-NetworkVersionTooOld                           , InvalidRequest       , BAD_REQUEST ;
-UnprocessedNetworkTask                         , InvalidRequest       , BAD_REQUEST ;
 UnretrievableDocument                          , Internal             , BAD_REQUEST ;
 UnretrievableErrorCode                         , InvalidRequest       , BAD_REQUEST ;
 UnsupportedMediaType                           , InvalidRequest       , UNSUPPORTED_MEDIA_TYPE ;
-InvalidS3SnapshotRequest                       , Internal             , BAD_REQUEST ;
-InvalidS3SnapshotParameters                    , Internal             , BAD_REQUEST ;
-S3SnapshotServerError                          , Internal             , BAD_GATEWAY ;
 
 // Experimental features
 VectorEmbeddingError                           , InvalidRequest       , BAD_REQUEST ;
@@ -441,7 +426,6 @@ InvalidChatCompletionSearchQueryParamPrompt    , InvalidRequest       , BAD_REQU
 InvalidChatCompletionSearchFilterParamPrompt   , InvalidRequest       , BAD_REQUEST ;
 InvalidChatCompletionSearchIndexUidParamPrompt , InvalidRequest       , BAD_REQUEST ;
 InvalidChatCompletionPreQueryPrompt            , InvalidRequest       , BAD_REQUEST ;
-RequiresEnterpriseEdition                      , InvalidRequest       , UNAVAILABLE_FOR_LEGAL_REASONS ;
 // Webhooks
 InvalidWebhooks                                , InvalidRequest       , BAD_REQUEST ;
 InvalidWebhookUrl                              , InvalidRequest       , BAD_REQUEST ;
@@ -695,18 +679,6 @@ impl fmt::Display for deserr_codes::InvalidNetworkSearchApiKey {
     }
 }
 
-impl fmt::Display for deserr_codes::InvalidSearchPersonalize {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "the value of `personalize` is invalid, expected a JSON object with `userContext` string.")
-    }
-}
-
-impl fmt::Display for deserr_codes::InvalidSearchPersonalizeUserContext {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "the value of `userContext` is invalid, expected a string.")
-    }
-}
-
 #[macro_export]
 macro_rules! internal_error {
     ($target:ty : $($other:path), *) => {

crates/meilisearch-types/src/keys.rs

@@ -380,9 +380,6 @@ pub enum Action {
     #[serde(rename = "webhooks.*")]
     #[deserr(rename = "webhooks.*")]
     WebhooksAll,
-    #[serde(rename = "indexes.compact")]
-    #[deserr(rename = "indexes.compact")]
-    IndexesCompact,
 }
 
 impl Action {
@@ -401,7 +398,6 @@ impl Action {
             INDEXES_UPDATE => Some(Self::IndexesUpdate),
             INDEXES_DELETE => Some(Self::IndexesDelete),
             INDEXES_SWAP => Some(Self::IndexesSwap),
-            INDEXES_COMPACT => Some(Self::IndexesCompact),
             TASKS_ALL => Some(Self::TasksAll),
             TASKS_CANCEL => Some(Self::TasksCancel),
             TASKS_DELETE => Some(Self::TasksDelete),
@@ -466,7 +462,6 @@ impl Action {
             IndexesUpdate => false,
             IndexesDelete => false,
             IndexesSwap => false,
-            IndexesCompact => false,
             TasksCancel => false,
             TasksDelete => false,
             TasksGet => true,
@@ -518,7 +513,6 @@ pub mod actions {
     pub const INDEXES_UPDATE: u8 = IndexesUpdate.repr();
     pub const INDEXES_DELETE: u8 = IndexesDelete.repr();
     pub const INDEXES_SWAP: u8 = IndexesSwap.repr();
-    pub const INDEXES_COMPACT: u8 = IndexesCompact.repr();
     pub const TASKS_ALL: u8 = TasksAll.repr();
     pub const TASKS_CANCEL: u8 = TasksCancel.repr();
     pub const TASKS_DELETE: u8 = TasksDelete.repr();
@@ -620,7 +614,6 @@ pub(crate) mod test {
         assert!(WebhooksDelete.repr() == 47 && WEBHOOKS_DELETE == 47);
         assert!(WebhooksCreate.repr() == 48 && WEBHOOKS_CREATE == 48);
         assert!(WebhooksAll.repr() == 49 && WEBHOOKS_ALL == 49);
-        assert!(IndexesCompact.repr() == 50 && INDEXES_COMPACT == 50);
     }
 
     #[test]

crates/meilisearch-types/src/lib.rs

@@ -2,17 +2,10 @@
 
 pub mod batch_view;
 pub mod batches;
-#[cfg(not(feature = "enterprise"))]
-pub mod community_edition;
 pub mod compression;
 pub mod deserr;
 pub mod document_formats;
-#[cfg(feature = "enterprise")]
 pub mod enterprise_edition;
-#[cfg(not(feature = "enterprise"))]
-pub use community_edition as current_edition;
-#[cfg(feature = "enterprise")]
-pub use enterprise_edition as current_edition;
 pub mod error;
 pub mod facet_values_sort;
 pub mod features;
@@ -20,7 +13,6 @@ pub mod index_uid;
 pub mod index_uid_pattern;
 pub mod keys;
 pub mod locales;
-pub mod network;
 pub mod settings;
 pub mod star_or;
 pub mod task_view;

crates/meilisearch-types/src/network.rs

@@ -1,27 +0,0 @@
-use std::collections::BTreeMap;
-
-use serde::{Deserialize, Serialize};
-use uuid::Uuid;
-
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)]
-#[serde(rename_all = "camelCase")]
-pub struct Network {
-    #[serde(default, rename = "self")]
-    pub local: Option<String>,
-    #[serde(default)]
-    pub remotes: BTreeMap<String, Remote>,
-    #[serde(default)]
-    pub leader: Option<String>,
-    #[serde(default)]
-    pub version: Uuid,
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
-#[serde(rename_all = "camelCase")]
-pub struct Remote {
-    pub url: String,
-    #[serde(default)]
-    pub search_api_key: Option<String>,
-    #[serde(default)]
-    pub write_api_key: Option<String>,
-}

crates/meilisearch-types/src/settings.rs

@@ -346,26 +346,24 @@ impl<T> Settings<T> {
                 continue;
             };
 
-            hide_secret(api_key, 0);
+            Self::hide_secret(api_key);
         }
     }
-}
 
-/// Redact a secret string, starting from the `secret_offset`th byte.
+    fn hide_secret(secret: &mut String) {
-pub fn hide_secret(secret: &mut String, secret_offset: usize) {
+        match secret.len() {
-    match secret.len().checked_sub(secret_offset) {
+            x if x < 10 => {
-        None => (),
+                secret.replace_range(.., "XXX...");
-        Some(x) if x < 10 => {
+            }
-            secret.replace_range(secret_offset.., "XXX...");
+            x if x < 20 => {
-        }
+                secret.replace_range(2.., "XXXX...");
-        Some(x) if x < 20 => {
+            }
-            secret.replace_range((secret_offset + 2).., "XXXX...");
+            x if x < 30 => {
-        }
+                secret.replace_range(3.., "XXXXX...");
-        Some(x) if x < 30 => {
+            }
-            secret.replace_range((secret_offset + 3).., "XXXXX...");
+            _x => {
-        }
+                secret.replace_range(5.., "XXXXXX...");
-        Some(_x) => {
+            }
-            secret.replace_range((secret_offset + 5).., "XXXXXX...");
         }
     }
 }

crates/meilisearch-types/src/task_view.rs

@@ -9,12 +9,12 @@ use utoipa::ToSchema;
 use crate::batches::BatchId;
 use crate::error::ResponseError;
 use crate::settings::{Settings, Unchecked};
-use crate::tasks::network::DbTaskNetwork;
 use crate::tasks::{
     serialize_duration, Details, DetailsExportIndexSettings, IndexSwap, Kind, Status, Task, TaskId,
+    TaskNetwork,
 };
 
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)]
+#[derive(Debug, Clone, PartialEq, Serialize, ToSchema)]
 #[serde(rename_all = "camelCase")]
 #[schema(rename_all = "camelCase")]
 pub struct TaskView {
@@ -54,10 +54,7 @@ pub struct TaskView {
     pub finished_at: Option<OffsetDateTime>,
 
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub network: Option<DbTaskNetwork>,
+    pub network: Option<TaskNetwork>,
-
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub custom_metadata: Option<String>,
 }
 
 impl TaskView {
@@ -76,7 +73,6 @@ impl TaskView {
             started_at: task.started_at,
             finished_at: task.finished_at,
             network: task.network.clone(),
-            custom_metadata: task.custom_metadata.clone(),
         }
     }
 }
@@ -146,16 +142,6 @@ pub struct DetailsView {
     pub old_index_uid: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub new_index_uid: Option<String>,
-    // index compaction
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub pre_compaction_size: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub post_compaction_size: Option<String>,
-    // network topology change
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub moved_documents: Option<u64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub message: Option<String>,
 }
 
 impl DetailsView {
@@ -166,17 +152,6 @@ impl DetailsView {
                 (None, Some(doc)) | (Some(doc), None) => Some(doc),
                 (Some(left), Some(right)) => Some(left + right),
             },
-            moved_documents: match (self.moved_documents, other.moved_documents) {
-                (None, None) => None,
-                (None, Some(doc)) | (Some(doc), None) => Some(doc),
-                (Some(left), Some(right)) => Some(left + right),
-            },
-            message: match (&mut self.message, &other.message) {
-                (None, None) => None,
-                (None, Some(message)) => Some(message.clone()),
-                (Some(message), None) => Some(std::mem::take(message)),
-                (Some(message), Some(_)) => Some(std::mem::take(message)),
-            },
             indexed_documents: match (self.indexed_documents, other.indexed_documents) {
                 (None, None) => None,
                 (None, Some(None)) | (Some(None), None) | (Some(None), Some(None)) => Some(None),
@@ -339,24 +314,6 @@ impl DetailsView {
                 // We should never be able to batch multiple renames at the same time.
                 (Some(left), Some(_right)) => Some(left),
             },
-            pre_compaction_size: match (
-                self.pre_compaction_size.clone(),
-                other.pre_compaction_size.clone(),
-            ) {
-                (None, None) => None,
-                (None, Some(size)) | (Some(size), None) => Some(size),
-                // We should never be able to batch multiple compactions at the same time.
-                (Some(left), Some(_right)) => Some(left),
-            },
-            post_compaction_size: match (
-                self.post_compaction_size.clone(),
-                other.post_compaction_size.clone(),
-            ) {
-                (None, None) => None,
-                (None, Some(size)) | (Some(size), None) => Some(size),
-                // We should never be able to batch multiple compactions at the same time.
-                (Some(left), Some(_right)) => Some(left),
-            },
         }
     }
 }
@@ -458,20 +415,6 @@ impl From<Details> for DetailsView {
                 upgrade_to: Some(format!("v{}.{}.{}", to.0, to.1, to.2)),
                 ..Default::default()
             },
-            Details::IndexCompaction { pre_compaction_size, post_compaction_size, .. } => {
-                DetailsView {
-                    pre_compaction_size: pre_compaction_size
-                        .map(|size| size.get_appropriate_unit(UnitType::Both).to_string()),
-                    post_compaction_size: post_compaction_size
-                        .map(|size| size.get_appropriate_unit(UnitType::Both).to_string()),
-                    ..Default::default()
-                }
-            }
-            Details::NetworkTopologyChange { moved_documents, message } => DetailsView {
-                moved_documents: Some(moved_documents),
-                message: Some(message),
-                ..Default::default()
-            },
         }
     }
 }

crates/meilisearch-types/src/tasks.rs

@@ -23,8 +23,6 @@ use crate::{versioning, InstanceUid};
 
 pub type TaskId = u32;
 
-pub mod network;
-
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Task {
@@ -46,10 +44,7 @@ pub struct Task {
     pub kind: KindWithContent,
 
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub network: Option<network::DbTaskNetwork>,
+    pub network: Option<TaskNetwork>,
-
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub custom_metadata: Option<String>,
 }
 
 impl Task {
@@ -63,7 +58,6 @@ impl Task {
             | TaskDeletion { .. }
             | Export { .. }
             | UpgradeDatabase { .. }
-            | NetworkTopologyChange { .. }
             | IndexSwap { .. } => None,
             DocumentAdditionOrUpdate { index_uid, .. }
             | DocumentEdition { index_uid, .. }
@@ -73,8 +67,7 @@ impl Task {
             | SettingsUpdate { index_uid, .. }
             | IndexCreation { index_uid, .. }
             | IndexUpdate { index_uid, .. }
-            | IndexDeletion { index_uid }
+            | IndexDeletion { index_uid } => Some(index_uid),
-            | IndexCompaction { index_uid } => Some(index_uid),
         }
     }
 
@@ -101,9 +94,7 @@ impl Task {
             | KindWithContent::DumpCreation { .. }
             | KindWithContent::SnapshotCreation
             | KindWithContent::Export { .. }
-            | KindWithContent::UpgradeDatabase { .. }
+            | KindWithContent::UpgradeDatabase { .. } => None,
-            | KindWithContent::NetworkTopologyChange { .. }
-            | KindWithContent::IndexCompaction { .. } => None,
         }
     }
 }
@@ -179,10 +170,6 @@ pub enum KindWithContent {
     UpgradeDatabase {
         from: (u32, u32, u32),
     },
-    IndexCompaction {
-        index_uid: String,
-    },
-    NetworkTopologyChange(network::NetworkTopologyChange),
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
@@ -219,8 +206,6 @@ impl KindWithContent {
             KindWithContent::SnapshotCreation => Kind::SnapshotCreation,
             KindWithContent::Export { .. } => Kind::Export,
             KindWithContent::UpgradeDatabase { .. } => Kind::UpgradeDatabase,
-            KindWithContent::IndexCompaction { .. } => Kind::IndexCompaction,
-            KindWithContent::NetworkTopologyChange { .. } => Kind::NetworkTopologyChange,
         }
     }
 
@@ -233,7 +218,6 @@ impl KindWithContent {
             | TaskCancelation { .. }
             | TaskDeletion { .. }
             | Export { .. }
-            | NetworkTopologyChange { .. }
             | UpgradeDatabase { .. } => vec![],
             DocumentAdditionOrUpdate { index_uid, .. }
             | DocumentEdition { index_uid, .. }
@@ -242,8 +226,7 @@ impl KindWithContent {
             | DocumentClear { index_uid }
             | SettingsUpdate { index_uid, .. }
             | IndexCreation { index_uid, .. }
-            | IndexDeletion { index_uid }
+            | IndexDeletion { index_uid } => vec![index_uid],
-            | IndexCompaction { index_uid } => vec![index_uid],
             IndexUpdate { index_uid, new_index_uid, .. } => {
                 let mut indexes = vec![index_uid.as_str()];
                 if let Some(new_uid) = new_index_uid {
@@ -342,15 +325,6 @@ impl KindWithContent {
                     versioning::VERSION_PATCH,
                 ),
             }),
-            KindWithContent::IndexCompaction { index_uid } => Some(Details::IndexCompaction {
-                index_uid: index_uid.clone(),
-                pre_compaction_size: None,
-                post_compaction_size: None,
-            }),
-            KindWithContent::NetworkTopologyChange { .. } => Some(Details::NetworkTopologyChange {
-                moved_documents: 0,
-                message: "processing tasks for previous network versions".into(),
-            }),
         }
     }
 
@@ -403,7 +377,7 @@ impl KindWithContent {
                 })
             }
             KindWithContent::IndexSwap { .. } => {
-                unimplemented!("do not call `default_finished_details` for `IndexSwap` tasks")
+                todo!()
             }
             KindWithContent::TaskCancelation { query, tasks } => Some(Details::TaskCancelation {
                 matched_tasks: tasks.len(),
@@ -433,14 +407,6 @@ impl KindWithContent {
                     versioning::VERSION_PATCH,
                 ),
             }),
-            KindWithContent::IndexCompaction { index_uid } => Some(Details::IndexCompaction {
-                index_uid: index_uid.clone(),
-                pre_compaction_size: None,
-                post_compaction_size: None,
-            }),
-            KindWithContent::NetworkTopologyChange(network_topology_change) => {
-                Some(network_topology_change.to_details())
-            }
         }
     }
 }
@@ -503,14 +469,6 @@ impl From<&KindWithContent> for Option<Details> {
                     versioning::VERSION_PATCH,
                 ),
             }),
-            KindWithContent::IndexCompaction { index_uid } => Some(Details::IndexCompaction {
-                index_uid: index_uid.clone(),
-                pre_compaction_size: None,
-                post_compaction_size: None,
-            }),
-            KindWithContent::NetworkTopologyChange(network_topology_change) => {
-                Some(network_topology_change.to_details())
-            }
         }
     }
 }
@@ -621,8 +579,6 @@ pub enum Kind {
     SnapshotCreation,
     Export,
     UpgradeDatabase,
-    IndexCompaction,
-    NetworkTopologyChange,
 }
 
 impl Kind {
@@ -634,15 +590,13 @@ impl Kind {
             | Kind::SettingsUpdate
             | Kind::IndexCreation
             | Kind::IndexDeletion
-            | Kind::IndexUpdate
+            | Kind::IndexUpdate => true,
-            | Kind::IndexCompaction => true,
             Kind::IndexSwap
             | Kind::TaskCancelation
             | Kind::TaskDeletion
             | Kind::DumpCreation
             | Kind::Export
             | Kind::UpgradeDatabase
-            | Kind::NetworkTopologyChange
             | Kind::SnapshotCreation => false,
         }
     }
@@ -664,8 +618,6 @@ impl Display for Kind {
             Kind::SnapshotCreation => write!(f, "snapshotCreation"),
             Kind::Export => write!(f, "export"),
             Kind::UpgradeDatabase => write!(f, "upgradeDatabase"),
-            Kind::IndexCompaction => write!(f, "indexCompaction"),
-            Kind::NetworkTopologyChange => write!(f, "networkTopologyChange"),
         }
     }
 }
@@ -701,10 +653,6 @@ impl FromStr for Kind {
             Ok(Kind::Export)
         } else if kind.eq_ignore_ascii_case("upgradeDatabase") {
             Ok(Kind::UpgradeDatabase)
-        } else if kind.eq_ignore_ascii_case("indexCompaction") {
-            Ok(Kind::IndexCompaction)
-        } else if kind.eq_ignore_ascii_case("networkTopologyChange") {
-            Ok(Kind::NetworkTopologyChange)
         } else {
             Err(ParseTaskKindError(kind.to_owned()))
         }
@@ -790,15 +738,36 @@ pub enum Details {
         from: (u32, u32, u32),
         to: (u32, u32, u32),
     },
-    IndexCompaction {
+}
-        index_uid: String,
+
-        pre_compaction_size: Option<Byte>,
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)]
-        post_compaction_size: Option<Byte>,
+#[serde(untagged, rename_all = "camelCase")]
-    },
+pub enum TaskNetwork {
-    NetworkTopologyChange {
+    Origin { origin: Origin },
-        moved_documents: u64,
+    Remotes { remote_tasks: BTreeMap<String, RemoteTask> },
-        message: String,
+}
-    },
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct Origin {
+    pub remote_name: String,
+    pub task_uid: usize,
+}
+
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct RemoteTask {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    task_uid: Option<TaskId>,
+    error: Option<ResponseError>,
+}
+
+impl From<Result<TaskId, ResponseError>> for RemoteTask {
+    fn from(res: Result<TaskId, ResponseError>) -> RemoteTask {
+        match res {
+            Ok(task_uid) => RemoteTask { task_uid: Some(task_uid), error: None },
+            Err(err) => RemoteTask { task_uid: None, error: Some(err) },
+        }
+    }
 }
 
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)]
@@ -831,19 +800,12 @@ impl Details {
             Self::ClearAll { deleted_documents } => *deleted_documents = Some(0),
             Self::TaskCancelation { canceled_tasks, .. } => *canceled_tasks = Some(0),
             Self::TaskDeletion { deleted_tasks, .. } => *deleted_tasks = Some(0),
-            Self::IndexCompaction { pre_compaction_size, post_compaction_size, .. } => {
-                *pre_compaction_size = None;
-                *post_compaction_size = None;
-            }
             Self::SettingsUpdate { .. }
             | Self::IndexInfo { .. }
             | Self::Dump { .. }
             | Self::Export { .. }
             | Self::UpgradeDatabase { .. }
             | Self::IndexSwap { .. } => (),
-            Self::NetworkTopologyChange { moved_documents: _, message } => {
-                *message = format!("Failed. Previous status: {}", message);
-            }
         }
 
         details
@@ -899,17 +861,6 @@ pub enum BatchStopReason {
     SettingsWithDocumentOperation {
         id: TaskId,
     },
-    NetworkTask {
-        id: TaskId,
-    },
-    NetworkTaskOlderTasks {
-        id: TaskId,
-        inner_reason: String,
-    },
-    NetworkTaskImportTasks {
-        id: TaskId,
-        inner_reason: String,
-    },
 }
 
 impl BatchStopReason {
@@ -998,24 +949,6 @@ impl Display for BatchStopReason {
                     "stopped before task with id {id} because it is a document operation which cannot be batched with settings changes"
                 )
             }
-            BatchStopReason::NetworkTask { id } => {
-                write!(
-                    f,
-                    "stopped after task with id {id} because it is a network topology change task"
-                )
-            }
-            BatchStopReason::NetworkTaskOlderTasks { id, inner_reason } => {
-                write!(
-                    f,
-                    "stopped after batching network task with id {id} and a batch of older tasks: {inner_reason}"
-                )
-            }
-            BatchStopReason::NetworkTaskImportTasks { id, inner_reason } => {
-                write!(
-                    f,
-                    "stopped after batching network task with id {id} and a batch of import tasks: {inner_reason}"
-                )
-            }
         }
     }
 }

crates/meilisearch-types/src/tasks/network.rs

@@ -1,782 +0,0 @@
-use std::collections::BTreeMap;
-
-use base64::Engine as _;
-use itertools::{EitherOrBoth, Itertools as _};
-use milli::{CboRoaringBitmapCodec, DocumentId};
-use roaring::RoaringBitmap;
-use serde::{Deserialize, Serialize};
-use utoipa::ToSchema;
-use uuid::Uuid;
-
-use crate::error::ResponseError;
-use crate::network::Network;
-use crate::tasks::{Details, TaskId};
-
-#[cfg(not(feature = "enterprise"))]
-mod community_edition;
-#[cfg(feature = "enterprise")]
-mod enterprise_edition;
-
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)]
-#[serde(untagged, rename_all = "camelCase")]
-// This type is used in the database, care should be taken when modifying it.
-pub enum DbTaskNetwork {
-    /// Tasks that were duplicated from `origin`
-    Origin { origin: Origin },
-    /// Tasks that were duplicated as `remote_tasks`
-    Remotes {
-        remote_tasks: BTreeMap<String, RemoteTask>,
-        #[serde(default)]
-        network_version: Uuid,
-    },
-    /// Document import tasks sent in the context of `network_change`
-    Import { import_from: ImportData, network_change: Origin },
-}
-
-impl DbTaskNetwork {
-    pub fn network_version(&self) -> Uuid {
-        match self {
-            DbTaskNetwork::Origin { origin } => origin.network_version,
-            DbTaskNetwork::Remotes { remote_tasks: _, network_version } => *network_version,
-            DbTaskNetwork::Import { import_from: _, network_change } => {
-                network_change.network_version
-            }
-        }
-    }
-
-    pub fn import_data(&self) -> Option<&ImportData> {
-        match self {
-            DbTaskNetwork::Origin { .. } | DbTaskNetwork::Remotes { .. } => None,
-            DbTaskNetwork::Import { import_from, .. } => Some(import_from),
-        }
-    }
-
-    pub fn origin(&self) -> Option<&Origin> {
-        match self {
-            DbTaskNetwork::Origin { origin } => Some(origin),
-            DbTaskNetwork::Remotes { .. } => None,
-            DbTaskNetwork::Import { network_change, .. } => Some(network_change),
-        }
-    }
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum TaskNetwork {
-    /// Tasks that were duplicated from `origin`
-    Origin { origin: Origin },
-    /// Tasks that were duplicated as `remote_tasks`
-    Remotes { remote_tasks: BTreeMap<String, RemoteTask>, network_version: Uuid },
-    /// Document import tasks sent in the context of `network_change`
-    Import { import_from: ImportData, network_change: Origin, metadata: ImportMetadata },
-}
-
-impl TaskNetwork {
-    pub fn network_version(&self) -> Uuid {
-        match self {
-            TaskNetwork::Origin { origin } => origin.network_version,
-            TaskNetwork::Remotes { remote_tasks: _, network_version } => *network_version,
-            TaskNetwork::Import { import_from: _, network_change, metadata: _ } => {
-                network_change.network_version
-            }
-        }
-    }
-}
-
-impl From<TaskNetwork> for DbTaskNetwork {
-    fn from(value: TaskNetwork) -> Self {
-        match value {
-            TaskNetwork::Origin { origin } => DbTaskNetwork::Origin { origin },
-            TaskNetwork::Remotes { remote_tasks, network_version } => {
-                DbTaskNetwork::Remotes { remote_tasks, network_version }
-            }
-            TaskNetwork::Import { import_from, network_change, metadata: _ } => {
-                DbTaskNetwork::Import { import_from, network_change }
-            }
-        }
-    }
-}
-
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct Origin {
-    pub remote_name: String,
-    pub task_uid: u32,
-    #[serde(default)]
-    pub network_version: Uuid,
-}
-
-/// Import data stored in a task
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct ImportData {
-    /// Remote that this task is imported from
-    pub remote_name: String,
-    /// Index relevant to this task
-    pub index_name: Option<String>,
-    /// Number of documents in this task
-    pub document_count: u64,
-}
-
-/// Import metadata associated with a task but not stored in the task
-#[derive(Debug, PartialEq, Clone)]
-pub struct ImportMetadata {
-    /// Total number of indexes to import from this host
-    pub index_count: u64,
-    /// Key unique to this (network_change, index, host, key).
-    ///
-    /// In practice, an internal document id of one of the documents to import.
-    pub task_key: Option<DocumentId>,
-    /// Total number of documents to import for this index from this host.
-    pub total_index_documents: u64,
-}
-
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct RemoteTask {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    task_uid: Option<TaskId>,
-    error: Option<ResponseError>,
-}
-
-impl From<Result<TaskId, ResponseError>> for RemoteTask {
-    fn from(res: Result<TaskId, ResponseError>) -> RemoteTask {
-        match res {
-            Ok(task_uid) => RemoteTask { task_uid: Some(task_uid), error: None },
-            Err(err) => RemoteTask { task_uid: None, error: Some(err) },
-        }
-    }
-}
-
-/// Contains the full state of a network topology change.
-///
-/// A network topology change task is unique in that it can be processed in multiple different batches, as its resolution
-/// depends on various document additions tasks being processed.
-///
-/// A network topology task has 4 states:
-///
-/// 1. Processing any task that was meant for an earlier version of the network. This is necessary to know that we have the right version of
-///    documents.
-/// 2. Sending all documents that must be moved to other remotes.
-/// 3. Processing any task coming from the remotes.
-/// 4. Finished.
-///
-/// Furthermore, it maintains some stats
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct NetworkTopologyChange {
-    state: NetworkTopologyState,
-    in_remotes: BTreeMap<String, InRemote>,
-    old_network: Network,
-    new_network: Network,
-    stats: NetworkTopologyStats,
-}
-
-impl NetworkTopologyChange {
-    pub fn new(old_network: Network, new_network: Network) -> Self {
-        let in_name = new_network.local.as_deref();
-        let out_name = old_network.local.as_deref().or(in_name);
-
-        let in_remotes = if in_name.is_some() {
-            old_network
-                .remotes
-                .keys()
-                .chain(new_network.remotes.keys())
-                // don't await imports from ourselves
-                .filter(|name| Some(name.as_str()) != out_name)
-                .cloned()
-                .map(|name| (name, InRemote::new()))
-                .collect()
-        } else {
-            Default::default()
-        };
-        Self {
-            state: NetworkTopologyState::WaitingForOlderTasks,
-            in_remotes,
-            stats: NetworkTopologyStats { moved_documents: 0 },
-            new_network,
-            old_network,
-        }
-    }
-
-    pub fn in_name(&self) -> Option<&str> {
-        self.new_network.local.as_deref()
-    }
-
-    pub fn out_name(&self) -> Option<&str> {
-        self.old_network.local.as_deref().or_else(|| self.in_name())
-    }
-
-    pub fn state(&self) -> NetworkTopologyState {
-        self.state
-    }
-
-    pub fn to_details(&self) -> Details {
-        let message = match self.state {
-            NetworkTopologyState::WaitingForOlderTasks => {
-                "Waiting for tasks enqueued before the network change to finish processing".into()
-            }
-            NetworkTopologyState::ExportingDocuments => "Exporting documents".into(),
-            NetworkTopologyState::ImportingDocuments => {
-                let mut finished_count = 0;
-                let mut first_ongoing = None;
-                let mut ongoing_total_indexes = 0;
-                let mut ongoing_processed_documents = 0;
-                let mut ongoing_missing_documents = 0;
-                let mut ongoing_total_documents = 0;
-                let mut other_ongoing_count = 0;
-                let mut first_waiting = None;
-                let mut other_waiting_count = 0;
-                for (remote_name, in_remote) in &self.in_remotes {
-                    match &in_remote.import_state {
-                        ImportState::WaitingForInitialTask => {
-                            first_waiting = match first_waiting {
-                                None => Some(remote_name),
-                                first_waiting => {
-                                    other_waiting_count += 1;
-                                    first_waiting
-                                }
-                            };
-                        }
-                        ImportState::Ongoing { import_index_state, total_indexes } => {
-                            first_ongoing = match first_ongoing {
-                                None => {
-                                    ongoing_total_indexes = *total_indexes;
-                                    Some(remote_name)
-                                }
-                                first_ongoing => {
-                                    other_ongoing_count += 1;
-                                    first_ongoing
-                                }
-                            };
-                            for import_state in import_index_state.values() {
-                                match import_state {
-                                    ImportIndexState::Ongoing {
-                                        total_documents,
-                                        processed_documents,
-                                        received_documents,
-                                        task_keys: _,
-                                    } => {
-                                        ongoing_total_documents += total_documents;
-                                        ongoing_processed_documents += processed_documents;
-                                        ongoing_missing_documents +=
-                                            total_documents.saturating_sub(*received_documents);
-                                    }
-                                    ImportIndexState::Finished { total_documents } => {
-                                        ongoing_total_documents += total_documents;
-                                        ongoing_processed_documents += total_documents;
-                                    }
-                                }
-                            }
-                        }
-                        ImportState::Finished { total_indexes, total_documents } => {
-                            finished_count += 1;
-                            ongoing_total_indexes = *total_indexes;
-                            ongoing_total_documents += *total_documents;
-                            ongoing_processed_documents += *total_documents;
-                        }
-                    }
-                }
-                format!(
-                    "Importing documents from {total} remotes{waiting}{ongoing}{finished}",
-                    total = self.in_remotes.len(),
-                    waiting = if let Some(first_waiting) = first_waiting {
-                        format!(
-                            ", waiting on first task from `{}`{others}",
-                            first_waiting,
-                            others = if other_waiting_count > 0 {
-                                format!(" and {other_waiting_count} other remotes")
-                            } else {
-                                "".into()
-                            }
-                        )
-                    } else {
-                        "".into()
-                    },
-                    ongoing = if let Some(first_ongoing) = first_ongoing {
-                        format!(", awaiting {ongoing_missing_documents} and processed {ongoing_processed_documents} out of {ongoing_total_documents} documents in {ongoing_total_indexes} indexes from `{first_ongoing}`{others}",
-                others=if other_ongoing_count > 0 {format!(" and {other_ongoing_count} other remotes")} else {"".into()})
-                    } else {
-                        "".into()
-                    },
-                    finished = if finished_count >= 0 {
-                        format!(", {finished_count} remotes finished processing")
-                    } else {
-                        "".into()
-                    }
-                )
-            }
-            NetworkTopologyState::Finished => "Finished".into(),
-        };
-        Details::NetworkTopologyChange { moved_documents: self.stats.moved_documents, message }
-    }
-
-    pub fn merge(&mut self, other: NetworkTopologyChange) {
-        // The topology change has a guarantee of forward progress, so for each field we're going to keep the "most advanced" values.
-        let Self { state, new_network: _, old_network: _, in_remotes, stats } = self;
-
-        *state = Ord::max(*state, other.state);
-        *stats = Ord::max(*stats, other.stats);
-
-        for (old_value, new_value) in other.in_remotes.into_values().zip(in_remotes.values_mut()) {
-            new_value.import_state = match (old_value.import_state, std::mem::take(&mut new_value.import_state)) {
-                    // waiting for initial task is always older
-                    (ImportState::WaitingForInitialTask, newer)
-                    | (newer, ImportState::WaitingForInitialTask)
-
-                    // finished is always newer
-                    | (_, newer @ ImportState::Finished { .. })
-                    | (newer @ ImportState::Finished { .. }, _) => newer,
-                    (
-                        ImportState::Ongoing { import_index_state: left_import, total_indexes: left_total_indexes },
-                        ImportState::Ongoing { import_index_state: right_import, total_indexes: right_total_indexes },
-                    ) => {
-                        let import_index_state = left_import.into_iter().merge_join_by(right_import.into_iter(), |(k,_), (x, _)|k.cmp(x)).map(|eob|
-                            match eob {
-                                EitherOrBoth::Both((name, left), (_, right)) => {
-                                    let newer = merge_import_index_state(left, right);
-                                    (name, newer)
-                                },
-                                EitherOrBoth::Left(import) |
-                                EitherOrBoth::Right(import) => import,
-                            }
-                        ).collect();
-
-                        ImportState::Ongoing{ import_index_state, total_indexes : u64::max(left_total_indexes, right_total_indexes) }
-                    },
-                }
-        }
-    }
-
-    pub fn network_for_state(&self) -> &Network {
-        match self.state {
-            NetworkTopologyState::WaitingForOlderTasks => &self.old_network,
-            NetworkTopologyState::ExportingDocuments
-            | NetworkTopologyState::ImportingDocuments
-            | NetworkTopologyState::Finished => &self.new_network,
-        }
-    }
-}
-
-fn merge_import_index_state(left: ImportIndexState, right: ImportIndexState) -> ImportIndexState {
-    match (left, right) {
-        (_, newer @ ImportIndexState::Finished { .. }) => newer,
-        (newer @ ImportIndexState::Finished { .. }, _) => newer,
-        (
-            ImportIndexState::Ongoing {
-                total_documents: left_total_documents,
-                received_documents: left_received_documents,
-                processed_documents: left_processed_documents,
-                task_keys: mut left_task_keys,
-            },
-            ImportIndexState::Ongoing {
-                total_documents: right_total_documents,
-                received_documents: right_received_documents,
-                processed_documents: right_processed_documents,
-                task_keys: right_task_keys,
-            },
-        ) => {
-            let total_documents = u64::max(left_total_documents, right_total_documents);
-            let received_documents = u64::max(left_received_documents, right_received_documents);
-            let processed_documents = u64::max(left_processed_documents, right_processed_documents);
-            left_task_keys.0 |= &right_task_keys.0;
-            let task_keys = left_task_keys;
-
-            ImportIndexState::Ongoing {
-                total_documents,
-                received_documents,
-                processed_documents,
-                task_keys,
-            }
-        }
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, Eq, PartialOrd, Ord)]
-#[serde(rename_all = "camelCase")]
-pub enum NetworkTopologyState {
-    WaitingForOlderTasks,
-    ExportingDocuments,
-    ImportingDocuments,
-    Finished,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, Eq, PartialOrd, Ord)]
-#[serde(rename_all = "camelCase")]
-pub struct NetworkTopologyStats {
-    #[serde(default)]
-    pub moved_documents: u64,
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct InRemote {
-    import_state: ImportState,
-}
-
-impl InRemote {
-    pub fn new() -> Self {
-        Self { import_state: ImportState::WaitingForInitialTask }
-    }
-}
-
-#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-enum ImportState {
-    /// Initially Meilisearch doesn't know how many documents it should expect from a remote.
-    /// Any task from each remote contains the information of how many indexes will be imported,
-    /// and the number of documents to import for the index of the task.
-    #[default]
-    WaitingForInitialTask,
-    Ongoing {
-        import_index_state: BTreeMap<String, ImportIndexState>,
-        total_indexes: u64,
-    },
-    Finished {
-        total_indexes: u64,
-        total_documents: u64,
-    },
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-enum ImportIndexState {
-    Ongoing {
-        total_documents: u64,
-        received_documents: u64,
-        processed_documents: u64,
-        task_keys: TaskKeys,
-    },
-    Finished {
-        total_documents: u64,
-    },
-}
-
-#[derive(Debug, Clone, PartialEq)]
-pub struct TaskKeys(pub RoaringBitmap);
-
-impl Serialize for TaskKeys {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        let TaskKeys(task_keys) = self;
-        let mut bytes = Vec::new();
-        CboRoaringBitmapCodec::serialize_into_vec(task_keys, &mut bytes);
-        let encoded = base64::prelude::BASE64_STANDARD.encode(&bytes);
-        serializer.serialize_str(&encoded)
-    }
-}
-
-impl<'de> Deserialize<'de> for TaskKeys {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: serde::Deserializer<'de>,
-    {
-        deserializer.deserialize_str(TaskKeysVisitor)
-    }
-}
-
-struct TaskKeysVisitor;
-impl<'de> serde::de::Visitor<'de> for TaskKeysVisitor {
-    type Value = TaskKeys;
-
-    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
-        formatter.write_str("a base64 encoded cbo roaring bitmap")
-    }
-
-    fn visit_str<E>(self, encoded: &str) -> Result<Self::Value, E>
-    where
-        E: serde::de::Error,
-    {
-        let decoded = base64::prelude::BASE64_STANDARD.decode(encoded).map_err(|_err| {
-            E::invalid_value(serde::de::Unexpected::Str(encoded), &"a base64 string")
-        })?;
-        self.visit_bytes(&decoded)
-    }
-
-    fn visit_bytes<E>(self, decoded: &[u8]) -> Result<Self::Value, E>
-    where
-        E: serde::de::Error,
-    {
-        let task_keys = CboRoaringBitmapCodec::deserialize_from(decoded).map_err(|_err| {
-            E::invalid_value(serde::de::Unexpected::Bytes(decoded), &"a cbo roaring bitmap")
-        })?;
-        Ok(TaskKeys(task_keys))
-    }
-}
-
-pub enum ReceiveTaskError {
-    UnknownRemote(String),
-    DuplicateTask(DocumentId),
-}
-
-pub mod headers {
-    use std::borrow::Cow;
-    use std::num::ParseIntError;
-    use std::string::FromUtf8Error;
-
-    use milli::DocumentId;
-    use uuid::Uuid;
-
-    use crate::tasks::TaskId;
-
-    /// Implement on response types to extract header values
-    pub trait GetHeader: Sized {
-        type Error: std::fmt::Debug + std::fmt::Display;
-        fn get_header(&self, name: &str) -> Result<Option<&str>, Self::Error>;
-
-        fn get_origin_remote(&self) -> Result<Option<Cow<'_, str>>, DecodeError<Self>> {
-            let Some(encoded) = get_header_and_legacy(self, PROXY_ORIGIN_REMOTE_HEADER)? else {
-                return Ok(None);
-            };
-
-            Ok(Some(urlencoding::decode(encoded).map_err(|inner| DecodeError::UrlDecoding {
-                inner,
-                header: PROXY_ORIGIN_REMOTE_HEADER,
-            })?))
-        }
-
-        fn get_origin_task_uid(&self) -> Result<Option<TaskId>, DecodeError<Self>> {
-            let Some(encoded) = get_header_and_legacy(self, PROXY_ORIGIN_TASK_UID_HEADER)? else {
-                return Ok(None);
-            };
-
-            let decoded = urlencoding::decode(encoded).map_err(|inner| {
-                DecodeError::UrlDecoding { inner, header: PROXY_ORIGIN_TASK_UID_HEADER }
-            })?;
-
-            let parsed = decoded.parse().map_err(|inner| DecodeError::ParseInt {
-                inner,
-                header: PROXY_ORIGIN_TASK_UID_HEADER,
-            })?;
-
-            Ok(Some(parsed))
-        }
-
-        fn get_origin_network_version(&self) -> Result<Option<Uuid>, DecodeError<Self>> {
-            let Some(encoded) = get_header_and_legacy(self, PROXY_ORIGIN_NETWORK_VERSION_HEADER)?
-            else {
-                return Ok(None);
-            };
-
-            let decoded = urlencoding::decode(encoded).map_err(|inner| {
-                DecodeError::UrlDecoding { inner, header: PROXY_ORIGIN_NETWORK_VERSION_HEADER }
-            })?;
-
-            let parsed = decoded.parse().map_err(|inner| DecodeError::ParseUuid {
-                inner,
-                header: PROXY_ORIGIN_NETWORK_VERSION_HEADER,
-            })?;
-
-            Ok(Some(parsed))
-        }
-
-        fn get_import_remote(&self) -> Result<Option<Cow<'_, str>>, DecodeError<Self>> {
-            let Some(encoded) = get_header_and_legacy(self, PROXY_IMPORT_REMOTE_HEADER)? else {
-                return Ok(None);
-            };
-
-            Ok(Some(urlencoding::decode(encoded).map_err(|inner| DecodeError::UrlDecoding {
-                inner,
-                header: PROXY_IMPORT_REMOTE_HEADER,
-            })?))
-        }
-
-        fn get_import_index_count(&self) -> Result<Option<u64>, DecodeError<Self>> {
-            let Some(encoded) = get_header_and_legacy(self, PROXY_IMPORT_INDEX_COUNT_HEADER)?
-            else {
-                return Ok(None);
-            };
-
-            let decoded = urlencoding::decode(encoded).map_err(|inner| {
-                DecodeError::UrlDecoding { inner, header: PROXY_IMPORT_INDEX_COUNT_HEADER }
-            })?;
-
-            let parsed = decoded.parse().map_err(|inner| DecodeError::ParseInt {
-                inner,
-                header: PROXY_IMPORT_INDEX_COUNT_HEADER,
-            })?;
-
-            Ok(Some(parsed))
-        }
-
-        fn get_import_index(&self) -> Result<Option<Cow<'_, str>>, DecodeError<Self>> {
-            let Some(encoded) = get_header_and_legacy(self, PROXY_IMPORT_INDEX_HEADER)? else {
-                return Ok(None);
-            };
-
-            Ok(Some(urlencoding::decode(encoded).map_err(|inner| DecodeError::UrlDecoding {
-                inner,
-                header: PROXY_IMPORT_INDEX_HEADER,
-            })?))
-        }
-
-        fn get_import_task_key(&self) -> Result<Option<DocumentId>, DecodeError<Self>> {
-            let Some(encoded) = get_header_and_legacy(self, PROXY_IMPORT_TASK_KEY_HEADER)? else {
-                return Ok(None);
-            };
-
-            let decoded = urlencoding::decode(encoded).map_err(|inner| {
-                DecodeError::UrlDecoding { inner, header: PROXY_IMPORT_TASK_KEY_HEADER }
-            })?;
-
-            let parsed = decoded.parse().map_err(|inner| DecodeError::ParseInt {
-                inner,
-                header: PROXY_IMPORT_TASK_KEY_HEADER,
-            })?;
-
-            Ok(Some(parsed))
-        }
-
-        fn get_import_docs(&self) -> Result<Option<u64>, DecodeError<Self>> {
-            let Some(encoded) = get_header_and_legacy(self, PROXY_IMPORT_DOCS_HEADER)? else {
-                return Ok(None);
-            };
-
-            let decoded = urlencoding::decode(encoded).map_err(|inner| {
-                DecodeError::UrlDecoding { inner, header: PROXY_IMPORT_DOCS_HEADER }
-            })?;
-
-            let parsed = decoded.parse().map_err(|inner| DecodeError::ParseInt {
-                inner,
-                header: PROXY_IMPORT_DOCS_HEADER,
-            })?;
-
-            Ok(Some(parsed))
-        }
-
-        fn get_import_index_docs(&self) -> Result<Option<u64>, DecodeError<Self>> {
-            let Some(encoded) = get_header_and_legacy(self, PROXY_IMPORT_TOTAL_INDEX_DOCS_HEADER)?
-            else {
-                return Ok(None);
-            };
-
-            let decoded = urlencoding::decode(encoded).map_err(|inner| {
-                DecodeError::UrlDecoding { inner, header: PROXY_IMPORT_TOTAL_INDEX_DOCS_HEADER }
-            })?;
-
-            let parsed = decoded.parse().map_err(|inner| DecodeError::ParseInt {
-                inner,
-                header: PROXY_IMPORT_TOTAL_INDEX_DOCS_HEADER,
-            })?;
-
-            Ok(Some(parsed))
-        }
-    }
-
-    /// Implement on query types to set header values
-    pub trait SetHeader: Sized {
-        fn set_header(self, name: &str, value: &str) -> Self;
-
-        fn set_origin_remote(self, value: &str) -> Self {
-            let encoded = urlencoding::encode(value);
-            set_header_and_legacy(self, PROXY_ORIGIN_REMOTE_HEADER, &encoded)
-        }
-
-        fn set_origin_task_uid(self, value: TaskId) -> Self {
-            let value = value.to_string();
-            let encoded = urlencoding::encode(&value);
-            set_header_and_legacy(self, PROXY_ORIGIN_TASK_UID_HEADER, &encoded)
-        }
-
-        fn set_origin_network_version(self, value: Uuid) -> Self {
-            let value = value.to_string();
-            let encoded = urlencoding::encode(&value);
-            set_header_and_legacy(self, PROXY_ORIGIN_NETWORK_VERSION_HEADER, &encoded)
-        }
-        fn set_import_remote(self, value: &str) -> Self {
-            let encoded = urlencoding::encode(value);
-            set_header_and_legacy(self, PROXY_IMPORT_REMOTE_HEADER, &encoded)
-        }
-
-        fn set_import_index_count(self, value: u64) -> Self {
-            let value = value.to_string();
-            let encoded = urlencoding::encode(&value);
-            set_header_and_legacy(self, PROXY_IMPORT_INDEX_COUNT_HEADER, &encoded)
-        }
-
-        fn set_import_index(self, value: &str) -> Self {
-            let encoded = urlencoding::encode(value);
-            set_header_and_legacy(self, PROXY_IMPORT_INDEX_HEADER, &encoded)
-        }
-
-        fn set_import_task_key(self, value: DocumentId) -> Self {
-            let value = value.to_string();
-            let encoded = urlencoding::encode(&value);
-            set_header_and_legacy(self, PROXY_IMPORT_TASK_KEY_HEADER, &encoded)
-        }
-
-        fn set_import_docs(self, value: u64) -> Self {
-            let value = value.to_string();
-            let encoded = urlencoding::encode(&value);
-            set_header_and_legacy(self, PROXY_IMPORT_DOCS_HEADER, &encoded)
-        }
-
-        fn set_import_index_docs(self, value: u64) -> Self {
-            let value = value.to_string();
-            let encoded = urlencoding::encode(&value);
-            set_header_and_legacy(self, PROXY_IMPORT_TOTAL_INDEX_DOCS_HEADER, &encoded)
-        }
-    }
-
-    #[derive(Debug, thiserror::Error)]
-    pub enum DecodeError<T: GetHeader> {
-        #[error("while getting header: {inner}")]
-        InResponse { inner: T::Error, header: &'static str },
-        #[error("while url-decoding: {inner}")]
-        UrlDecoding { inner: FromUtf8Error, header: &'static str },
-        #[error("while parsing as an integer: {inner}")]
-        ParseInt { inner: ParseIntError, header: &'static str },
-        #[error("while parsing as a UUID: {inner}")]
-        ParseUuid { inner: uuid::Error, header: &'static str },
-    }
-
-    impl<T: GetHeader> DecodeError<T> {
-        pub fn header(&self) -> &'static str {
-            match self {
-                DecodeError::InResponse { inner: _, header }
-                | DecodeError::UrlDecoding { inner: _, header }
-                | DecodeError::ParseInt { inner: _, header }
-                | DecodeError::ParseUuid { inner: _, header } => header,
-            }
-        }
-    }
-
-    pub const PROXY_ORIGIN_REMOTE_HEADER: &str = "X-Meili-Proxy-Origin-Remote";
-    pub const PROXY_ORIGIN_TASK_UID_HEADER: &str = "X-Meili-Proxy-Origin-TaskUid";
-    pub const PROXY_ORIGIN_NETWORK_VERSION_HEADER: &str = "X-Meili-Proxy-Origin-Network-Version";
-    pub const PROXY_IMPORT_REMOTE_HEADER: &str = "X-Meili-Proxy-Import-Remote";
-    pub const PROXY_IMPORT_INDEX_COUNT_HEADER: &str = "X-Meili-Proxy-Import-Index-Count";
-    pub const PROXY_IMPORT_INDEX_HEADER: &str = "X-Meili-Proxy-Import-Index";
-    pub const PROXY_IMPORT_TASK_KEY_HEADER: &str = "X-Meili-Proxy-Import-Task-Key";
-    pub const PROXY_IMPORT_DOCS_HEADER: &str = "X-Meili-Proxy-Import-Docs";
-    pub const PROXY_IMPORT_TOTAL_INDEX_DOCS_HEADER: &str = "X-Meili-Proxy-Import-Total-Index-Docs";
-
-    fn get_header_and_legacy<'a, T: GetHeader>(
-        t: &'a T,
-        header: &'static str,
-    ) -> Result<Option<&'a str>, DecodeError<T>> {
-        Ok(Some(
-            if let Some(encoded) =
-                t.get_header(header).map_err(|inner| DecodeError::InResponse { inner, header })?
-            {
-                encoded
-            } else {
-                let header = header.strip_prefix("X-").unwrap();
-                let Some(encoded) = t
-                    .get_header(header)
-                    .map_err(|inner| DecodeError::InResponse { inner, header })?
-                else {
-                    return Ok(None);
-                };
-                encoded
-            },
-        ))
-    }
-
-    fn set_header_and_legacy<T: SetHeader>(t: T, name: &'static str, value: &str) -> T {
-        let t = t.set_header(name, value);
-        let name = name.strip_prefix("X-").unwrap();
-        t.set_header(name, value)
-    }
-}