WIP

Remove unused dependency `allocator-api2`

.github/dependabot.yml

@@ -7,6 +7,5 @@ updates:
     schedule:
       interval: "monthly"
     labels:
-      - 'skip changelog'
       - 'dependencies'
     rebase-strategy: disabled

.github/pull_request_template.md

@@ -5,12 +5,12 @@ Fixes #...
 ## Requirements
 
 ⚠️ Ensure the following requirements before merging ⚠️
-- [] Automated tests have been added.
-- [] If some tests cannot be automated, manual rigorous tests should be applied.
-- [] ⚠️ If there is any change in the DB: 
+- [ ] Automated tests have been added.
+- [ ] If some tests cannot be automated, manual rigorous tests should be applied.
+- [ ] ⚠️ If there is any change in the DB: 
     - [ ] Test that any impacted DB still works as expected after using `--experimental-dumpless-upgrade` on a DB created with the last released Meilisearch
     - [ ] Test that during the upgrade, **search is still available** (artificially make the upgrade longer if needed)
     - [ ] Set the `db change` label.
-- [] If necessary, the feature have been tested in the Cloud production environment (with [prototypes](./documentation/prototypes.md)) and the Cloud UI is ready.
-- [] If necessary, the [documentation](https://github.com/meilisearch/documentation) related to the implemented feature in the PR is ready.
-- [] If necessary, the [integrations](https://github.com/meilisearch/integration-guides) related to the implemented feature in the PR are ready.
+- [ ] If necessary, the feature have been tested in the Cloud production environment (with [prototypes](./documentation/prototypes.md)) and the Cloud UI is ready.
+- [ ] If necessary, the [documentation](https://github.com/meilisearch/documentation) related to the implemented feature in the PR is ready.
+- [ ] If necessary, the [integrations](https://github.com/meilisearch/integration-guides) related to the implemented feature in the PR are ready.

.github/release-draft-template.yml

@@ -1,23 +0,0 @@
-name-template: 'v$RESOLVED_VERSION'
-tag-template: 'v$RESOLVED_VERSION'
-exclude-labels:
-  - 'skip changelog'
-version-resolver:
-  major:
-    labels:
-      - 'breaking-change'
-  minor:
-    labels:
-      - 'enhancement'
-  default: patch
-template: |
-  $CHANGES
-
-  Thanks again to $CONTRIBUTORS! 🎉
-no-changes-template: 'Changes are coming soon 😎'
-sort-direction: 'ascending'
-replacers:
-  - search: '/(?:and )?@dependabot-preview(?:\[bot\])?,?/g'
-    replace: ''
-  - search: '/(?:and )?@dependabot(?:\[bot\])?,?/g'
-    replace: ''

.github/workflows/bench-manual.yml

@@ -17,8 +17,8 @@ jobs:
     runs-on: benchmarks
     timeout-minutes: 180 # 3h
     steps:
-      - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: actions/checkout@v5
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
 

.github/workflows/bench-pr.yml

@@ -60,13 +60,13 @@ jobs:
         with:
           repo_token: ${{ env.GH_TOKEN }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         if: success()
         with:
           fetch-depth: 0 # fetch full history to be able to get main commit sha
           ref: ${{ steps.comment-branch.outputs.head_ref }}
 
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
 

.github/workflows/bench-push-indexing.yml

@@ -11,8 +11,8 @@ jobs:
     runs-on: benchmarks
     timeout-minutes: 180 # 3h
     steps:
-      - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: actions/checkout@v5
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
 

.github/workflows/benchmarks-manual.yml

@@ -17,8 +17,8 @@ jobs:
     runs-on: benchmarks
     timeout-minutes: 4320 # 72h
     steps:
-      - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: actions/checkout@v5
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
 

.github/workflows/benchmarks-pr.yml

@@ -44,7 +44,7 @@ jobs:
             exit 1
           fi
 
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
 
@@ -61,7 +61,7 @@ jobs:
         with:
           repo_token: ${{ env.GH_TOKEN }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         if: success()
         with:
           fetch-depth: 0 # fetch full history to be able to get main commit sha

.github/workflows/benchmarks-push-indexing.yml

@@ -15,8 +15,8 @@ jobs:
     runs-on: benchmarks
     timeout-minutes: 4320 # 72h
     steps:
-      - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: actions/checkout@v5
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
 

.github/workflows/benchmarks-push-search-geo.yml

@@ -14,8 +14,8 @@ jobs:
     name: Run and upload benchmarks
     runs-on: benchmarks
     steps:
-      - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: actions/checkout@v5
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
 

.github/workflows/benchmarks-push-search-songs.yml

@@ -14,8 +14,8 @@ jobs:
     name: Run and upload benchmarks
     runs-on: benchmarks
     steps:
-      - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: actions/checkout@v5
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
 

.github/workflows/benchmarks-push-search-wiki.yml

@@ -14,8 +14,8 @@ jobs:
     name: Run and upload benchmarks
     runs-on: benchmarks
     steps:
-      - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: actions/checkout@v5
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
 

.github/workflows/db-change-missing.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - name: Check db change labels
         id: check_labels
         env:

.github/workflows/dependency-issue.yml

@@ -13,7 +13,7 @@ jobs:
       ISSUE_TEMPLATE: issue-template.md
       GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v5
     - name: Download the issue template
       run: curl -s https://raw.githubusercontent.com/meilisearch/meilisearch/main/.github/templates/dependency-issue.md > $ISSUE_TEMPLATE
     - name: Create issue

.github/workflows/flaky-tests.yml

@@ -12,12 +12,12 @@ jobs:
       # Use ubuntu-22.04 to compile with glibc 2.35
       image: ubuntu:22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
       - name: Install needed dependencies
         run: |
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: dtolnay/rust-toolchain@1.89
       - name: Install cargo-flaky
         run: cargo install cargo-flaky
       - name: Run cargo flaky in the dumps

.github/workflows/fuzzer-indexing.yml

@@ -11,8 +11,8 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 4320 # 72h
     steps:
-      - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: actions/checkout@v5
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
 

.github/workflows/latest-git-tag.yml

@@ -10,7 +10,7 @@ jobs:
     name: Check the version validity
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
       - name: Check release validity
         if: github.event_name == 'release'
         run: bash .github/scripts/check-release.sh
@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: check-version
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
       - uses: rickstaa/action-create-tag@v1
         with:
           tag: "latest"

.github/workflows/publish-apt-brew-pkg.yml

@@ -9,7 +9,7 @@ jobs:
     name: Check the version validity
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
       - name: Check release validity
         run: bash .github/scripts/check-release.sh
 
@@ -25,10 +25,10 @@ jobs:
         run: |
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: dtolnay/rust-toolchain@1.89
       - name: Install cargo-deb
         run: cargo install cargo-deb
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
       - name: Build deb package
         run: cargo deb -p meilisearch -o target/debian/meilisearch.deb
       - name: Upload debian pkg to release

.github/workflows/publish-docker-images.yml

@@ -19,7 +19,7 @@ jobs:
     permissions:
       id-token: write # This is needed to use Cosign in keyless mode
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
 
       # If we are running a cron or manual job ('schedule' or 'workflow_dispatch' event), it means we are publishing the `nightly` tag, so not considered stable.
       # If we have pushed a tag, and the tag has the v<nmumber>.<number>.<number> format, it means we are publishing an official release, so considered stable.
@@ -65,7 +65,7 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: Install cosign
-        uses: sigstore/cosign-installer@d58896d6a1865668819e1d91763c7751a165e159 # tag=v3.9.2
+        uses: sigstore/cosign-installer@d7543c93d881b35a8faa02e8e3605f69b7a1ce62 # tag=v3.10.0
 
       - name: Login to Docker Hub
         uses: docker/login-action@v3

.github/workflows/publish-release-assets.yml

@@ -1,4 +1,4 @@
-name: Publish binaries to GitHub release
+name: Publish assets to GitHub release
 
 on:
   workflow_dispatch:
@@ -11,9 +11,9 @@ jobs:
   check-version:
     name: Check the version validity
     runs-on: ubuntu-latest
-    # No need to check the version for dry run (cron)
+    # No need to check the version for dry run (cron or workflow_dispatch)
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
       # Check if the tag has the v<nmumber>.<number>.<number> format.
       # If yes, it means we are publishing an official release.
       # If no, we are releasing a RC, so no need to check the version.
@@ -40,15 +40,15 @@ jobs:
       # Use ubuntu-22.04 to compile with glibc 2.35
       image: ubuntu:22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
       - name: Install needed dependencies
         run: |
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: dtolnay/rust-toolchain@1.89
       - name: Build
         run: cargo build --release --locked
-      # No need to upload binaries for dry run (cron)
+      # No need to upload binaries for dry run (cron or workflow_dispatch)
       - name: Upload binaries to release
         if: github.event_name == 'release'
         uses: svenstaro/upload-release-action@2.11.2
@@ -74,11 +74,11 @@ jobs:
             artifact_name: meilisearch.exe
             asset_name: meilisearch-windows-amd64.exe
     steps:
-      - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: actions/checkout@v5
+      - uses: dtolnay/rust-toolchain@1.89
       - name: Build
         run: cargo build --release --locked
-      # No need to upload binaries for dry run (cron)
+      # No need to upload binaries for dry run (cron or workflow_dispatch)
       - name: Upload binaries to release
         if: github.event_name == 'release'
         uses: svenstaro/upload-release-action@2.11.2
@@ -99,9 +99,9 @@ jobs:
             asset_name: meilisearch-macos-apple-silicon
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
       - name: Installing Rust toolchain
-        uses: dtolnay/rust-toolchain@1.85
+        uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
           target: ${{ matrix.target }}
@@ -111,7 +111,7 @@ jobs:
           command: build
           args: --release --target ${{ matrix.target }}
       - name: Upload the binary to release
-        # No need to upload binaries for dry run (cron)
+        # No need to upload binaries for dry run (cron or workflow_dispatch)
         if: github.event_name == 'release'
         uses: svenstaro/upload-release-action@2.11.2
         with:
@@ -136,7 +136,7 @@ jobs:
             asset_name: meilisearch-linux-aarch64
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
       - name: Install needed dependencies
         run: |
           apt-get update -y && apt upgrade -y
@@ -148,7 +148,7 @@ jobs:
           add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
           apt-get update -y && apt-get install -y docker-ce
       - name: Installing Rust toolchain
-        uses: dtolnay/rust-toolchain@1.85
+        uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
           target: ${{ matrix.target }}
@@ -176,7 +176,7 @@ jobs:
       - name: List target output files
         run: ls -lR ./target
       - name: Upload the binary to release
-        # No need to upload binaries for dry run (cron)
+        # No need to upload binaries for dry run (cron or workflow_dispatch)
         if: github.event_name == 'release'
         uses: svenstaro/upload-release-action@2.11.2
         with:
@@ -184,3 +184,29 @@ jobs:
           file: target/${{ matrix.target }}/release/meilisearch
           asset_name: ${{ matrix.asset_name }}
           tag: ${{ github.ref }}
+
+  publish-openapi-file:
+    name: Publish OpenAPI file
+    needs: check-version
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+      - name: Setup Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+      - name: Generate OpenAPI file
+        run: |
+          cd crates/openapi-generator
+          cargo run --release -- --pretty --output ../../meilisearch.json
+      - name: Upload OpenAPI to Release
+        # No need to upload for dry run (cron or workflow_dispatch)
+        if: github.event_name == 'release'
+        uses: svenstaro/upload-release-action@2.11.2
+        with:
+          repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
+          file: ./meilisearch.json
+          asset_name: meilisearch-openapi.json
+          tag: ${{ github.ref }}

.github/workflows/release-drafter.yml

@@ -1,20 +0,0 @@
-name: Release Drafter
-
-permissions:
-  contents: read
-  pull-requests: write
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  update_release_draft:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: release-drafter/release-drafter@v6
-        with:
-          config-name: release-draft-template.yml
-        env:
-          GITHUB_TOKEN: ${{ secrets.RELEASE_DRAFTER_TOKEN }}

.github/workflows/sdks-tests.yml

@@ -22,7 +22,7 @@ jobs:
     outputs:
       docker-image: ${{ steps.define-image.outputs.docker-image }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
       - name: Define the Docker image we need to use
         id: define-image
         run: |
@@ -46,11 +46,11 @@ jobs:
       MEILISEARCH_VERSION: ${{ needs.define-docker-image.outputs.docker-image }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         with:
           repository: meilisearch/meilisearch-dotnet
       - name: Setup .NET Core
-        uses: actions/setup-dotnet@v4
+        uses: actions/setup-dotnet@v5
         with:
           dotnet-version: "8.0.x"
       - name: Install dependencies
@@ -75,7 +75,7 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         with:
           repository: meilisearch/meilisearch-dart
       - uses: dart-lang/setup-dart@v1
@@ -100,10 +100,10 @@ jobs:
           - '7700:7700'
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: stable
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         with:
           repository: meilisearch/meilisearch-go
       - name: Get dependencies
@@ -129,19 +129,19 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         with:
           repository: meilisearch/meilisearch-java
       - name: Set up Java
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
-          java-version: 8
-          distribution: 'zulu'
+          java-version: 17
+          distribution: 'temurin'
           cache: gradle
       - name: Grant execute permission for gradlew
         run: chmod +x gradlew
       - name: Build and run unit and integration tests
-        run: ./gradlew build integrationTest
+        run: ./gradlew build integrationTest --info
 
   meilisearch-js-tests:
     needs: define-docker-image
@@ -156,11 +156,11 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         with:
           repository: meilisearch/meilisearch-js
       - name: Setup node
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v5
         with:
           cache: 'yarn'
       - name: Install dependencies
@@ -191,7 +191,7 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         with:
           repository: meilisearch/meilisearch-php
       - name: Install PHP
@@ -220,11 +220,11 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         with:
           repository: meilisearch/meilisearch-python
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
       - name: Install pipenv
         uses: dschep/install-pipenv-action@v1
       - name: Install dependencies
@@ -245,7 +245,7 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         with:
           repository: meilisearch/meilisearch-ruby
       - name: Set up Ruby 3
@@ -270,7 +270,7 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         with:
           repository: meilisearch/meilisearch-rust
       - name: Build
@@ -291,7 +291,7 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         with:
           repository: meilisearch/meilisearch-swift
       - name: Run tests
@@ -314,11 +314,11 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         with:
           repository: meilisearch/meilisearch-js-plugins
       - name: Setup node
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v5
         with:
           cache: yarn
       - name: Install dependencies
@@ -347,7 +347,7 @@ jobs:
     env:
       RAILS_VERSION: '7.0'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         with:
           repository: meilisearch/meilisearch-rails
       - name: Install SQLite dependencies
@@ -377,7 +377,7 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         with:
           repository: meilisearch/meilisearch-symfony
       - name: Install PHP

.github/workflows/test-suite.yml

@@ -21,13 +21,13 @@ jobs:
       # Use ubuntu-22.04 to compile with glibc 2.35
       image: ubuntu:22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
       - name: Install needed dependencies
         run: |
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
       - name: Setup test with Rust stable
-        uses: dtolnay/rust-toolchain@1.85
+        uses: dtolnay/rust-toolchain@1.89
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.8.0
       - name: Run cargo check without any default features
@@ -49,10 +49,10 @@ jobs:
       matrix:
         os: [macos-13, windows-2022]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.8.0
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: dtolnay/rust-toolchain@1.89
       - name: Run cargo check without any default features
         uses: actions-rs/cargo@v1
         with:
@@ -72,12 +72,12 @@ jobs:
       image: ubuntu:22.04
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
       - name: Install needed dependencies
         run: |
           apt-get update
           apt-get install --assume-yes build-essential curl
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: dtolnay/rust-toolchain@1.89
       - name: Run cargo build with almost all features
         run: |
           cargo build --workspace --locked --release --features "$(cargo xtask list-features --exclude-feature cuda,test-ollama)"
@@ -91,7 +91,7 @@ jobs:
     env:
       MEILI_TEST_OLLAMA_SERVER: "http://localhost:11434"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
       - name: Install Ollama
         run: |
           curl -fsSL https://ollama.com/install.sh | sudo -E sh
@@ -124,12 +124,12 @@ jobs:
       image: ubuntu:22.04
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
       - name: Install needed dependencies
         run: |
           apt-get update
           apt-get install --assume-yes build-essential curl
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: dtolnay/rust-toolchain@1.89
       - name: Run cargo tree without default features and check lindera is not present
         run: |
           if cargo tree -f '{p} {f}' -e normal --no-default-features | grep -qz lindera; then
@@ -148,12 +148,12 @@ jobs:
       # Use ubuntu-22.04 to compile with glibc 2.35
       image: ubuntu:22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
       - name: Install needed dependencies
         run: |
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: dtolnay/rust-toolchain@1.89
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.8.0
       - name: Run tests in debug
@@ -166,8 +166,8 @@ jobs:
     name: Run Clippy
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: actions/checkout@v5
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
           components: clippy
@@ -183,8 +183,8 @@ jobs:
     name: Run Rustfmt
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: actions/checkout@v5
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
           toolchain: nightly-2024-07-09

.github/workflows/update-cargo-toml-version.yml

@@ -17,8 +17,8 @@ jobs:
     name: Update version in Cargo.toml
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.85
+      - uses: actions/checkout@v5
+      - uses: dtolnay/rust-toolchain@1.89
         with:
           profile: minimal
       - name: Install sd
@@ -41,5 +41,4 @@ jobs:
             --title "Update version for the next release ($NEW_VERSION) in Cargo.toml" \
             --body '⚠️ This PR is automatically generated. Check the new version is the expected one and Cargo.lock has been updated before merging.' \
             --label 'skip changelog' \
-            --milestone $NEW_VERSION \
             --base $GITHUB_REF_NAME

CONTRIBUTING.md

@@ -107,12 +107,18 @@ Run `cargo xtask --help` from the root of the repository to find out what is ava
 
 To update the openAPI file in the code, see [sprint_issue.md](https://github.com/meilisearch/meilisearch/blob/main/.github/ISSUE_TEMPLATE/sprint_issue.md#reminders-when-modifying-the-api).
 
-If you want to update the openAPI file on the [open-api repository](https://github.com/meilisearch/open-api):
-- Pull the latest version of the latest rc of Meilisearch `git checkout release-vX.Y.Z; git pull`
+If you want to generate OpenAPI file manually:
+
+With swagger:
 - Starts Meilisearch with the `swagger` feature flag: `cargo run --features swagger`
 - On a browser, open the following URL: http://localhost:7700/scalar
 - Click the « Download openAPI file »
-- Open a PR replacing [this file](https://github.com/meilisearch/open-api/blob/main/open-api.json) with the one downloaded
+
+With the internal crate:
+```bash
+cd crates/openapi-generator
+cargo run --release -- --pretty --output meilisearch.json
+```
 
 ### Logging
 

Cargo.toml

@@ -19,10 +19,11 @@ members = [
     "crates/tracing-trace",
     "crates/xtask",
     "crates/build-info",
+    "crates/openapi-generator",
 ]
 
 [workspace.package]
-version = "1.16.0"
+version = "1.26.0"
 authors = [
     "Quentin de Quelen <quentin@dequelen.me>",
     "Clément Renault <clement@meilisearch.com>",

Dockerfile

@@ -1,5 +1,5 @@
 # Compile
-FROM    rust:1.85-alpine3.20 AS compiler
+FROM    rust:1.89-alpine3.22 AS compiler
 
 RUN     apk add -q --no-cache build-base openssl-dev
 
@@ -20,7 +20,7 @@ RUN     set -eux; \
         cargo build --release -p meilisearch -p meilitool
 
 # Run
-FROM    alpine:3.20
+FROM    alpine:3.22
 LABEL   org.opencontainers.image.source="https://github.com/meilisearch/meilisearch"
 
 ENV     MEILI_HTTP_ADDR 0.0.0.0:7700

LICENSE

@@ -1,21 +1,9 @@
-MIT License
+# License
 
 Copyright (c) 2019-2025 Meili SAS
 
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+Part of this work fall under the Meilisearch Enterprise Edition (EE) and are licensed under the Business Source License 1.1, please refer to [LICENSE-EE](./LICENSE-EE) for details.
 
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
+The other parts of this work are licensed under the [MIT license](./LICENSE-MIT).
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+`SPDX-License-Identifier: MIT AND BUSL-1.1`

LICENSE-EE

@@ -0,0 +1,67 @@
+Business Source License 1.1 – Adapted for Meili SAS
+This license is based on the Business Source License version 1.1, as published by MariaDB Corporation Ab.
+
+Parameters
+
+Licensor: Meili SAS
+
+Licensed Work: Any file explicitly marked as “Enterprise Edition (EE)” or “governed by the Business Source License” residing in enterprise_editions modules/folders.
+
+Additional Use Grant:
+You may use, modify, and distribute the Licensed Work for non-production purposes only, such as testing, development, or evaluation.
+
+Production use of the Licensed Work requires a commercial license agreement with Meilisearch. Contact bonjour@meilisearch.com for licensing.
+
+Change License: MIT
+
+Change Date: Four years from the date the Licensed Work is published.
+
+This License does not apply to any code outside of the Licensed Work, which remains under the MIT license.
+
+For information about alternative licensing arrangements for the Licensed Work,
+please contact bonjour@meilisearch.com or sales@meilisearch.com.
+
+Notice
+
+Business Source License 1.1
+
+Terms
+
+The Licensor hereby grants you the right to copy, modify, create derivative
+works, redistribute, and make non-production use of the Licensed Work. The
+Licensor may make an Additional Use Grant, above, permitting limited production use.
+
+Effective on the Change Date, or the fourth anniversary of the first publicly
+available distribution of a specific version of the Licensed Work under this
+License, whichever comes first, the Licensor hereby grants you rights under
+the terms of the Change License, and the rights granted in the paragraph
+above terminate.
+
+If your use of the Licensed Work does not comply with the requirements
+currently in effect as described in this License, you must purchase a
+commercial license from the Licensor, its affiliated entities, or authorized
+resellers, or you must refrain from using the Licensed Work.
+
+All copies of the original and modified Licensed Work, and derivative works
+of the Licensed Work, are subject to this License. This License applies
+separately for each version of the Licensed Work and the Change Date may vary
+for each version of the Licensed Work released by Licensor.
+
+You must conspicuously display this License on each original or modified copy
+of the Licensed Work. If you receive the Licensed Work in original or
+modified form from a third party, the terms and conditions set forth in this
+License apply to your use of that work.
+
+Any use of the Licensed Work in violation of this License will automatically
+terminate your rights under this License for the current and all other
+versions of the Licensed Work.
+
+This License does not grant you any right in any trademark or logo of
+Licensor or its affiliates (provided that you may use a trademark or logo of
+Licensor as expressly required by this License).
+
+TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
+AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
+EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
+TITLE.

LICENSE-MIT

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019-2025 Meili SAS
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -39,6 +39,7 @@
 ## 🖥 Examples
 
 - [**Movies**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=organization) — An application to help you find streaming platforms to watch movies using [hybrid search](https://www.meilisearch.com/solutions/hybrid-search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos).
+- [**Flickr**](https://flickr.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=organization) — Search and explore one hundred million Flickr images with semantic search.
 - [**Ecommerce**](https://ecommerce.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Ecommerce website using disjunctive [facets](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos), range and rating filtering, and pagination.
 - [**Songs**](https://music.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search through 47 million of songs.
 - [**SaaS**](https://saas.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search for contacts, deals, and companies in this [multi-tenant](https://www.meilisearch.com/docs/learn/security/multitenancy_tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) CRM application.
@@ -89,6 +90,26 @@ We also offer a wide range of dedicated guides to all Meilisearch features, such
 
 Finally, for more in-depth information, refer to our articles explaining fundamental Meilisearch concepts such as [documents](https://www.meilisearch.com/docs/learn/core_concepts/documents?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced) and [indexes](https://www.meilisearch.com/docs/learn/core_concepts/indexes?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced).
 
+## 🧾 Editions & Licensing
+
+Meilisearch is available in two editions:
+
+### 🧪 Community Edition (CE)
+
+- Fully open source under the [MIT license](./LICENSE)
+- Core search engine with fast and relevant full-text, semantic or hybrid search
+- Free to use for anyone, including commercial usage
+
+### 🏢 Enterprise Edition (EE)
+
+- Includes advanced features such as:
+  - Sharding
+- Governed by a [commercial license](./LICENSE-EE) or the [Business Source License 1.1](https://mariadb.com/bsl11)
+- Not allowed in production without a commercial agreement with Meilisearch.
+  - You may use, modify, and distribute the Licensed Work for non-production purposes only, such as testing, development, or evaluation.
+
+Want access to Enterprise features? → Contact us at [sales@meilisearch.com](maito:sales@meilisearch.com).
+
 ## 📊 Telemetry
 
 Meilisearch collects **anonymized** user data to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) whenever you want.
@@ -101,7 +122,7 @@ If you want to know more about the kind of data we collect and what we use it fo
 
 Meilisearch is a search engine created by [Meili](https://www.meilisearch.com/careers), a software development company headquartered in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact)
 
-🗞 [Subscribe to our newsletter](https://meilisearch.us2.list-manage.com/subscribe?u=27870f7b71c908a8b359599fb&id=79582d828e) if you don't want to miss any updates! We promise we won't clutter your mailbox: we only send one edition every two months.
+🗞 [Subscribe to our newsletter](https://share-eu1.hsforms.com/1LN5N0x_GQgq7ss7tXmSykwfg3aq) if you don't want to miss any updates! We promise we won't clutter your mailbox: we only send one edition every two months.
 
 💌 Want to make a suggestion or give feedback? Here are some of the channels where you can reach us:
 

crates/benchmarks/Cargo.toml

@@ -55,3 +55,7 @@ harness = false
 [[bench]]
 name = "sort"
 harness = false
+
+[[bench]]
+name = "filter_starts_with"
+harness = false

crates/benchmarks/benches/filter_starts_with.rs

@@ -0,0 +1,66 @@
+mod datasets_paths;
+mod utils;
+
+use criterion::{criterion_group, criterion_main};
+use milli::update::Settings;
+use milli::FilterableAttributesRule;
+use utils::Conf;
+
+#[cfg(not(windows))]
+#[global_allocator]
+static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
+
+fn base_conf(builder: &mut Settings) {
+    let displayed_fields = ["geonameid", "name"].iter().map(|s| s.to_string()).collect();
+    builder.set_displayed_fields(displayed_fields);
+
+    let filterable_fields =
+        ["name"].iter().map(|s| FilterableAttributesRule::Field(s.to_string())).collect();
+    builder.set_filterable_fields(filterable_fields);
+}
+
+#[rustfmt::skip]
+const BASE_CONF: Conf = Conf {
+    dataset: datasets_paths::SMOL_ALL_COUNTRIES,
+    dataset_format: "jsonl",
+    queries: &[
+        "",
+    ],
+    configure: base_conf,
+    primary_key: Some("geonameid"),
+    ..Conf::BASE
+};
+
+fn filter_starts_with(c: &mut criterion::Criterion) {
+    #[rustfmt::skip]
+    let confs = &[
+        utils::Conf {
+            group_name: "1 letter",
+            filter: Some("name STARTS WITH e"),
+            ..BASE_CONF
+        },
+
+        utils::Conf {
+            group_name: "2 letters",
+            filter: Some("name STARTS WITH es"),
+            ..BASE_CONF
+        },
+
+        utils::Conf {
+            group_name: "3 letters",
+            filter: Some("name STARTS WITH est"),
+            ..BASE_CONF
+        },
+
+        utils::Conf {
+            group_name: "6 letters",
+            filter: Some("name STARTS WITH estoni"),
+            ..BASE_CONF
+        }
+    ];
+
+    utils::run_benches(c, confs);
+}
+
+criterion_group!(benches, filter_starts_with);
+criterion_main!(benches);

crates/benchmarks/benches/indexing.rs

@@ -154,6 +154,7 @@ fn indexing_songs_default(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -221,6 +222,7 @@ fn reindexing_songs_default(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -266,6 +268,7 @@ fn reindexing_songs_default(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -335,6 +338,7 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -412,6 +416,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -457,6 +462,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -498,6 +504,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -566,6 +573,7 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -633,6 +641,7 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -700,6 +709,7 @@ fn indexing_wiki(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -766,6 +776,7 @@ fn reindexing_wiki(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -811,6 +822,7 @@ fn reindexing_wiki(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -879,6 +891,7 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -956,6 +969,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -1002,6 +1016,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -1044,6 +1059,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -1111,6 +1127,7 @@ fn indexing_movies_default(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -1177,6 +1194,7 @@ fn reindexing_movies_default(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -1222,6 +1240,7 @@ fn reindexing_movies_default(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -1290,6 +1309,7 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -1404,6 +1424,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -1449,6 +1470,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -1490,6 +1512,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -1580,6 +1603,7 @@ fn indexing_nested_movies_default(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -1671,6 +1695,7 @@ fn deleting_nested_movies_in_batches_default(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -1754,6 +1779,7 @@ fn indexing_nested_movies_without_faceted_fields(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -1821,6 +1847,7 @@ fn indexing_geo(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -1887,6 +1914,7 @@ fn reindexing_geo(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -1932,6 +1960,7 @@ fn reindexing_geo(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 
@@ -2000,6 +2029,7 @@ fn deleting_geo_in_batches_default(c: &mut Criterion) {
                         &mut new_fields_ids_map,
                         &|| false,
                         Progress::default(),
+                        None,
                     )
                     .unwrap();
 

crates/benchmarks/benches/utils.rs

@@ -123,6 +123,7 @@ pub fn base_setup(conf: &Conf) -> Index {
             &mut new_fields_ids_map,
             &|| false,
             Progress::default(),
+            None,
         )
         .unwrap();
 

crates/dump/src/lib.rs

@@ -9,6 +9,7 @@ use meilisearch_types::error::ResponseError;
 use meilisearch_types::keys::Key;
 use meilisearch_types::milli::update::IndexDocumentsMethod;
 use meilisearch_types::settings::Unchecked;
+use meilisearch_types::tasks::enterprise_edition::network::{NetworkTopologyChange, DbTaskNetwork};
 use meilisearch_types::tasks::{
     Details, ExportIndexSettings, IndexSwap, KindWithContent, Status, Task, TaskId,
 };
@@ -94,6 +95,10 @@ pub struct TaskDump {
         default
     )]
     pub finished_at: Option<OffsetDateTime>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub network: Option<DbTaskNetwork>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub custom_metadata: Option<String>,
 }
 
 // A `Kind` specific version made for the dump. If modified you may break the dump.
@@ -129,6 +134,7 @@ pub enum KindDump {
     },
     IndexUpdate {
         primary_key: Option<String>,
+        uid: Option<String>,
     },
     IndexSwap {
         swaps: Vec<IndexSwap>,
@@ -155,6 +161,10 @@ pub enum KindDump {
     UpgradeDatabase {
         from: (u32, u32, u32),
     },
+    IndexCompaction {
+        index_uid: String,
+    },
+    NetworkTopologyChange(NetworkTopologyChange),
 }
 
 impl From<Task> for TaskDump {
@@ -171,6 +181,8 @@ impl From<Task> for TaskDump {
             enqueued_at: task.enqueued_at,
             started_at: task.started_at,
             finished_at: task.finished_at,
+            network: task.network,
+            custom_metadata: task.custom_metadata,
         }
     }
 }
@@ -210,8 +222,8 @@ impl From<KindWithContent> for KindDump {
             KindWithContent::IndexCreation { primary_key, .. } => {
                 KindDump::IndexCreation { primary_key }
             }
-            KindWithContent::IndexUpdate { primary_key, .. } => {
-                KindDump::IndexUpdate { primary_key }
+            KindWithContent::IndexUpdate { primary_key, new_index_uid: uid, .. } => {
+                KindDump::IndexUpdate { primary_key, uid }
             }
             KindWithContent::IndexSwap { swaps } => KindDump::IndexSwap { swaps },
             KindWithContent::TaskCancelation { query, tasks } => {
@@ -236,6 +248,12 @@ impl From<KindWithContent> for KindDump {
             KindWithContent::UpgradeDatabase { from: version } => {
                 KindDump::UpgradeDatabase { from: version }
             }
+            KindWithContent::IndexCompaction { index_uid } => {
+                KindDump::IndexCompaction { index_uid }
+            }
+            KindWithContent::NetworkTopologyChange(network_topology_change) => {
+                KindDump::NetworkTopologyChange(network_topology_change)
+            }
         }
     }
 }
@@ -249,8 +267,9 @@ pub(crate) mod test {
     use big_s::S;
     use maplit::{btreemap, btreeset};
     use meilisearch_types::batches::{Batch, BatchEnqueuedAt, BatchStats};
+    use meilisearch_types::enterprise_edition::network::{Network, Remote};
     use meilisearch_types::facet_values_sort::FacetValuesSort;
-    use meilisearch_types::features::{Network, Remote, RuntimeTogglableFeatures};
+    use meilisearch_types::features::RuntimeTogglableFeatures;
     use meilisearch_types::index_uid_pattern::IndexUidPattern;
     use meilisearch_types::keys::{Action, Key};
     use meilisearch_types::milli::update::Setting;
@@ -326,6 +345,7 @@ pub(crate) mod test {
             facet_search: Setting::NotSet,
             prefix_search: Setting::NotSet,
             chat: Setting::NotSet,
+            vector_store: Setting::NotSet,
             _kind: std::marker::PhantomData,
         };
         settings.check()
@@ -383,6 +403,8 @@ pub(crate) mod test {
                     enqueued_at: datetime!(2022-11-11 0:00 UTC),
                     started_at: Some(datetime!(2022-11-20 0:00 UTC)),
                     finished_at: Some(datetime!(2022-11-21 0:00 UTC)),
+                    network: None,
+                    custom_metadata: None,
                 },
                 None,
             ),
@@ -407,6 +429,8 @@ pub(crate) mod test {
                     enqueued_at: datetime!(2022-11-11 0:00 UTC),
                     started_at: None,
                     finished_at: None,
+                    network: None,
+                    custom_metadata: None,
                 },
                 Some(vec![
                     json!({ "id": 4, "race": "leonberg" }).as_object().unwrap().clone(),
@@ -426,6 +450,8 @@ pub(crate) mod test {
                     enqueued_at: datetime!(2022-11-15 0:00 UTC),
                     started_at: None,
                     finished_at: None,
+                    network: None,
+                    custom_metadata: None,
                 },
                 None,
             ),
@@ -538,7 +564,9 @@ pub(crate) mod test {
     fn create_test_network() -> Network {
         Network {
             local: Some("myself".to_string()),
-            remotes: maplit::btreemap! {"other".to_string() => Remote { url: "http://test".to_string(), search_api_key: Some("apiKey".to_string()) }},
+            remotes: maplit::btreemap! {"other".to_string() => Remote { url: "http://test".to_string(), search_api_key: Some("apiKey".to_string()), write_api_key: Some("docApiKey".to_string()) }},
+            leader: None,
+            version: Default::default(),
         }
     }
 

crates/dump/src/reader/compat/v2_to_v3.rs

@@ -97,6 +97,7 @@ impl CompatV2ToV3 {
     }
 }
 
+#[allow(clippy::large_enum_variant)]
 pub enum CompatIndexV2ToV3 {
     V2(v2::V2IndexReader),
     Compat(Box<CompatIndexV1ToV2>),

crates/dump/src/reader/compat/v5_to_v6.rs

@@ -85,7 +85,7 @@ impl CompatV5ToV6 {
                             v6::Kind::IndexCreation { primary_key }
                         }
                         v5::tasks::TaskContent::IndexUpdate { primary_key, .. } => {
-                            v6::Kind::IndexUpdate { primary_key }
+                            v6::Kind::IndexUpdate { primary_key, uid: None }
                         }
                         v5::tasks::TaskContent::IndexDeletion { .. } => v6::Kind::IndexDeletion,
                         v5::tasks::TaskContent::DocumentAddition {
@@ -140,9 +140,11 @@ impl CompatV5ToV6 {
                         v5::Details::Settings { settings } => {
                             v6::Details::SettingsUpdate { settings: Box::new(settings.into()) }
                         }
-                        v5::Details::IndexInfo { primary_key } => {
-                            v6::Details::IndexInfo { primary_key }
-                        }
+                        v5::Details::IndexInfo { primary_key } => v6::Details::IndexInfo {
+                            primary_key,
+                            new_index_uid: None,
+                            old_index_uid: None,
+                        },
                         v5::Details::DocumentDeletion {
                             received_document_ids,
                             deleted_documents,
@@ -161,6 +163,8 @@ impl CompatV5ToV6 {
                     enqueued_at: task_view.enqueued_at,
                     started_at: task_view.started_at,
                     finished_at: task_view.finished_at,
+                    network: None,
+                    custom_metadata: None,
                 };
 
                 (task, content_file)
@@ -418,6 +422,7 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
             facet_search: v6::Setting::NotSet,
             prefix_search: v6::Setting::NotSet,
             chat: v6::Setting::NotSet,
+            vector_store: v6::Setting::NotSet,
             _kind: std::marker::PhantomData,
         }
     }

crates/dump/src/reader/v6/mod.rs

@@ -4,7 +4,7 @@ use std::io::{BufRead, BufReader, ErrorKind};
 use std::path::Path;
 
 pub use meilisearch_types::milli;
-use meilisearch_types::milli::vector::hf::OverridePooling;
+use meilisearch_types::milli::vector::embedder::hf::OverridePooling;
 use tempfile::TempDir;
 use time::OffsetDateTime;
 use tracing::debug;
@@ -24,7 +24,7 @@ pub type Batch = meilisearch_types::batches::Batch;
 pub type Key = meilisearch_types::keys::Key;
 pub type ChatCompletionSettings = meilisearch_types::features::ChatCompletionSettings;
 pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures;
-pub type Network = meilisearch_types::features::Network;
+pub type Network = meilisearch_types::enterprise_edition::network::Network;
 pub type Webhooks = meilisearch_types::webhooks::WebhooksDumpView;
 
 // ===== Other types to clarify the code of the compat module

crates/dump/src/writer.rs

@@ -5,7 +5,8 @@ use std::path::PathBuf;
 use flate2::write::GzEncoder;
 use flate2::Compression;
 use meilisearch_types::batches::Batch;
-use meilisearch_types::features::{ChatCompletionSettings, Network, RuntimeTogglableFeatures};
+use meilisearch_types::enterprise_edition::network::Network;
+use meilisearch_types::features::{ChatCompletionSettings, RuntimeTogglableFeatures};
 use meilisearch_types::keys::Key;
 use meilisearch_types::settings::{Checked, Settings};
 use meilisearch_types::webhooks::WebhooksDumpView;

crates/file-store/src/lib.rs

@@ -60,7 +60,7 @@ impl FileStore {
 
     /// Returns the file corresponding to the requested uuid.
     pub fn get_update(&self, uuid: Uuid) -> Result<StdFile> {
-        let path = self.get_update_path(uuid);
+        let path = self.update_path(uuid);
         let file = match StdFile::open(path) {
             Ok(file) => file,
             Err(e) => {
@@ -72,7 +72,7 @@ impl FileStore {
     }
 
     /// Returns the path that correspond to this uuid, the path could not exists.
-    pub fn get_update_path(&self, uuid: Uuid) -> PathBuf {
+    pub fn update_path(&self, uuid: Uuid) -> PathBuf {
         self.path.join(uuid.to_string())
     }
 
@@ -148,11 +148,10 @@ impl File {
         Ok(Self { path: PathBuf::new(), file: None })
     }
 
-    pub fn persist(self) -> Result<()> {
-        if let Some(file) = self.file {
-            file.persist(&self.path)?;
-        }
-        Ok(())
+    pub fn persist(self) -> Result<Option<StdFile>> {
+        let Some(file) = self.file else { return Ok(None) };
+
+        Ok(Some(file.persist(&self.path)?))
     }
 }
 

crates/filter-parser/Cargo.toml

@@ -15,6 +15,7 @@ license.workspace = true
 nom = "7.1.3"
 nom_locate = "4.2.0"
 unescaper = "0.1.6"
+levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] }
 
 [dev-dependencies]
 # fixed version due to format breakages in v1.40

crates/filter-parser/src/condition.rs

@@ -7,12 +7,14 @@
 
 use nom::branch::alt;
 use nom::bytes::complete::tag;
-use nom::character::complete::multispace1;
-use nom::combinator::cut;
-use nom::sequence::{terminated, tuple};
+use nom::character::complete::{char, multispace0, multispace1};
+use nom::combinator::{cut, map, value};
+use nom::sequence::{preceded, terminated, tuple};
 use Condition::*;
 
-use crate::{parse_value, FilterCondition, IResult, Span, Token};
+use crate::error::IResultExt;
+use crate::value::{parse_vector_value, parse_vector_value_cut};
+use crate::{parse_value, Error, ErrorKind, FilterCondition, IResult, Span, Token, VectorFilter};
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Condition<'a> {
@@ -113,6 +115,83 @@ pub fn parse_not_exists(input: Span) -> IResult<FilterCondition> {
     Ok((input, FilterCondition::Not(Box::new(FilterCondition::Condition { fid: key, op: Exists }))))
 }
 
+fn parse_vectors(input: Span) -> IResult<(Token, Option<Token>, VectorFilter)> {
+    let (input, _) = multispace0(input)?;
+    let (input, fid) = tag("_vectors")(input)?;
+
+    if let Ok((input, _)) = multispace1::<_, crate::Error>(input) {
+        return Ok((input, (Token::from(fid), None, VectorFilter::None)));
+    }
+
+    let (input, _) = char('.')(input)?;
+
+    // From this point, we are certain this is a vector filter, so our errors must be final.
+    // We could use nom's `cut` but it's better to be explicit about the errors
+
+    if let Ok((_, space)) = tag::<_, _, ()>(" ")(input) {
+        return Err(crate::Error::failure_from_kind(space, ErrorKind::VectorFilterMissingEmbedder));
+    }
+
+    let (input, embedder_name) =
+        parse_vector_value_cut(input, ErrorKind::VectorFilterInvalidEmbedder)?;
+
+    let (input, filter) = alt((
+        map(
+            preceded(tag(".fragments"), |input| {
+                let (input, _) = tag(".")(input).map_cut(ErrorKind::VectorFilterMissingFragment)?;
+                parse_vector_value_cut(input, ErrorKind::VectorFilterInvalidFragment)
+            }),
+            VectorFilter::Fragment,
+        ),
+        value(VectorFilter::UserProvided, tag(".userProvided")),
+        value(VectorFilter::DocumentTemplate, tag(".documentTemplate")),
+        value(VectorFilter::Regenerate, tag(".regenerate")),
+        value(VectorFilter::None, nom::combinator::success("")),
+    ))(input)?;
+
+    if let Ok((input, point)) = tag::<_, _, ()>(".")(input) {
+        let opt_value = parse_vector_value(input).ok().map(|(_, v)| v);
+        let value =
+            opt_value.as_ref().map(|v| v.value().to_owned()).unwrap_or_else(|| point.to_string());
+        let context = opt_value.map(|v| v.original_span()).unwrap_or(point);
+        let previous_kind = match filter {
+            VectorFilter::Fragment(_) => Some("fragments"),
+            VectorFilter::DocumentTemplate => Some("documentTemplate"),
+            VectorFilter::UserProvided => Some("userProvided"),
+            VectorFilter::Regenerate => Some("regenerate"),
+            VectorFilter::None => None,
+        };
+        return Err(Error::failure_from_kind(
+            context,
+            ErrorKind::VectorFilterUnknownSuffix(previous_kind, value),
+        ));
+    }
+
+    let (input, _) = multispace1(input).map_cut(ErrorKind::VectorFilterLeftover)?;
+
+    Ok((input, (Token::from(fid), Some(embedder_name), filter)))
+}
+
+/// vectors_exists          = vectors ("EXISTS" | ("NOT" WS+ "EXISTS"))
+pub fn parse_vectors_exists(input: Span) -> IResult<FilterCondition> {
+    let (input, (fid, embedder, filter)) = parse_vectors(input)?;
+
+    // Try parsing "EXISTS" first
+    if let Ok((input, _)) = tag::<_, _, ()>("EXISTS")(input) {
+        return Ok((input, FilterCondition::VectorExists { fid, embedder, filter }));
+    }
+
+    // Try parsing "NOT EXISTS"
+    if let Ok((input, _)) = tuple::<_, _, (), _>((tag("NOT"), multispace1, tag("EXISTS")))(input) {
+        return Ok((
+            input,
+            FilterCondition::Not(Box::new(FilterCondition::VectorExists { fid, embedder, filter })),
+        ));
+    }
+
+    Err(crate::Error::failure_from_kind(input, ErrorKind::VectorFilterOperation))
+}
+
 /// contains        = value "CONTAINS" value
 pub fn parse_contains(input: Span) -> IResult<FilterCondition> {
     let (input, (fid, contains, value)) =

crates/filter-parser/src/error.rs

@@ -42,6 +42,23 @@ pub fn cut_with_err<'a, O>(
     }
 }
 
+pub trait IResultExt<'a> {
+    fn map_cut(self, kind: ErrorKind<'a>) -> Self;
+}
+
+impl<'a, T> IResultExt<'a> for IResult<'a, T> {
+    fn map_cut(self, kind: ErrorKind<'a>) -> Self {
+        self.map_err(move |e: nom::Err<Error<'a>>| {
+            let input = match e {
+                nom::Err::Incomplete(_) => return e,
+                nom::Err::Error(e) => *e.context(),
+                nom::Err::Failure(e) => *e.context(),
+            };
+            Error::failure_from_kind(input, kind)
+        })
+    }
+}
+
 #[derive(Debug)]
 pub struct Error<'a> {
     context: Span<'a>,
@@ -58,9 +75,21 @@ pub enum ExpectedValueKind {
 pub enum ErrorKind<'a> {
     ReservedGeo(&'a str),
     GeoRadius,
+    GeoRadiusArgumentCount(usize),
     GeoBoundingBox,
+    GeoPolygon,
+    GeoPolygonNotEnoughPoints(usize),
+    GeoCoordinatesNotPair(usize),
     MisusedGeoRadius,
     MisusedGeoBoundingBox,
+    VectorFilterLeftover,
+    VectorFilterInvalidQuotes,
+    VectorFilterMissingEmbedder,
+    VectorFilterInvalidEmbedder,
+    VectorFilterMissingFragment,
+    VectorFilterInvalidFragment,
+    VectorFilterUnknownSuffix(Option<&'static str>, String),
+    VectorFilterOperation,
     InvalidPrimary,
     InvalidEscapedNumber,
     ExpectedEof,
@@ -91,6 +120,10 @@ impl<'a> Error<'a> {
         Self { context, kind }
     }
 
+    pub fn failure_from_kind(context: Span<'a>, kind: ErrorKind<'a>) -> nom::Err<Self> {
+        nom::Err::Failure(Self::new_from_kind(context, kind))
+    }
+
     pub fn new_from_external(context: Span<'a>, error: impl std::error::Error) -> Self {
         Self::new_from_kind(context, ErrorKind::External(error.to_string()))
     }
@@ -128,6 +161,20 @@ impl Display for Error<'_> {
         // first line being the diagnostic and the second line being the incriminated filter.
         let escaped_input = input.escape_debug();
 
+        fn key_suggestion<'a>(key: &str, keys: &[&'a str]) -> Option<&'a str> {
+            let typos =
+                levenshtein_automata::LevenshteinAutomatonBuilder::new(2, true).build_dfa(key);
+            for key in keys.iter() {
+                match typos.eval(key) {
+                    levenshtein_automata::Distance::Exact(_) => {
+                        return Some(key);
+                    }
+                    levenshtein_automata::Distance::AtLeast(_) => continue,
+                }
+            }
+            None
+        }
+
         match &self.kind {
             ErrorKind::ExpectedValue(_) if input.trim().is_empty() => {
                 writeln!(f, "Was expecting a value but instead got nothing.")?
@@ -146,7 +193,7 @@ impl Display for Error<'_> {
             }
             ErrorKind::InvalidPrimary => {
                 let text = if input.trim().is_empty() { "but instead got nothing.".to_string() } else { format!("at `{}`.", escaped_input) };
-                writeln!(f, "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` {}", text)?
+                writeln!(f, "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, `_geoBoundingBox` or `_geoPolygon` {text}")?
             }
             ErrorKind::InvalidEscapedNumber => {
                 writeln!(f, "Found an invalid escaped sequence number: `{}`.", escaped_input)?
@@ -155,11 +202,23 @@ impl Display for Error<'_> {
                 writeln!(f, "Found unexpected characters at the end of the filter: `{}`. You probably forgot an `OR` or an `AND` rule.", escaped_input)?
             }
             ErrorKind::GeoRadius => {
-                writeln!(f, "The `_geoRadius` filter expects three arguments: `_geoRadius(latitude, longitude, radius)`.")?
+                writeln!(f, "The `_geoRadius` filter must be in the form: `_geoRadius(latitude, longitude, radius, optionalResolution)`.")?
+            }
+            ErrorKind::GeoRadiusArgumentCount(count) => {
+                writeln!(f, "Was expecting 3 or 4 arguments for `_geoRadius`, but instead found {count}.")?
             }
             ErrorKind::GeoBoundingBox => {
                 writeln!(f, "The `_geoBoundingBox` filter expects two pairs of arguments: `_geoBoundingBox([latitude, longitude], [latitude, longitude])`.")?
             }
+            ErrorKind::GeoPolygon => {
+                writeln!(f, "The `_geoPolygon` filter doesn't match the expected format: `_geoPolygon([latitude, longitude], [latitude, longitude])`.")?
+            }
+            ErrorKind::GeoPolygonNotEnoughPoints(n) => {
+                writeln!(f, "The `_geoPolygon` filter expects at least 3 points but only {n} were specified")?;
+            }
+            ErrorKind::GeoCoordinatesNotPair(number) => {
+                writeln!(f, "Was expecting 2 coordinates but instead found {number}.")?
+            }
             ErrorKind::ReservedGeo(name) => {
                 writeln!(f, "`{}` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.", name.escape_debug())?
             }
@@ -169,6 +228,44 @@ impl Display for Error<'_> {
             ErrorKind::MisusedGeoBoundingBox => {
                 writeln!(f, "The `_geoBoundingBox` filter is an operation and can't be used as a value.")?
             }
+            ErrorKind::VectorFilterLeftover => {
+                writeln!(f, "The vector filter has leftover tokens.")?
+            }
+            ErrorKind::VectorFilterUnknownSuffix(_, value) if value.as_str() == "." => {
+                writeln!(f, "Was expecting one of `.fragments`, `.userProvided`, `.documentTemplate`, `.regenerate` or nothing, but instead found a point without a valid value.")?;
+            }
+            ErrorKind::VectorFilterUnknownSuffix(None, value) if ["fragments", "userProvided", "documentTemplate", "regenerate"].contains(&value.as_str()) => {
+                // This will happen with "_vectors.rest.\"userProvided\"" for instance
+                writeln!(f, "Was expecting this part to be unquoted.")?
+            }
+            ErrorKind::VectorFilterUnknownSuffix(None, value) => {
+                if let Some(suggestion) = key_suggestion(value, &["fragments", "userProvided", "documentTemplate", "regenerate"]) {
+                    writeln!(f, "Was expecting one of `fragments`, `userProvided`, `documentTemplate`, `regenerate` or nothing, but instead found `{value}`. Did you mean `{suggestion}`?")?;
+                } else {
+                    writeln!(f, "Was expecting one of `fragments`, `userProvided`, `documentTemplate`, `regenerate` or nothing, but instead found `{value}`.")?;
+                }
+            }
+            ErrorKind::VectorFilterUnknownSuffix(Some(previous_filter_kind), value) => {
+                writeln!(f, "Vector filter can only accept one of `fragments`, `userProvided`, `documentTemplate` or `regenerate`, but found both `{previous_filter_kind}` and `{value}`.")?
+            },
+            ErrorKind::VectorFilterInvalidFragment => {
+                writeln!(f, "The vector filter's fragment name is invalid.")?
+            }
+            ErrorKind::VectorFilterMissingFragment => {
+                writeln!(f, "The vector filter is missing a fragment name.")?
+            }
+            ErrorKind::VectorFilterMissingEmbedder => {
+                writeln!(f, "Was expecting embedder name but found nothing.")?
+            }
+            ErrorKind::VectorFilterInvalidEmbedder => {
+                writeln!(f, "The vector filter's embedder name is invalid.")?
+            }
+            ErrorKind::VectorFilterOperation => {
+                writeln!(f, "Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter.")?
+            }
+            ErrorKind::VectorFilterInvalidQuotes => {
+                writeln!(f, "The quotes in one of the values are inconsistent.")?
+            }
             ErrorKind::ReservedKeyword(word) => {
                 writeln!(f, "`{word}` is a reserved keyword and thus cannot be used as a field name unless it is put inside quotes. Use \"{word}\" or \'{word}\' instead.")?
             }

crates/filter-parser/src/lib.rs

@@ -19,6 +19,7 @@
 //! word           = (alphanumeric | _ | - | .)+
 //! geoRadius      = "_geoRadius(" WS* float WS* "," WS* float WS* "," float WS* ")"
 //! geoBoundingBox = "_geoBoundingBox([" WS * float WS* "," WS* float WS* "], [" WS* float WS* "," WS* float WS* "]")
+//! geoPolygon     = "_geoPolygon([[" WS* float WS* "," WS* float WS* "],+])"
 //! ```
 //!
 //! Other BNF grammar used to handle some specific errors:
@@ -65,6 +66,9 @@ use nom_locate::LocatedSpan;
 pub(crate) use value::parse_value;
 use value::word_exact;
 
+use crate::condition::parse_vectors_exists;
+use crate::error::IResultExt;
+
 pub type Span<'a> = LocatedSpan<&'a str, &'a str>;
 
 type IResult<'a, Ret> = nom::IResult<Span<'a>, Ret, Error<'a>>;
@@ -113,7 +117,7 @@ impl<'a> Token<'a> {
         self.span
     }
 
-    pub fn parse_finite_float(&self) -> Result<f64, Error> {
+    pub fn parse_finite_float(&self) -> Result<f64, Error<'a>> {
         let value: f64 = self.value().parse().map_err(|e| self.as_external_error(e))?;
         if value.is_finite() {
             Ok(value)
@@ -136,6 +140,15 @@ impl<'a> From<&'a str> for Token<'a> {
     }
 }
 
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum VectorFilter<'a> {
+    Fragment(Token<'a>),
+    DocumentTemplate,
+    UserProvided,
+    Regenerate,
+    None,
+}
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum FilterCondition<'a> {
     Not(Box<Self>),
@@ -143,8 +156,10 @@ pub enum FilterCondition<'a> {
     In { fid: Token<'a>, els: Vec<Token<'a>> },
     Or(Vec<Self>),
     And(Vec<Self>),
-    GeoLowerThan { point: [Token<'a>; 2], radius: Token<'a> },
+    VectorExists { fid: Token<'a>, embedder: Option<Token<'a>>, filter: VectorFilter<'a> },
+    GeoLowerThan { point: [Token<'a>; 2], radius: Token<'a>, resolution: Option<Token<'a>> },
     GeoBoundingBox { top_right_point: [Token<'a>; 2], bottom_left_point: [Token<'a>; 2] },
+    GeoPolygon { points: Vec<[Token<'a>; 2]> },
 }
 
 pub enum TraversedElement<'a> {
@@ -153,7 +168,7 @@ pub enum TraversedElement<'a> {
 }
 
 impl<'a> FilterCondition<'a> {
-    pub fn use_contains_operator(&self) -> Option<&Token> {
+    pub fn use_contains_operator(&self) -> Option<&Token<'a>> {
         match self {
             FilterCondition::Condition { fid: _, op } => match op {
                 Condition::GreaterThan(_)
@@ -165,21 +180,38 @@ impl<'a> FilterCondition<'a> {
                 | Condition::Exists
                 | Condition::LowerThan(_)
                 | Condition::LowerThanOrEqual(_)
-                | Condition::Between { .. } => None,
-                Condition::Contains { keyword, word: _ }
-                | Condition::StartsWith { keyword, word: _ } => Some(keyword),
+                | Condition::Between { .. }
+                | Condition::StartsWith { .. } => None,
+                Condition::Contains { keyword, word: _ } => Some(keyword),
             },
             FilterCondition::Not(this) => this.use_contains_operator(),
             FilterCondition::Or(seq) | FilterCondition::And(seq) => {
                 seq.iter().find_map(|filter| filter.use_contains_operator())
             }
-            FilterCondition::GeoLowerThan { .. }
+            FilterCondition::VectorExists { .. }
+            | FilterCondition::GeoLowerThan { .. }
             | FilterCondition::GeoBoundingBox { .. }
+            | FilterCondition::GeoPolygon { .. }
             | FilterCondition::In { .. } => None,
         }
     }
 
-    pub fn fids(&self, depth: usize) -> Box<dyn Iterator<Item = &Token> + '_> {
+    pub fn use_vector_filter(&self) -> Option<&Token<'a>> {
+        match self {
+            FilterCondition::Condition { .. } => None,
+            FilterCondition::Not(this) => this.use_vector_filter(),
+            FilterCondition::Or(seq) | FilterCondition::And(seq) => {
+                seq.iter().find_map(|filter| filter.use_vector_filter())
+            }
+            FilterCondition::GeoLowerThan { .. }
+            | FilterCondition::GeoBoundingBox { .. }
+            | FilterCondition::GeoPolygon { .. }
+            | FilterCondition::In { .. } => None,
+            FilterCondition::VectorExists { fid, .. } => Some(fid),
+        }
+    }
+
+    pub fn fids(&self, depth: usize) -> Box<dyn Iterator<Item = &Token<'a>> + '_> {
         if depth == 0 {
             return Box::new(std::iter::empty());
         }
@@ -200,7 +232,7 @@ impl<'a> FilterCondition<'a> {
     }
 
     /// Returns the first token found at the specified depth, `None` if no token at this depth.
-    pub fn token_at_depth(&self, depth: usize) -> Option<&Token> {
+    pub fn token_at_depth(&self, depth: usize) -> Option<&Token<'a>> {
         match self {
             FilterCondition::Condition { fid, .. } if depth == 0 => Some(fid),
             FilterCondition::Or(subfilters) => {
@@ -263,10 +295,7 @@ fn parse_in_body(input: Span) -> IResult<Vec<Token>> {
     let (input, _) = ws(word_exact("IN"))(input)?;
 
     // everything after `IN` can be a failure
-    let (input, _) =
-        cut_with_err(tag("["), |_| Error::new_from_kind(input, ErrorKind::InOpeningBracket))(
-            input,
-        )?;
+    let (input, _) = tag("[")(input).map_cut(ErrorKind::InOpeningBracket)?;
 
     let (input, content) = cut(parse_value_list)(input)?;
 
@@ -371,23 +400,27 @@ fn parse_not(input: Span, depth: usize) -> IResult<FilterCondition> {
 /// If we parse `_geoRadius` we MUST parse the rest of the expression.
 fn parse_geo_radius(input: Span) -> IResult<FilterCondition> {
     // we want to allow space BEFORE the _geoRadius but not after
-    let parsed = preceded(
-        tuple((multispace0, word_exact("_geoRadius"))),
-        // if we were able to parse `_geoRadius` and can't parse the rest of the input we return a failure
-        cut(delimited(char('('), separated_list1(tag(","), ws(recognize_float)), char(')'))),
-    )(input)
-    .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::GeoRadius)));
+
+    let (input, _) = tuple((multispace0, word_exact("_geoRadius")))(input)?;
+
+    // if we were able to parse `_geoRadius` and can't parse the rest of the input we return a failure
+
+    let parsed =
+        delimited(char('('), separated_list1(tag(","), ws(recognize_float)), char(')'))(input)
+            .map_cut(ErrorKind::GeoRadius);
 
     let (input, args) = parsed?;
 
-    if args.len() != 3 {
-        return Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::GeoRadius)));
+    if !(3..=4).contains(&args.len()) {
+        return Err(Error::failure_from_kind(input, ErrorKind::GeoRadiusArgumentCount(args.len())));
     }
 
     let res = FilterCondition::GeoLowerThan {
         point: [args[0].into(), args[1].into()],
         radius: args[2].into(),
+        resolution: args.get(3).cloned().map(Token::from),
     };
+
     Ok((input, res))
 }
 
@@ -395,24 +428,31 @@ fn parse_geo_radius(input: Span) -> IResult<FilterCondition> {
 /// If we parse `_geoBoundingBox` we MUST parse the rest of the expression.
 fn parse_geo_bounding_box(input: Span) -> IResult<FilterCondition> {
     // we want to allow space BEFORE the _geoBoundingBox but not after
-    let parsed = preceded(
-        tuple((multispace0, word_exact("_geoBoundingBox"))),
-        // if we were able to parse `_geoBoundingBox` and can't parse the rest of the input we return a failure
-        cut(delimited(
-            char('('),
-            separated_list1(
-                tag(","),
-                ws(delimited(char('['), separated_list1(tag(","), ws(recognize_float)), char(']'))),
-            ),
-            char(')'),
-        )),
+
+    let (input, _) = tuple((multispace0, word_exact("_geoBoundingBox")))(input)?;
+
+    // if we were able to parse `_geoBoundingBox` and can't parse the rest of the input we return a failure
+
+    let (input, args) = delimited(
+        char('('),
+        separated_list1(
+            tag(","),
+            ws(delimited(char('['), separated_list1(tag(","), ws(recognize_float)), char(']'))),
+        ),
+        char(')'),
     )(input)
-    .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::GeoBoundingBox)));
+    .map_cut(ErrorKind::GeoBoundingBox)?;
 
-    let (input, args) = parsed?;
+    if args.len() != 2 {
+        return Err(Error::failure_from_kind(input, ErrorKind::GeoBoundingBox));
+    }
 
-    if args.len() != 2 || args[0].len() != 2 || args[1].len() != 2 {
-        return Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::GeoBoundingBox)));
+    if let Some(offending) = args.iter().find(|a| a.len() != 2) {
+        let context = offending.first().unwrap_or(&input);
+        return Err(Error::failure_from_kind(
+            *context,
+            ErrorKind::GeoCoordinatesNotPair(offending.len()),
+        ));
     }
 
     let res = FilterCondition::GeoBoundingBox {
@@ -422,6 +462,47 @@ fn parse_geo_bounding_box(input: Span) -> IResult<FilterCondition> {
     Ok((input, res))
 }
 
+/// geoPolygon     = "_geoPolygon([[" WS* float WS* "," WS* float WS* "],+])"
+/// If we parse `_geoPolygon` we MUST parse the rest of the expression.
+fn parse_geo_polygon(input: Span) -> IResult<FilterCondition> {
+    // we want to allow space BEFORE the _geoPolygon but not after
+
+    let (input, _) = tuple((multispace0, word_exact("_geoPolygon")))(input)?;
+
+    // if we were able to parse `_geoPolygon` and can't parse the rest of the input we return a failure
+
+    let (input, args): (_, Vec<Vec<LocatedSpan<_, _>>>) = delimited(
+        char('('),
+        separated_list1(
+            tag(","),
+            ws(delimited(char('['), separated_list1(tag(","), ws(recognize_float)), char(']'))),
+        ),
+        preceded(opt(ws(char(','))), char(')')), // Tolerate trailing comma
+    )(input)
+    .map_cut(ErrorKind::GeoPolygon)?;
+
+    if args.len() < 3 {
+        let context = args.last().and_then(|a| a.last()).unwrap_or(&input);
+        return Err(Error::failure_from_kind(
+            *context,
+            ErrorKind::GeoPolygonNotEnoughPoints(args.len()),
+        ));
+    }
+
+    if let Some(offending) = args.iter().find(|a| a.len() != 2) {
+        let context = offending.first().unwrap_or(&input);
+        return Err(Error::failure_from_kind(
+            *context,
+            ErrorKind::GeoCoordinatesNotPair(offending.len()),
+        ));
+    }
+
+    let res = FilterCondition::GeoPolygon {
+        points: args.into_iter().map(|a| [a[0].into(), a[1].into()]).collect(),
+    };
+    Ok((input, res))
+}
+
 /// geoPoint      = WS* "_geoPoint(float WS* "," WS* float WS* "," WS* float)
 fn parse_geo_point(input: Span) -> IResult<FilterCondition> {
     // we want to forbid space BEFORE the _geoPoint but not after
@@ -433,7 +514,7 @@ fn parse_geo_point(input: Span) -> IResult<FilterCondition> {
     ))(input)
     .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoPoint"))))?;
     // if we succeeded we still return a `Failure` because geoPoints are not allowed
-    Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoPoint"))))
+    Err(Error::failure_from_kind(input, ErrorKind::ReservedGeo("_geoPoint")))
 }
 
 /// geoPoint      = WS* "_geoDistance(float WS* "," WS* float WS* "," WS* float)
@@ -447,7 +528,7 @@ fn parse_geo_distance(input: Span) -> IResult<FilterCondition> {
     ))(input)
     .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoDistance"))))?;
     // if we succeeded we still return a `Failure` because `geoDistance` filters are not allowed
-    Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoDistance"))))
+    Err(Error::failure_from_kind(input, ErrorKind::ReservedGeo("_geoDistance")))
 }
 
 /// geo      = WS* "_geo(float WS* "," WS* float WS* "," WS* float)
@@ -461,7 +542,7 @@ fn parse_geo(input: Span) -> IResult<FilterCondition> {
     ))(input)
     .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geo"))))?;
     // if we succeeded we still return a `Failure` because `_geo` filter is not allowed
-    Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::ReservedGeo("_geo"))))
+    Err(Error::failure_from_kind(input, ErrorKind::ReservedGeo("_geo")))
 }
 
 fn parse_error_reserved_keyword(input: Span) -> IResult<FilterCondition> {
@@ -491,8 +572,8 @@ fn parse_primary(input: Span, depth: usize) -> IResult<FilterCondition> {
                 Error::new_from_kind(input, ErrorKind::MissingClosingDelimiter(c.char()))
             }),
         ),
-        parse_geo_radius,
-        parse_geo_bounding_box,
+        // Made a random block of functions because we reached the maximum number of elements per alt
+        alt((parse_geo_radius, parse_geo_bounding_box, parse_geo_polygon)),
         parse_in,
         parse_not_in,
         parse_condition,
@@ -500,8 +581,7 @@ fn parse_primary(input: Span, depth: usize) -> IResult<FilterCondition> {
         parse_is_not_null,
         parse_is_empty,
         parse_is_not_empty,
-        parse_exists,
-        parse_not_exists,
+        alt((parse_vectors_exists, parse_exists, parse_not_exists)),
         parse_to,
         parse_contains,
         parse_not_contains,
@@ -557,9 +637,28 @@ impl std::fmt::Display for FilterCondition<'_> {
                 }
                 write!(f, "]")
             }
-            FilterCondition::GeoLowerThan { point, radius } => {
+            FilterCondition::VectorExists { fid: _, embedder, filter: inner } => {
+                write!(f, "_vectors")?;
+                if let Some(embedder) = embedder {
+                    write!(f, ".{:?}", embedder.value())?;
+                }
+                match inner {
+                    VectorFilter::Fragment(fragment) => {
+                        write!(f, ".fragments.{:?}", fragment.value())?
+                    }
+                    VectorFilter::DocumentTemplate => write!(f, ".documentTemplate")?,
+                    VectorFilter::UserProvided => write!(f, ".userProvided")?,
+                    VectorFilter::Regenerate => write!(f, ".regenerate")?,
+                    VectorFilter::None => (),
+                }
+                write!(f, " EXISTS")
+            }
+            FilterCondition::GeoLowerThan { point, radius, resolution: None } => {
                 write!(f, "_geoRadius({}, {}, {})", point[0], point[1], radius)
             }
+            FilterCondition::GeoLowerThan { point, radius, resolution: Some(resolution) } => {
+                write!(f, "_geoRadius({}, {}, {}, {})", point[0], point[1], radius, resolution)
+            }
             FilterCondition::GeoBoundingBox {
                 top_right_point: top_left_point,
                 bottom_left_point: bottom_right_point,
@@ -573,6 +672,13 @@ impl std::fmt::Display for FilterCondition<'_> {
                     bottom_right_point[1]
                 )
             }
+            FilterCondition::GeoPolygon { points } => {
+                write!(f, "_geoPolygon([")?;
+                for point in points {
+                    write!(f, "[{}, {}], ", point[0], point[1])?;
+                }
+                write!(f, "])")
+            }
         }
     }
 }
@@ -611,7 +717,7 @@ pub mod tests {
     /// Create a raw [Token]. You must specify the string that appear BEFORE your element followed by your element
     pub fn rtok<'a>(before: &'a str, value: &'a str) -> Token<'a> {
         // if the string is empty we still need to return 1 for the line number
-        let lines = before.is_empty().then_some(1).unwrap_or_else(|| before.lines().count());
+        let lines = if before.is_empty() { 1 } else { before.lines().count() };
         let offset = before.chars().count();
         // the extra field is not checked in the tests so we can set it to nothing
         unsafe { Span::new_from_raw_offset(offset, lines as u32, value, "") }.into()
@@ -630,6 +736,9 @@ pub mod tests {
         insta::assert_snapshot!(p(r"title = 'foo\\\\\\\\'"), @r#"{title} = {foo\\\\}"#);
         // but it also works with other sequences
         insta::assert_snapshot!(p(r#"title = 'foo\x20\n\t\"\'"'"#), @"{title} = {foo \n\t\"\'\"}");
+
+        insta::assert_snapshot!(p(r#"_vectors." valid.name  ".fragments."also.. valid! " EXISTS"#), @r#"_vectors." valid.name  ".fragments."also.. valid! " EXISTS"#);
+        insta::assert_snapshot!(p("_vectors.\"\n\t\r\\\"\" EXISTS"), @r#"_vectors."\n\t\r\"" EXISTS"#);
     }
 
     #[test]
@@ -692,6 +801,18 @@ pub mod tests {
         insta::assert_snapshot!(p("NOT subscribers IS NOT EMPTY"), @"{subscribers} IS EMPTY");
         insta::assert_snapshot!(p("subscribers  IS   NOT   EMPTY"), @"NOT ({subscribers} IS EMPTY)");
 
+        // Test _vectors EXISTS + _vectors NOT EXITS
+        insta::assert_snapshot!(p("_vectors EXISTS"), @"_vectors EXISTS");
+        insta::assert_snapshot!(p("_vectors.embedderName EXISTS"), @r#"_vectors."embedderName" EXISTS"#);
+        insta::assert_snapshot!(p("_vectors.embedderName.documentTemplate EXISTS"), @r#"_vectors."embedderName".documentTemplate EXISTS"#);
+        insta::assert_snapshot!(p("_vectors.embedderName.regenerate EXISTS"), @r#"_vectors."embedderName".regenerate EXISTS"#);
+        insta::assert_snapshot!(p("_vectors.embedderName.regenerate EXISTS"), @r#"_vectors."embedderName".regenerate EXISTS"#);
+        insta::assert_snapshot!(p("_vectors.embedderName.fragments.fragmentName EXISTS"), @r#"_vectors."embedderName".fragments."fragmentName" EXISTS"#);
+        insta::assert_snapshot!(p("  _vectors.embedderName.fragments.fragmentName   EXISTS"), @r#"_vectors."embedderName".fragments."fragmentName" EXISTS"#);
+        insta::assert_snapshot!(p("NOT _vectors EXISTS"), @"NOT (_vectors EXISTS)");
+        insta::assert_snapshot!(p(" NOT  _vectors   EXISTS"), @"NOT (_vectors EXISTS)");
+        insta::assert_snapshot!(p("  _vectors  NOT  EXISTS"), @"NOT (_vectors EXISTS)");
+
         // Test EXISTS + NOT EXITS
         insta::assert_snapshot!(p("subscribers EXISTS"), @"{subscribers} EXISTS");
         insta::assert_snapshot!(p("NOT subscribers EXISTS"), @"NOT ({subscribers} EXISTS)");
@@ -721,12 +842,17 @@ pub mod tests {
         insta::assert_snapshot!(p("_geoRadius(12, 13, 14)"), @"_geoRadius({12}, {13}, {14})");
         insta::assert_snapshot!(p("NOT _geoRadius(12, 13, 14)"), @"NOT (_geoRadius({12}, {13}, {14}))");
         insta::assert_snapshot!(p("_geoRadius(12,13,14)"), @"_geoRadius({12}, {13}, {14})");
+        insta::assert_snapshot!(p("_geoRadius(12,13,14,1000)"), @"_geoRadius({12}, {13}, {14}, {1000})");
 
         // Test geo bounding box
         insta::assert_snapshot!(p("_geoBoundingBox([12, 13], [14, 15])"), @"_geoBoundingBox([{12}, {13}], [{14}, {15}])");
         insta::assert_snapshot!(p("NOT _geoBoundingBox([12, 13], [14, 15])"), @"NOT (_geoBoundingBox([{12}, {13}], [{14}, {15}]))");
         insta::assert_snapshot!(p("_geoBoundingBox([12,13],[14,15])"), @"_geoBoundingBox([{12}, {13}], [{14}, {15}])");
 
+        // Test geo polygon
+        insta::assert_snapshot!(p("_geoPolygon([12, 13], [14, 15], [16, 17])"), @"_geoPolygon([[{12}, {13}], [{14}, {15}], [{16}, {17}], ])");
+        insta::assert_snapshot!(p("_geoPolygon([12, 13], [14, 15], [-1.2,2939.2], [1,1])"), @"_geoPolygon([[{12}, {13}], [{14}, {15}], [{-1.2}, {2939.2}], [{1}, {1}], ])");
+
         // Test OR + AND
         insta::assert_snapshot!(p("channel = ponce AND 'dog race' != 'bernese mountain'"), @"AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ]");
         insta::assert_snapshot!(p("channel = ponce OR 'dog race' != 'bernese mountain'"), @"OR[{channel} = {ponce}, {dog race} != {bernese mountain}, ]");
@@ -783,50 +909,80 @@ pub mod tests {
         11:12 channel = 🐻 AND followers < 100
         "###);
 
-        insta::assert_snapshot!(p("'OR'"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `\'OR\'`.
+        insta::assert_snapshot!(p("'OR'"), @r"
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, `_geoBoundingBox` or `_geoPolygon` at `\'OR\'`.
         1:5 'OR'
-        "###);
+        ");
 
         insta::assert_snapshot!(p("OR"), @r###"
         Was expecting a value but instead got `OR`, which is a reserved keyword. To use `OR` as a field name or a value, surround it by quotes.
         1:3 OR
         "###);
 
-        insta::assert_snapshot!(p("channel Ponce"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `channel Ponce`.
+        insta::assert_snapshot!(p("channel Ponce"), @r"
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, `_geoBoundingBox` or `_geoPolygon` at `channel Ponce`.
         1:14 channel Ponce
-        "###);
+        ");
 
-        insta::assert_snapshot!(p("channel = Ponce OR"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` but instead got nothing.
+        insta::assert_snapshot!(p("channel = Ponce OR"), @r"
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, `_geoBoundingBox` or `_geoPolygon` but instead got nothing.
         19:19 channel = Ponce OR
-        "###);
+        ");
 
-        insta::assert_snapshot!(p("_geoRadius"), @r###"
-        The `_geoRadius` filter expects three arguments: `_geoRadius(latitude, longitude, radius)`.
-        1:11 _geoRadius
-        "###);
+        insta::assert_snapshot!(p("_geoRadius"), @r"
+        The `_geoRadius` filter must be in the form: `_geoRadius(latitude, longitude, radius, optionalResolution)`.
+        11:11 _geoRadius
+        ");
 
-        insta::assert_snapshot!(p("_geoRadius = 12"), @r###"
-        The `_geoRadius` filter expects three arguments: `_geoRadius(latitude, longitude, radius)`.
-        1:16 _geoRadius = 12
-        "###);
+        insta::assert_snapshot!(p("_geoRadius = 12"), @r"
+        The `_geoRadius` filter must be in the form: `_geoRadius(latitude, longitude, radius, optionalResolution)`.
+        11:16 _geoRadius = 12
+        ");
 
-        insta::assert_snapshot!(p("_geoBoundingBox"), @r###"
+        insta::assert_snapshot!(p("_geoBoundingBox"), @r"
         The `_geoBoundingBox` filter expects two pairs of arguments: `_geoBoundingBox([latitude, longitude], [latitude, longitude])`.
-        1:16 _geoBoundingBox
-        "###);
+        16:16 _geoBoundingBox
+        ");
 
-        insta::assert_snapshot!(p("_geoBoundingBox = 12"), @r###"
+        insta::assert_snapshot!(p("_geoBoundingBox = 12"), @r"
         The `_geoBoundingBox` filter expects two pairs of arguments: `_geoBoundingBox([latitude, longitude], [latitude, longitude])`.
-        1:21 _geoBoundingBox = 12
-        "###);
+        16:21 _geoBoundingBox = 12
+        ");
 
-        insta::assert_snapshot!(p("_geoBoundingBox(1.0, 1.0)"), @r###"
+        insta::assert_snapshot!(p("_geoBoundingBox(1.0, 1.0)"), @r"
         The `_geoBoundingBox` filter expects two pairs of arguments: `_geoBoundingBox([latitude, longitude], [latitude, longitude])`.
-        1:26 _geoBoundingBox(1.0, 1.0)
-        "###);
+        17:26 _geoBoundingBox(1.0, 1.0)
+        ");
+
+        insta::assert_snapshot!(p("_geoPolygon([1,2,3])"), @r"
+        The `_geoPolygon` filter expects at least 3 points but only 1 were specified
+        18:19 _geoPolygon([1,2,3])
+        ");
+
+        insta::assert_snapshot!(p("_geoPolygon(1,2,3)"), @r"
+        The `_geoPolygon` filter doesn't match the expected format: `_geoPolygon([latitude, longitude], [latitude, longitude])`.
+        13:19 _geoPolygon(1,2,3)
+        ");
+
+        insta::assert_snapshot!(p("_geoPolygon([1,2],[1,2],[1,2,3])"), @r"
+        Was expecting 2 coordinates but instead found 3.
+        26:27 _geoPolygon([1,2],[1,2],[1,2,3])
+        ");
+
+        insta::assert_snapshot!(p("_geoPolygon([1,2],[1,2,3])"), @r"
+        The `_geoPolygon` filter expects at least 3 points but only 2 were specified
+        24:25 _geoPolygon([1,2],[1,2,3])
+        ");
+
+        insta::assert_snapshot!(p("_geoPolygon(1)"), @r"
+        The `_geoPolygon` filter doesn't match the expected format: `_geoPolygon([latitude, longitude], [latitude, longitude])`.
+        13:15 _geoPolygon(1)
+        ");
+
+        insta::assert_snapshot!(p("_geoPolygon([1,2)"), @r"
+        The `_geoPolygon` filter doesn't match the expected format: `_geoPolygon([latitude, longitude], [latitude, longitude])`.
+        17:18 _geoPolygon([1,2)
+        ");
 
         insta::assert_snapshot!(p("_geoPoint(12, 13, 14)"), @r###"
         `_geoPoint` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
@@ -883,15 +1039,15 @@ pub mod tests {
         34:35 channel = mv OR followers >= 1000)
         "###);
 
-        insta::assert_snapshot!(p("colour NOT EXIST"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `colour NOT EXIST`.
+        insta::assert_snapshot!(p("colour NOT EXIST"), @r"
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, `_geoBoundingBox` or `_geoPolygon` at `colour NOT EXIST`.
         1:17 colour NOT EXIST
-        "###);
+        ");
 
-        insta::assert_snapshot!(p("subscribers 100 TO1000"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `subscribers 100 TO1000`.
+        insta::assert_snapshot!(p("subscribers 100 TO1000"), @r"
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, `_geoBoundingBox` or `_geoPolygon` at `subscribers 100 TO1000`.
         1:23 subscribers 100 TO1000
-        "###);
+        ");
 
         insta::assert_snapshot!(p("channel = ponce ORdog != 'bernese mountain'"), @r###"
         Found unexpected characters at the end of the filter: `ORdog != \'bernese mountain\'`. You probably forgot an `OR` or an `AND` rule.
@@ -946,43 +1102,108 @@ pub mod tests {
         "###
         );
 
+        insta::assert_snapshot!(p(r#"_vectors _vectors EXISTS"#), @r"
+        Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter.
+        10:25 _vectors _vectors EXISTS
+        ");
+        insta::assert_snapshot!(p(r#"_vectors. embedderName EXISTS"#), @r"
+        Was expecting embedder name but found nothing.
+        10:11 _vectors. embedderName EXISTS
+        ");
+        insta::assert_snapshot!(p(r#"_vectors .embedderName EXISTS"#), @r"
+        Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter.
+        10:30 _vectors .embedderName EXISTS
+        ");
+        insta::assert_snapshot!(p(r#"_vectors.embedderName. EXISTS"#), @r"
+        Was expecting one of `.fragments`, `.userProvided`, `.documentTemplate`, `.regenerate` or nothing, but instead found a point without a valid value.
+        22:23 _vectors.embedderName. EXISTS
+        ");
+        insta::assert_snapshot!(p(r#"_vectors."embedderName EXISTS"#), @r#"
+        The quotes in one of the values are inconsistent.
+        10:30 _vectors."embedderName EXISTS
+        "#);
+        insta::assert_snapshot!(p(r#"_vectors."embedderNam"e EXISTS"#), @r#"
+        The vector filter has leftover tokens.
+        23:31 _vectors."embedderNam"e EXISTS
+        "#);
+        insta::assert_snapshot!(p(r#"_vectors.embedderName.documentTemplate. EXISTS"#), @r"
+        Was expecting one of `.fragments`, `.userProvided`, `.documentTemplate`, `.regenerate` or nothing, but instead found a point without a valid value.
+        39:40 _vectors.embedderName.documentTemplate. EXISTS
+        ");
+        insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments EXISTS"#), @r"
+        The vector filter is missing a fragment name.
+        32:39 _vectors.embedderName.fragments EXISTS
+        ");
+        insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments. EXISTS"#), @r"
+        The vector filter's fragment name is invalid.
+        33:40 _vectors.embedderName.fragments. EXISTS
+        ");
+        insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments.test test EXISTS"#), @r"
+        Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter.
+        38:49 _vectors.embedderName.fragments.test test EXISTS
+        ");
+        insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments. test EXISTS"#), @r"
+        The vector filter's fragment name is invalid.
+        33:45 _vectors.embedderName.fragments. test EXISTS
+        ");
+        insta::assert_snapshot!(p(r#"_vectors.embedderName .fragments. test EXISTS"#), @r"
+        Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter.
+        23:46 _vectors.embedderName .fragments. test EXISTS
+        ");
+        insta::assert_snapshot!(p(r#"_vectors.embedderName .fragments.test EXISTS"#), @r"
+        Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter.
+        23:45 _vectors.embedderName .fragments.test EXISTS
+        ");
+        insta::assert_snapshot!(p(r#"_vectors.embedderName.fargments.test EXISTS"#), @r"
+        Was expecting one of `fragments`, `userProvided`, `documentTemplate`, `regenerate` or nothing, but instead found `fargments`. Did you mean `fragments`?
+        23:32 _vectors.embedderName.fargments.test EXISTS
+        ");
+        insta::assert_snapshot!(p(r#"_vectors.embedderName."userProvided" EXISTS"#), @r#"
+        Was expecting this part to be unquoted.
+        24:36 _vectors.embedderName."userProvided" EXISTS
+        "#);
+        insta::assert_snapshot!(p(r#"_vectors.embedderName.userProvided.fragments.test EXISTS"#), @r"
+        Vector filter can only accept one of `fragments`, `userProvided`, `documentTemplate` or `regenerate`, but found both `userProvided` and `fragments`.
+        36:45 _vectors.embedderName.userProvided.fragments.test EXISTS
+        ");
+
         insta::assert_snapshot!(p(r#"NOT OR EXISTS AND EXISTS NOT EXISTS"#), @r###"
         Was expecting a value but instead got `OR`, which is a reserved keyword. To use `OR` as a field name or a value, surround it by quotes.
         5:7 NOT OR EXISTS AND EXISTS NOT EXISTS
         "###);
 
-        insta::assert_snapshot!(p(r#"value NULL"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value NULL`.
+        insta::assert_snapshot!(p(r#"value NULL"#), @r"
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, `_geoBoundingBox` or `_geoPolygon` at `value NULL`.
         1:11 value NULL
-        "###);
-        insta::assert_snapshot!(p(r#"value NOT NULL"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value NOT NULL`.
+        ");
+        insta::assert_snapshot!(p(r#"value NOT NULL"#), @r"
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, `_geoBoundingBox` or `_geoPolygon` at `value NOT NULL`.
         1:15 value NOT NULL
-        "###);
-        insta::assert_snapshot!(p(r#"value EMPTY"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value EMPTY`.
+        ");
+        insta::assert_snapshot!(p(r#"value EMPTY"#), @r"
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, `_geoBoundingBox` or `_geoPolygon` at `value EMPTY`.
         1:12 value EMPTY
-        "###);
-        insta::assert_snapshot!(p(r#"value NOT EMPTY"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value NOT EMPTY`.
+        ");
+        insta::assert_snapshot!(p(r#"value NOT EMPTY"#), @r"
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, `_geoBoundingBox` or `_geoPolygon` at `value NOT EMPTY`.
         1:16 value NOT EMPTY
-        "###);
-        insta::assert_snapshot!(p(r#"value IS"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value IS`.
+        ");
+        insta::assert_snapshot!(p(r#"value IS"#), @r"
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, `_geoBoundingBox` or `_geoPolygon` at `value IS`.
         1:9 value IS
-        "###);
-        insta::assert_snapshot!(p(r#"value IS NOT"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value IS NOT`.
+        ");
+        insta::assert_snapshot!(p(r#"value IS NOT"#), @r"
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, `_geoBoundingBox` or `_geoPolygon` at `value IS NOT`.
         1:13 value IS NOT
-        "###);
-        insta::assert_snapshot!(p(r#"value IS EXISTS"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value IS EXISTS`.
+        ");
+        insta::assert_snapshot!(p(r#"value IS EXISTS"#), @r"
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, `_geoBoundingBox` or `_geoPolygon` at `value IS EXISTS`.
         1:16 value IS EXISTS
-        "###);
-        insta::assert_snapshot!(p(r#"value IS NOT EXISTS"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value IS NOT EXISTS`.
+        ");
+        insta::assert_snapshot!(p(r#"value IS NOT EXISTS"#), @r"
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, `_geoBoundingBox` or `_geoPolygon` at `value IS NOT EXISTS`.
         1:20 value IS NOT EXISTS
-        "###);
+        ");
     }
 
     #[test]

crates/filter-parser/src/value.rs

@@ -80,6 +80,51 @@ pub fn word_exact<'a, 'b: 'a>(tag: &'b str) -> impl Fn(Span<'a>) -> IResult<'a,
     }
 }
 
+/// vector_value          = ( non_dot_word | singleQuoted | doubleQuoted)
+pub fn parse_vector_value(input: Span) -> IResult<Token> {
+    pub fn non_dot_word(input: Span) -> IResult<Token> {
+        let (input, word) = take_while1(|c| is_value_component(c) && c != '.')(input)?;
+        Ok((input, word.into()))
+    }
+
+    let (input, value) = alt((
+        delimited(char('\''), cut(|input| quoted_by('\'', input)), cut(char('\''))),
+        delimited(char('"'), cut(|input| quoted_by('"', input)), cut(char('"'))),
+        non_dot_word,
+    ))(input)?;
+
+    match unescaper::unescape(value.value()) {
+        Ok(content) => {
+            if content.len() != value.value().len() {
+                Ok((input, Token::new(value.original_span(), Some(content))))
+            } else {
+                Ok((input, value))
+            }
+        }
+        Err(unescaper::Error::IncompleteStr(_)) => Err(nom::Err::Incomplete(nom::Needed::Unknown)),
+        Err(unescaper::Error::ParseIntError { .. }) => Err(nom::Err::Error(Error::new_from_kind(
+            value.original_span(),
+            ErrorKind::InvalidEscapedNumber,
+        ))),
+        Err(unescaper::Error::InvalidChar { .. }) => Err(nom::Err::Error(Error::new_from_kind(
+            value.original_span(),
+            ErrorKind::MalformedValue,
+        ))),
+    }
+}
+
+pub fn parse_vector_value_cut<'a>(input: Span<'a>, kind: ErrorKind<'a>) -> IResult<'a, Token<'a>> {
+    parse_vector_value(input).map_err(|e| match e {
+        nom::Err::Failure(e) => match e.kind() {
+            ErrorKind::Char(c) if *c == '"' || *c == '\'' => {
+                crate::Error::failure_from_kind(input, ErrorKind::VectorFilterInvalidQuotes)
+            }
+            _ => crate::Error::failure_from_kind(input, kind),
+        },
+        _ => crate::Error::failure_from_kind(input, kind),
+    })
+}
+
 /// value          = WS* ( word | singleQuoted | doubleQuoted) WS+
 pub fn parse_value(input: Span) -> IResult<Token> {
     // to get better diagnostic message we are going to strip the left whitespaces from the input right now
@@ -99,31 +144,21 @@ pub fn parse_value(input: Span) -> IResult<Token> {
     }
 
     match parse_geo_radius(input) {
-        Ok(_) => {
-            return Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::MisusedGeoRadius)))
-        }
+        Ok(_) => return Err(Error::failure_from_kind(input, ErrorKind::MisusedGeoRadius)),
         // if we encountered a failure it means the user badly wrote a _geoRadius filter.
         // But instead of showing them how to fix his syntax we are going to tell them they should not use this filter as a value.
         Err(e) if e.is_failure() => {
-            return Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::MisusedGeoRadius)))
+            return Err(Error::failure_from_kind(input, ErrorKind::MisusedGeoRadius))
         }
         _ => (),
     }
 
     match parse_geo_bounding_box(input) {
-        Ok(_) => {
-            return Err(nom::Err::Failure(Error::new_from_kind(
-                input,
-                ErrorKind::MisusedGeoBoundingBox,
-            )))
-        }
+        Ok(_) => return Err(Error::failure_from_kind(input, ErrorKind::MisusedGeoBoundingBox)),
         // if we encountered a failure it means the user badly wrote a _geoBoundingBox filter.
         // But instead of showing them how to fix his syntax we are going to tell them they should not use this filter as a value.
         Err(e) if e.is_failure() => {
-            return Err(nom::Err::Failure(Error::new_from_kind(
-                input,
-                ErrorKind::MisusedGeoBoundingBox,
-            )))
+            return Err(Error::failure_from_kind(input, ErrorKind::MisusedGeoBoundingBox))
         }
         _ => (),
     }

crates/fuzzers/src/bin/fuzz-indexing.rs

@@ -129,6 +129,7 @@ fn main() {
                                     &mut new_fields_ids_map,
                                     &|| false,
                                     Progress::default(),
+                                    None,
                                 )
                                 .unwrap();
 

crates/index-scheduler/Cargo.toml

@@ -14,6 +14,7 @@ license.workspace = true
 anyhow = "1.0.98"
 bincode = "1.3.3"
 byte-unit = "5.1.6"
+bytes = "1.10.1"
 bumpalo = "3.18.1"
 bumparaw-collections = "0.1.4"
 convert_case = "0.8.0"
@@ -32,6 +33,7 @@ rayon = "1.10.0"
 roaring = { version = "0.10.12", features = ["serde"] }
 serde = { version = "1.0.219", features = ["derive"] }
 serde_json = { version = "1.0.140", features = ["preserve_order"] }
+tar = "0.4.44"
 synchronoise = "1.0.1"
 tempfile = "3.20.0"
 thiserror = "2.0.12"
@@ -45,6 +47,9 @@ tracing = "0.1.41"
 ureq = "2.12.1"
 uuid = { version = "1.17.0", features = ["serde", "v4"] }
 backoff = "0.4.0"
+reqwest = { version = "0.12.23", features = ["rustls-tls", "http2"], default-features = false }
+rusty-s3 = "0.8.1"
+tokio = { version = "1.47.1", features = ["full"] }
 
 [dev-dependencies]
 big_s = "1.0.2"

crates/index-scheduler/src/dump.rs

@@ -1,3 +1,5 @@
+#![allow(clippy::result_large_err)]
+
 use std::collections::HashMap;
 use std::io;
 
@@ -147,6 +149,8 @@ impl<'a> Dump<'a> {
             canceled_by: task.canceled_by,
             details: task.details,
             status: task.status,
+            network: task.network,
+            custom_metadata: task.custom_metadata,
             kind: match task.kind {
                 KindDump::DocumentImport {
                     primary_key,
@@ -197,9 +201,10 @@ impl<'a> Dump<'a> {
                     index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
                     primary_key,
                 },
-                KindDump::IndexUpdate { primary_key } => KindWithContent::IndexUpdate {
+                KindDump::IndexUpdate { primary_key, uid } => KindWithContent::IndexUpdate {
                     index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
                     primary_key,
+                    new_index_uid: uid,
                 },
                 KindDump::IndexSwap { swaps } => KindWithContent::IndexSwap { swaps },
                 KindDump::TaskCancelation { query, tasks } => {
@@ -230,6 +235,12 @@ impl<'a> Dump<'a> {
                     }
                 }
                 KindDump::UpgradeDatabase { from } => KindWithContent::UpgradeDatabase { from },
+                KindDump::IndexCompaction { index_uid } => {
+                    KindWithContent::IndexCompaction { index_uid }
+                }
+                KindDump::NetworkTopologyChange(network_topology_change) => {
+                    KindWithContent::NetworkTopologyChange(network_topology_change)
+                }
             },
         };
 

crates/index-scheduler/src/error.rs

@@ -3,9 +3,13 @@ use std::fmt::Display;
 use meilisearch_types::batches::BatchId;
 use meilisearch_types::error::{Code, ErrorCode};
 use meilisearch_types::milli::index::RollbackOutcome;
+use meilisearch_types::milli::DocumentId;
+use meilisearch_types::tasks::enterprise_edition::network::ReceiveTaskError;
 use meilisearch_types::tasks::{Kind, Status};
 use meilisearch_types::{heed, milli};
+use reqwest::StatusCode;
 use thiserror::Error;
+use uuid::Uuid;
 
 use crate::TaskId;
 
@@ -67,6 +71,8 @@ pub enum Error {
     SwapDuplicateIndexesFound(Vec<String>),
     #[error("Index `{0}` not found.")]
     SwapIndexNotFound(String),
+    #[error("Cannot rename `{0}` to `{1}` as the index already exists. Hint: You can remove `{1}` first and then do your remove.")]
+    SwapIndexFoundDuringRename(String, String),
     #[error("Meilisearch cannot receive write operations because the limit of the task database has been reached. Please delete tasks to continue performing write operations.")]
     NoSpaceLeftInTaskQueue,
     #[error(
@@ -74,6 +80,10 @@ pub enum Error {
         .0.iter().map(|s| format!("`{}`", s)).collect::<Vec<_>>().join(", ")
     )]
     SwapIndexesNotFound(Vec<String>),
+    #[error("The following indexes are being renamed but cannot because their new name conflicts with an already existing index: {}. Renaming doesn't overwrite the other index name.",
+        .0.iter().map(|s| format!("`{}`", s)).collect::<Vec<_>>().join(", ")
+    )]
+    SwapIndexesFoundDuringRename(Vec<String>),
     #[error("Corrupted dump.")]
     CorruptedDump,
     #[error(
@@ -121,6 +131,14 @@ pub enum Error {
     #[error("Aborted task")]
     AbortedTask,
 
+    #[error("S3 error: status: {status}, body: {body}")]
+    S3Error { status: StatusCode, body: String },
+    #[error("S3 HTTP error: {0}")]
+    S3HttpError(reqwest::Error),
+    #[error("S3 XML error: {0}")]
+    S3XmlError(Box<dyn std::error::Error + Send + Sync>),
+    #[error("S3 bucket error: {0}")]
+    S3BucketError(rusty_s3::BucketError),
     #[error(transparent)]
     Dump(#[from] dump::Error),
     #[error(transparent)]
@@ -176,6 +194,15 @@ pub enum Error {
     #[error(transparent)]
     HeedTransaction(heed::Error),
 
+    #[error("No network topology change task is currently enqueued or processing")]
+    ImportTaskWithoutNetworkTask,
+    #[error("The network task version (`{network_task}`) does not match the import task version (`{import_task}`)")]
+    NetworkVersionMismatch { network_task: Uuid, import_task: Uuid },
+    #[error("The import task emanates from an unknown remote `{0}`")]
+    ImportTaskUnknownRemote(String),
+    #[error("The import task with key `{0}` was already received")]
+    ImportTaskAlreadyReceived(DocumentId),
+
     #[cfg(test)]
     #[error("Planned failure for tests.")]
     PlannedFailure,
@@ -203,6 +230,8 @@ impl Error {
             | Error::SwapIndexNotFound(_)
             | Error::NoSpaceLeftInTaskQueue
             | Error::SwapIndexesNotFound(_)
+            | Error::SwapIndexFoundDuringRename(_, _)
+            | Error::SwapIndexesFoundDuringRename(_)
             | Error::CorruptedDump
             | Error::InvalidTaskDate { .. }
             | Error::InvalidTaskUid { .. }
@@ -218,6 +247,10 @@ impl Error {
             | Error::TaskCancelationWithEmptyQuery
             | Error::FromRemoteWhenExporting { .. }
             | Error::AbortedTask
+            | Error::S3Error { .. }
+            | Error::S3HttpError(_)
+            | Error::S3XmlError(_)
+            | Error::S3BucketError(_)
             | Error::Dump(_)
             | Error::Heed(_)
             | Error::Milli { .. }
@@ -227,6 +260,10 @@ impl Error {
             | Error::Persist(_)
             | Error::FeatureNotEnabled(_)
             | Error::Export(_)
+            | Error::ImportTaskWithoutNetworkTask
+            | Error::NetworkVersionMismatch { .. }
+            | Error::ImportTaskAlreadyReceived(_)
+            | Error::ImportTaskUnknownRemote(_)
             | Error::Anyhow(_) => true,
             Error::CreateBatch(_)
             | Error::CorruptedTaskQueue
@@ -271,6 +308,8 @@ impl ErrorCode for Error {
             Error::SwapDuplicateIndexFound(_) => Code::InvalidSwapDuplicateIndexFound,
             Error::SwapIndexNotFound(_) => Code::IndexNotFound,
             Error::SwapIndexesNotFound(_) => Code::IndexNotFound,
+            Error::SwapIndexFoundDuringRename(_, _) => Code::IndexAlreadyExists,
+            Error::SwapIndexesFoundDuringRename(_) => Code::IndexAlreadyExists,
             Error::InvalidTaskDate { field, .. } => (*field).into(),
             Error::InvalidTaskUid { .. } => Code::InvalidTaskUids,
             Error::InvalidBatchUid { .. } => Code::InvalidBatchUids,
@@ -283,8 +322,18 @@ impl ErrorCode for Error {
             Error::BatchNotFound(_) => Code::BatchNotFound,
             Error::TaskDeletionWithEmptyQuery => Code::MissingTaskFilters,
             Error::TaskCancelationWithEmptyQuery => Code::MissingTaskFilters,
-            // TODO: not sure of the Code to use
             Error::NoSpaceLeftInTaskQueue => Code::NoSpaceLeftOnDevice,
+            Error::ImportTaskWithoutNetworkTask => Code::ImportTaskWithoutNetworkTask,
+            Error::NetworkVersionMismatch { .. } => Code::NetworkVersionMismatch,
+            Error::ImportTaskAlreadyReceived(_) => Code::ImportTaskAlreadyReceived,
+            Error::ImportTaskUnknownRemote(_) => Code::ImportTaskUnknownRemote,
+            Error::S3Error { status, .. } if status.is_client_error() => {
+                Code::InvalidS3SnapshotRequest
+            }
+            Error::S3Error { .. } => Code::S3SnapshotServerError,
+            Error::S3HttpError(_) => Code::S3SnapshotServerError,
+            Error::S3XmlError(_) => Code::S3SnapshotServerError,
+            Error::S3BucketError(_) => Code::InvalidS3SnapshotParameters,
             Error::Dump(e) => e.error_code(),
             Error::Milli { error, .. } => error.error_code(),
             Error::ProcessBatchPanicked(_) => Code::Internal,
@@ -316,3 +365,12 @@ impl ErrorCode for Error {
         }
     }
 }
+
+impl From<ReceiveTaskError> for Error {
+    fn from(value: ReceiveTaskError) -> Self {
+        match value {
+            ReceiveTaskError::UnknownRemote(unknown) => Error::ImportTaskUnknownRemote(unknown),
+            ReceiveTaskError::DuplicateTask(dup) => Error::ImportTaskAlreadyReceived(dup),
+        }
+    }
+}

crates/index-scheduler/src/features.rs

@@ -1,6 +1,7 @@
 use std::sync::{Arc, RwLock};
 
-use meilisearch_types::features::{InstanceTogglableFeatures, Network, RuntimeTogglableFeatures};
+use meilisearch_types::enterprise_edition::network::Network;
+use meilisearch_types::features::{InstanceTogglableFeatures, RuntimeTogglableFeatures};
 use meilisearch_types::heed::types::{SerdeJson, Str};
 use meilisearch_types::heed::{Database, Env, RwTxn, WithoutTls};
 
@@ -85,7 +86,7 @@ impl RoFeatures {
             Ok(())
         } else {
             Err(FeatureNotEnabledError {
-                disabled_action: "Using `CONTAINS` or `STARTS WITH` in a filter",
+                disabled_action: "Using `CONTAINS` in a filter",
                 feature: "contains filter",
                 issue_link: "https://github.com/orgs/meilisearch/discussions/763",
             }
@@ -157,6 +158,19 @@ impl RoFeatures {
             .into())
         }
     }
+
+    pub fn check_vector_store_setting(&self, disabled_action: &'static str) -> Result<()> {
+        if self.runtime.vector_store_setting {
+            Ok(())
+        } else {
+            Err(FeatureNotEnabledError {
+                disabled_action,
+                feature: "vector_store_setting",
+                issue_link: "https://github.com/orgs/meilisearch/discussions/860",
+            }
+            .into())
+        }
+    }
 }
 
 impl FeatureData {

crates/index-scheduler/src/index_mapper/mod.rs

@@ -143,10 +143,10 @@ impl IndexStats {
     ///
     /// - rtxn: a RO transaction for the index, obtained from `Index::read_txn()`.
     pub fn new(index: &Index, rtxn: &RoTxn) -> milli::Result<Self> {
-        let arroy_stats = index.arroy_stats(rtxn)?;
+        let vector_store_stats = index.vector_store_stats(rtxn)?;
         Ok(IndexStats {
-            number_of_embeddings: Some(arroy_stats.number_of_embeddings),
-            number_of_embedded_documents: Some(arroy_stats.documents.len()),
+            number_of_embeddings: Some(vector_store_stats.number_of_embeddings),
+            number_of_embedded_documents: Some(vector_store_stats.documents.len()),
             documents_database_stats: index.documents_stats(rtxn)?.unwrap_or_default(),
             number_of_documents: None,
             database_size: index.on_disk_size()?,
@@ -199,7 +199,7 @@ impl IndexMapper {
                 let uuid = Uuid::new_v4();
                 self.index_mapping.put(&mut wtxn, name, &uuid)?;
 
-                let index_path = self.base_path.join(uuid.to_string());
+                let index_path = self.index_path(uuid);
                 fs::create_dir_all(&index_path)?;
 
                 // Error if the UUIDv4 somehow already exists in the map, since it should be fresh.
@@ -286,7 +286,7 @@ impl IndexMapper {
         };
 
         let index_map = self.index_map.clone();
-        let index_path = self.base_path.join(uuid.to_string());
+        let index_path = self.index_path(uuid);
         let index_name = name.to_string();
         thread::Builder::new()
             .name(String::from("index_deleter"))
@@ -341,6 +341,31 @@ impl IndexMapper {
         Ok(())
     }
 
+    /// Closes the specified index.
+    ///
+    /// This operation involves closing the underlying environment and so can take a long time to complete.
+    ///
+    /// # Panics
+    ///
+    /// - If the Index corresponding to the passed name is concurrently being deleted/resized or cannot be found in the
+    ///   in memory hash map.
+    pub fn close_index(&self, rtxn: &RoTxn, name: &str) -> Result<()> {
+        let uuid = self
+            .index_mapping
+            .get(rtxn, name)?
+            .ok_or_else(|| Error::IndexNotFound(name.to_string()))?;
+
+        // We remove the index from the in-memory index map.
+        self.index_map.write().unwrap().close_for_resize(&uuid, self.enable_mdb_writemap, 0);
+
+        Ok(())
+    }
+
+    /// The number of indexes in the database
+    pub fn index_count(&self, rtxn: &RoTxn) -> Result<u64> {
+        Ok(self.index_mapping.len(rtxn)?)
+    }
+
     /// Return an index, may open it if it wasn't already opened.
     pub fn index(&self, rtxn: &RoTxn, name: &str) -> Result<Index> {
         if let Some((current_name, current_index)) =
@@ -388,7 +413,7 @@ impl IndexMapper {
                     } else {
                         continue;
                     };
-                    let index_path = self.base_path.join(uuid.to_string());
+                    let index_path = self.index_path(uuid);
                     // take the lock to reopen the environment.
                     reopen
                         .reopen(&mut self.index_map.write().unwrap(), &index_path)
@@ -405,7 +430,7 @@ impl IndexMapper {
                     // if it's not already there.
                     match index_map.get(&uuid) {
                         Missing => {
-                            let index_path = self.base_path.join(uuid.to_string());
+                            let index_path = self.index_path(uuid);
 
                             break index_map
                                 .create(
@@ -432,6 +457,14 @@ impl IndexMapper {
         Ok(index)
     }
 
+    /// Returns the path of the index.
+    ///
+    /// The folder located at this path is containing the data.mdb,
+    /// the lock.mdb and an optional data.mdb.cpy file.
+    pub fn index_path(&self, uuid: Uuid) -> PathBuf {
+        self.base_path.join(uuid.to_string())
+    }
+
     pub fn rollback_index(
         &self,
         rtxn: &RoTxn,
@@ -472,7 +505,7 @@ impl IndexMapper {
             };
         }
 
-        let index_path = self.base_path.join(uuid.to_string());
+        let index_path = self.index_path(uuid);
         Index::rollback(milli::heed::EnvOpenOptions::new().read_txn_without_tls(), index_path, to)
             .map_err(|err| crate::Error::from_milli(err, Some(name.to_string())))
     }
@@ -526,6 +559,20 @@ impl IndexMapper {
         Ok(())
     }
 
+    /// Rename an index.
+    pub fn rename(&self, wtxn: &mut RwTxn, current: &str, new: &str) -> Result<()> {
+        let uuid = self
+            .index_mapping
+            .get(wtxn, current)?
+            .ok_or_else(|| Error::IndexNotFound(current.to_string()))?;
+        if self.index_mapping.get(wtxn, new)?.is_some() {
+            return Err(Error::IndexAlreadyExists(new.to_string()));
+        }
+        self.index_mapping.delete(wtxn, current)?;
+        self.index_mapping.put(wtxn, new, &uuid)?;
+        Ok(())
+    }
+
     /// The stats of an index.
     ///
     /// If available in the cache, they are directly returned.

crates/index-scheduler/src/insta_snapshot.rs

@@ -21,7 +21,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
     let IndexScheduler {
         cleanup_enabled: _,
         experimental_no_edition_2024_for_dumps: _,
-        processing_tasks,
+        runtime_tasks,
         env,
         version,
         queue,
@@ -36,6 +36,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
         run_loop_iteration: _,
         embedders: _,
         chat_settings: _,
+        runtime: _,
     } = scheduler;
 
     let rtxn = env.read_txn().unwrap();
@@ -49,7 +50,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
         snap.push_str(&format!("index scheduler running on version {indx_sched_version:?}\n"));
     }
 
-    let processing = processing_tasks.read().unwrap().clone();
+    let processing = runtime_tasks.read().unwrap().processing.clone();
     snap.push_str(&format!("### Autobatching Enabled = {}\n", scheduler.autobatching_enabled));
     snap.push_str(&format!(
         "### Processing batch {:?}:\n",
@@ -230,6 +231,8 @@ pub fn snapshot_task(task: &Task) -> String {
         details,
         status,
         kind,
+        network,
+        custom_metadata,
     } = task;
     snap.push('{');
     snap.push_str(&format!("uid: {uid}, "));
@@ -247,6 +250,12 @@ pub fn snapshot_task(task: &Task) -> String {
         snap.push_str(&format!("details: {}, ", &snapshot_details(details)));
     }
     snap.push_str(&format!("kind: {kind:?}"));
+    if let Some(network) = network {
+        snap.push_str(&format!("network: {network:?}, "))
+    }
+    if let Some(custom_metadata) = custom_metadata {
+        snap.push_str(&format!("custom_metadata: {custom_metadata:?}"))
+    }
 
     snap.push('}');
     snap
@@ -274,8 +283,8 @@ fn snapshot_details(d: &Details) -> String {
         Details::SettingsUpdate { settings } => {
             format!("{{ settings: {settings:?} }}")
         }
-        Details::IndexInfo { primary_key } => {
-            format!("{{ primary_key: {primary_key:?} }}")
+        Details::IndexInfo { primary_key, new_index_uid, old_index_uid } => {
+            format!("{{ primary_key: {primary_key:?}, old_new_uid: {old_index_uid:?}, new_index_uid: {new_index_uid:?} }}")
         }
         Details::DocumentDeletion {
             provided_ids: received_document_ids,
@@ -313,6 +322,12 @@ fn snapshot_details(d: &Details) -> String {
         Details::UpgradeDatabase { from, to } => {
             format!("{{ from: {from:?}, to: {to:?} }}")
         }
+        Details::IndexCompaction { index_uid, pre_compaction_size, post_compaction_size } => {
+            format!("{{ index_uid: {index_uid:?}, pre_compaction_size: {pre_compaction_size:?}, post_compaction_size: {post_compaction_size:?} }}")
+        }
+        Details::NetworkTopologyChange { moved_documents, received_documents, message } => {
+            format!("{{ moved_documents: {moved_documents:?}, received_documents: {received_documents:?}, message: {message:?}")
+        }
     }
 }
 

crates/index-scheduler/src/lib.rs

@@ -1,3 +1,6 @@
+// The main Error type is large and boxing the large variant make the pattern matching fails
+#![allow(clippy::result_large_err)]
+
 /*!
 This crate defines the index scheduler, which is responsible for:
 1. Keeping references to meilisearch's indexes and mapping them to their
@@ -51,8 +54,9 @@ pub use features::RoFeatures;
 use flate2::bufread::GzEncoder;
 use flate2::Compression;
 use meilisearch_types::batches::Batch;
+use meilisearch_types::enterprise_edition::network::Network;
 use meilisearch_types::features::{
-    ChatCompletionSettings, InstanceTogglableFeatures, Network, RuntimeTogglableFeatures,
+    ChatCompletionSettings, InstanceTogglableFeatures, RuntimeTogglableFeatures,
 };
 use meilisearch_types::heed::byteorder::BE;
 use meilisearch_types::heed::types::{DecodeIgnore, SerdeJson, Str, I128};
@@ -64,10 +68,10 @@ use meilisearch_types::milli::vector::{
 };
 use meilisearch_types::milli::{self, Index};
 use meilisearch_types::task_view::TaskView;
+use meilisearch_types::tasks::enterprise_edition::network::{DbTaskNetwork, TaskNetwork};
 use meilisearch_types::tasks::{KindWithContent, Task};
 use meilisearch_types::webhooks::{Webhook, WebhooksDumpView, WebhooksView};
 use milli::vector::db::IndexEmbeddingConfig;
-use processing::ProcessingTasks;
 pub use queue::Query;
 use queue::Queue;
 use roaring::RoaringBitmap;
@@ -78,6 +82,7 @@ use uuid::Uuid;
 use versioning::Versioning;
 
 use crate::index_mapper::IndexMapper;
+use crate::processing::RuntimeTasks;
 use crate::utils::clamp_to_page_size;
 
 pub(crate) type BEI128 = I128<BE>;
@@ -159,7 +164,7 @@ pub struct IndexScheduler {
     pub(crate) env: Env<WithoutTls>,
 
     /// The list of tasks currently processing
-    pub(crate) processing_tasks: Arc<RwLock<ProcessingTasks>>,
+    pub(crate) runtime_tasks: Arc<RwLock<RuntimeTasks>>,
 
     /// A database containing only the version of the index-scheduler
     pub version: versioning::Versioning,
@@ -212,13 +217,16 @@ pub struct IndexScheduler {
     /// A counter that is incremented before every call to [`tick`](IndexScheduler::tick)
     #[cfg(test)]
     run_loop_iteration: Arc<RwLock<usize>>,
+
+    /// The tokio runtime used for asynchronous tasks.
+    runtime: Option<tokio::runtime::Handle>,
 }
 
 impl IndexScheduler {
     fn private_clone(&self) -> IndexScheduler {
         IndexScheduler {
             env: self.env.clone(),
-            processing_tasks: self.processing_tasks.clone(),
+            runtime_tasks: self.runtime_tasks.clone(),
             version: self.version.clone(),
             queue: self.queue.private_clone(),
             scheduler: self.scheduler.private_clone(),
@@ -238,6 +246,7 @@ impl IndexScheduler {
             run_loop_iteration: self.run_loop_iteration.clone(),
             features: self.features.clone(),
             chat_settings: self.chat_settings,
+            runtime: self.runtime.clone(),
         }
     }
 
@@ -251,13 +260,23 @@ impl IndexScheduler {
     }
 
     /// Create an index scheduler and start its run loop.
-    #[allow(private_interfaces)] // because test_utils is private
     pub fn new(
         options: IndexSchedulerOptions,
         auth_env: Env<WithoutTls>,
         from_db_version: (u32, u32, u32),
-        #[cfg(test)] test_breakpoint_sdr: crossbeam_channel::Sender<(test_utils::Breakpoint, bool)>,
-        #[cfg(test)] planned_failures: Vec<(usize, test_utils::FailureLocation)>,
+        runtime: Option<tokio::runtime::Handle>,
+    ) -> Result<Self> {
+        let this = Self::new_without_run(options, auth_env, from_db_version, runtime)?;
+
+        this.run();
+        Ok(this)
+    }
+
+    fn new_without_run(
+        options: IndexSchedulerOptions,
+        auth_env: Env<WithoutTls>,
+        from_db_version: (u32, u32, u32),
+        runtime: Option<tokio::runtime::Handle>,
     ) -> Result<Self> {
         std::fs::create_dir_all(&options.tasks_path)?;
         std::fs::create_dir_all(&options.update_file_path)?;
@@ -312,9 +331,8 @@ impl IndexScheduler {
 
         wtxn.commit()?;
 
-        // allow unreachable_code to get rids of the warning in the case of a test build.
-        let this = Self {
-            processing_tasks: Arc::new(RwLock::new(ProcessingTasks::new())),
+        Ok(Self {
+            runtime_tasks: Arc::new(RwLock::new(RuntimeTasks::new())),
             version,
             queue,
             scheduler: Scheduler::new(&options, auth_env),
@@ -329,21 +347,38 @@ impl IndexScheduler {
             webhooks: Arc::new(webhooks),
             embedders: Default::default(),
 
-            #[cfg(test)]
-            test_breakpoint_sdr,
-            #[cfg(test)]
-            planned_failures,
+            #[cfg(test)] // Will be replaced in `new_tests` in test environments
+            test_breakpoint_sdr: crossbeam_channel::bounded(0).0,
+            #[cfg(test)] // Will be replaced in `new_tests` in test environments
+            planned_failures: Default::default(),
             #[cfg(test)]
             run_loop_iteration: Arc::new(RwLock::new(0)),
             features,
             chat_settings,
-        };
+            runtime,
+        })
+    }
+
+    /// Create an index scheduler and start its run loop.
+    #[cfg(test)]
+    fn new_test(
+        options: IndexSchedulerOptions,
+        auth_env: Env<WithoutTls>,
+        from_db_version: (u32, u32, u32),
+        runtime: Option<tokio::runtime::Handle>,
+        test_breakpoint_sdr: crossbeam_channel::Sender<(test_utils::Breakpoint, bool)>,
+        planned_failures: Vec<(usize, test_utils::FailureLocation)>,
+    ) -> Result<Self> {
+        let mut this = Self::new_without_run(options, auth_env, from_db_version, runtime)?;
+
+        this.test_breakpoint_sdr = test_breakpoint_sdr;
+        this.planned_failures = planned_failures;
 
         this.run();
         Ok(this)
     }
 
-    fn read_txn(&self) -> Result<RoTxn<WithoutTls>> {
+    fn read_txn(&self) -> Result<RoTxn<'_, WithoutTls>> {
         self.env.read_txn().map_err(|e| e.into())
     }
 
@@ -605,19 +640,19 @@ impl IndexScheduler {
     /// 3. The number of times the properties appeared.
     pub fn get_stats(&self) -> Result<BTreeMap<String, BTreeMap<String, u64>>> {
         let rtxn = self.read_txn()?;
-        self.queue.get_stats(&rtxn, &self.processing_tasks.read().unwrap())
+        self.queue.get_stats(&rtxn, &self.runtime_tasks.read().unwrap().processing)
     }
 
     // Return true if there is at least one task that is processing.
     pub fn is_task_processing(&self) -> Result<bool> {
-        Ok(!self.processing_tasks.read().unwrap().processing.is_empty())
+        Ok(!self.runtime_tasks.read().unwrap().processing.processing.is_empty())
     }
 
     /// Return true iff there is at least one task associated with this index
     /// that is processing.
     pub fn is_index_processing(&self, index: &str) -> Result<bool> {
         let rtxn = self.env.read_txn()?;
-        let processing_tasks = self.processing_tasks.read().unwrap().processing.clone();
+        let processing_tasks = self.runtime_tasks.read().unwrap().processing.processing.clone();
         let index_tasks = self.queue.tasks.index_tasks(&rtxn, index)?;
         let nbr_index_processing_tasks = processing_tasks.intersection_len(&index_tasks);
         Ok(nbr_index_processing_tasks > 0)
@@ -643,8 +678,8 @@ impl IndexScheduler {
         filters: &meilisearch_auth::AuthFilter,
     ) -> Result<(Vec<Task>, u64)> {
         let rtxn = self.read_txn()?;
-        let processing = self.processing_tasks.read().unwrap();
-        self.queue.get_tasks_from_authorized_indexes(&rtxn, query, filters, &processing)
+        let processing = &self.runtime_tasks.read().unwrap().processing;
+        self.queue.get_tasks_from_authorized_indexes(&rtxn, query, filters, processing)
     }
 
     /// Return the task ids matching the query along with the total number of tasks
@@ -662,8 +697,18 @@ impl IndexScheduler {
         filters: &meilisearch_auth::AuthFilter,
     ) -> Result<(RoaringBitmap, u64)> {
         let rtxn = self.read_txn()?;
-        let processing = self.processing_tasks.read().unwrap();
-        self.queue.get_task_ids_from_authorized_indexes(&rtxn, query, filters, &processing)
+        let processing = &self.runtime_tasks.read().unwrap().processing;
+        self.queue.get_task_ids_from_authorized_indexes(&rtxn, query, filters, processing)
+    }
+
+    pub fn set_task_network(&self, task_id: TaskId, network: DbTaskNetwork) -> Result<Task> {
+        let mut wtxn = self.env.write_txn()?;
+        let mut task =
+            self.queue.tasks.get_task(&wtxn, task_id)?.ok_or(Error::TaskNotFound(task_id))?;
+        task.network = Some(network);
+        self.queue.tasks.all_tasks.put(&mut wtxn, &task_id, &task)?;
+        wtxn.commit()?;
+        Ok(task)
     }
 
     /// Return the batches matching the query from the user's point of view along
@@ -681,8 +726,8 @@ impl IndexScheduler {
         filters: &meilisearch_auth::AuthFilter,
     ) -> Result<(Vec<Batch>, u64)> {
         let rtxn = self.read_txn()?;
-        let processing = self.processing_tasks.read().unwrap();
-        self.queue.get_batches_from_authorized_indexes(&rtxn, query, filters, &processing)
+        let processing = &self.runtime_tasks.read().unwrap().processing;
+        self.queue.get_batches_from_authorized_indexes(&rtxn, query, filters, processing)
     }
 
     /// Return the batch ids matching the query along with the total number of batches
@@ -700,8 +745,8 @@ impl IndexScheduler {
         filters: &meilisearch_auth::AuthFilter,
     ) -> Result<(RoaringBitmap, u64)> {
         let rtxn = self.read_txn()?;
-        let processing = self.processing_tasks.read().unwrap();
-        self.queue.get_batch_ids_from_authorized_indexes(&rtxn, query, filters, &processing)
+        let processing = &self.runtime_tasks.read().unwrap().processing;
+        self.queue.get_batch_ids_from_authorized_indexes(&rtxn, query, filters, processing)
     }
 
     /// Register a new task in the scheduler.
@@ -712,6 +757,31 @@ impl IndexScheduler {
         kind: KindWithContent,
         task_id: Option<TaskId>,
         dry_run: bool,
+    ) -> Result<Task> {
+        self.register_with_custom_metadata(kind, task_id, None, dry_run, None)
+    }
+
+    /// Register a new task in the scheduler, with metadata.
+    ///
+    /// If it fails and data was associated with the task, it tries to delete the associated data.
+    ///
+    /// # Parameters
+    ///
+    /// - task_network: network of the task to check.
+    ///
+    /// If the task is an import task, only accept it if:
+    ///
+    /// 1. There is an ongoing network topology change task
+    /// 2. The task to register matches the network version of the network topology change task
+    ///
+    /// Always accept the task if it is not an import task.
+    pub fn register_with_custom_metadata(
+        &self,
+        kind: KindWithContent,
+        task_id: Option<TaskId>,
+        custom_metadata: Option<String>,
+        dry_run: bool,
+        task_network: Option<TaskNetwork>,
     ) -> Result<Task> {
         // if the task doesn't delete or cancel anything and 40% of the task queue is full, we must refuse to enqueue the incoming task
         if !matches!(&kind, KindWithContent::TaskDeletion { tasks, .. } | KindWithContent::TaskCancelation { tasks, .. } if !tasks.is_empty())
@@ -722,13 +792,86 @@ impl IndexScheduler {
         }
 
         let mut wtxn = self.env.write_txn()?;
-        let task = self.queue.register(&mut wtxn, &kind, task_id, dry_run)?;
+
+        if let Some(TaskNetwork::Import { import_from, network_change, metadata }) = &task_network {
+            let mut network_tasks = self
+                .queue
+                .tasks
+                .get_kind(&wtxn, meilisearch_types::tasks::Kind::NetworkTopologyChange)?;
+            if network_tasks.is_empty() {
+                return Err(Error::ImportTaskWithoutNetworkTask);
+            }
+
+            let network_task = {
+                let processing = self.runtime_tasks.read().unwrap().processing.processing.clone();
+                if processing.is_disjoint(&network_tasks) {
+                    let enqueued = self
+                        .queue
+                        .tasks
+                        .get_status(&wtxn, meilisearch_types::tasks::Status::Enqueued)?;
+
+                    network_tasks &= enqueued;
+                    if let Some(network_task) = network_tasks.into_iter().next() {
+                        network_task
+                    } else {
+                        return Err(Error::ImportTaskWithoutNetworkTask);
+                    }
+                } else {
+                    network_tasks &= &*processing;
+                    network_tasks.into_iter().next().unwrap()
+                }
+            };
+
+            let mut network_task = self.queue.tasks.get_task(&wtxn, network_task)?.unwrap();
+            let network_task_version = network_task
+                .network
+                .as_ref()
+                .map(|network| network.network_version())
+                .unwrap_or_default();
+            if network_task_version != network_change.network_version {
+                return Err(Error::NetworkVersionMismatch {
+                    network_task: network_task_version,
+                    import_task: network_change.network_version,
+                });
+            }
+
+            let KindWithContent::NetworkTopologyChange(network_topology_change) =
+                &mut network_task.kind
+            else {
+                return Err(Error::CorruptedTaskQueue);
+            };
+
+            network_topology_change.receive_remote_task(
+                &import_from.remote_name,
+                &import_from.index_name,
+                metadata.task_key,
+                import_from.document_count,
+                metadata.index_count,
+                metadata.total_index_documents,
+            )?;
+
+            self.queue.tasks.update_task(&mut wtxn, &mut network_task)?;
+        }
+
+        let task = self.queue.register(
+            &mut wtxn,
+            &kind,
+            task_id,
+            custom_metadata,
+            dry_run,
+            task_network.map(DbTaskNetwork::from),
+        )?;
 
         // If the registered task is a task cancelation
         // we inform the processing tasks to stop (if necessary).
         if let KindWithContent::TaskCancelation { tasks, .. } = kind {
             let tasks_to_cancel = RoaringBitmap::from_iter(tasks);
-            if self.processing_tasks.read().unwrap().must_cancel_processing_tasks(&tasks_to_cancel)
+            if self
+                .runtime_tasks
+                .read()
+                .unwrap()
+                .processing
+                .must_cancel_processing_tasks(&tasks_to_cancel)
             {
                 self.scheduler.must_stop_processing.must_stop();
             }
@@ -746,7 +889,7 @@ impl IndexScheduler {
 
     /// Register a new task coming from a dump in the scheduler.
     /// By taking a mutable ref we're pretty sure no one will ever import a dump while actix is running.
-    pub fn register_dumped_task(&mut self) -> Result<Dump> {
+    pub fn register_dumped_task(&mut self) -> Result<Dump<'_>> {
         Dump::new(self)
     }
 
@@ -795,10 +938,8 @@ impl IndexScheduler {
                                 .queue
                                 .tasks
                                 .get_task(self.rtxn, task_id)
-                                .map_err(|err| io::Error::new(io::ErrorKind::Other, err))?
-                                .ok_or_else(|| {
-                                    io::Error::new(io::ErrorKind::Other, Error::CorruptedTaskQueue)
-                                })?;
+                                .map_err(io::Error::other)?
+                                .ok_or_else(|| io::Error::other(Error::CorruptedTaskQueue))?;
 
                             serde_json::to_writer(&mut self.buffer, &TaskView::from_task(&task))?;
                             self.buffer.push(b'\n');

crates/index-scheduler/src/processing.rs

@@ -6,6 +6,28 @@ use roaring::RoaringBitmap;
 
 use crate::utils::ProcessingBatch;
 
+pub struct RuntimeTasks {
+    pub processing: ProcessingTasks,
+    pub enqueued_network: EnqueuedNetworkTasks,
+}
+impl RuntimeTasks {
+    pub(crate) fn new() -> Self {
+        Self { processing: ProcessingTasks::new(), enqueued_network: Default::default() }
+    }
+}
+
+#[derive(Default)]
+pub struct EnqueuedNetworkTasks {
+    tasks: RoaringBitmap,
+}
+
+impl EnqueuedNetworkTasks {
+    pub fn swap(&mut self, mut new: RoaringBitmap) -> RoaringBitmap {
+        std::mem::swap(&mut self.tasks, &mut new);
+        new
+    }
+}
+
 #[derive(Clone, Default)]
 pub struct ProcessingTasks {
     pub batch: Option<Arc<ProcessingBatch>>,
@@ -75,6 +97,7 @@ make_enum_progress! {
     pub enum TaskCancelationProgress {
         RetrievingTasks,
         CancelingUpgrade,
+        CleaningCompactionLeftover,
         UpdatingTasks,
     }
 }
@@ -138,6 +161,17 @@ make_enum_progress! {
     }
 }
 
+make_enum_progress! {
+    pub enum IndexCompaction {
+        RetrieveTheIndex,
+        CreateTemporaryFile,
+        CopyAndCompactTheIndex,
+        PersistTheCompactedIndex,
+        CloseTheIndex,
+        ReopenTheIndex,
+    }
+}
+
 make_enum_progress! {
     pub enum InnerSwappingTwoIndexes {
         RetrieveTheTasks,

crates/index-scheduler/src/queue/batches.rs

@@ -275,19 +275,27 @@ impl BatchQueue {
     pub(crate) fn get_existing_batches(
         &self,
         rtxn: &RoTxn,
-        tasks: impl IntoIterator<Item = BatchId>,
+        batches: impl IntoIterator<Item = BatchId>,
         processing: &ProcessingTasks,
     ) -> Result<Vec<Batch>> {
-        tasks
+        batches
             .into_iter()
             .map(|batch_id| {
                 if Some(batch_id) == processing.batch.as_ref().map(|batch| batch.uid) {
                     let mut batch = processing.batch.as_ref().unwrap().to_batch();
                     batch.progress = processing.get_progress_view();
+                    // Add progress_trace from the current progress state
+                    if let Some(progress) = &processing.progress {
+                        batch.stats.progress_trace = progress
+                            .accumulated_durations()
+                            .into_iter()
+                            .map(|(k, v)| (k, v.into()))
+                            .collect();
+                    }
                     Ok(batch)
                 } else {
                     self.get_batch(rtxn, batch_id)
-                        .and_then(|task| task.ok_or(Error::CorruptedTaskQueue))
+                        .and_then(|batch| batch.ok_or(Error::CorruptedTaskQueue))
                 }
             })
             .collect::<Result<_>>()

crates/index-scheduler/src/queue/batches_test.rs

@@ -26,7 +26,7 @@ fn query_batches_from_and_limit() {
     handle.advance_n_successful_batches(3);
     snapshot!(snapshot_index_scheduler(&index_scheduler), name: "processed_all_tasks");
 
-    let proc = index_scheduler.processing_tasks.read().unwrap().clone();
+    let proc = index_scheduler.runtime_tasks.read().unwrap().processing.clone();
     let rtxn = index_scheduler.env.read_txn().unwrap();
     let query = Query { limit: Some(0), ..Default::default() };
     let (batches, _) = index_scheduler
@@ -104,6 +104,15 @@ fn query_batches_simple() {
     batches[0].started_at = OffsetDateTime::UNIX_EPOCH;
     assert!(batches[0].enqueued_at.is_some());
     batches[0].enqueued_at = None;
+
+    if !batches[0].stats.progress_trace.is_empty() {
+        batches[0].stats.progress_trace.clear();
+        batches[0]
+            .stats
+            .progress_trace
+            .insert("processing tasks".to_string(), "deterministic_duration".into());
+    }
+
     // Insta cannot snapshot our batches because the batch stats contains an enum as key: https://github.com/mitsuhiko/insta/issues/689
     let batch = serde_json::to_string_pretty(&batches[0]).unwrap();
     snapshot!(batch, @r###"
@@ -122,6 +131,9 @@ fn query_batches_simple() {
         },
         "indexUids": {
           "catto": 1
+        },
+        "progressTrace": {
+          "processing tasks": "deterministic_duration"
         }
       },
       "startedAt": "1970-01-01T00:00:00Z",
@@ -334,11 +346,11 @@ fn query_batches_special_rules() {
     let kind = index_creation_task("doggo", "sheep");
     let _task = index_scheduler.register(kind, None, false).unwrap();
     let kind = KindWithContent::IndexSwap {
-        swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()) }],
+        swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()), rename: false }],
     };
     let _task = index_scheduler.register(kind, None, false).unwrap();
     let kind = KindWithContent::IndexSwap {
-        swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "whalo".to_owned()) }],
+        swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "whalo".to_owned()), rename: false }],
     };
     let _task = index_scheduler.register(kind, None, false).unwrap();
 
@@ -347,7 +359,7 @@ fn query_batches_special_rules() {
     handle.advance_till([Start, BatchCreated]);
 
     let rtxn = index_scheduler.env.read_txn().unwrap();
-    let proc = index_scheduler.processing_tasks.read().unwrap().clone();
+    let proc = index_scheduler.runtime_tasks.read().unwrap().processing.clone();
 
     let query = Query { index_uids: Some(vec!["catto".to_owned()]), ..Default::default() };
     let (batches, _) = index_scheduler
@@ -442,7 +454,7 @@ fn query_batches_canceled_by() {
     let kind = index_creation_task("doggo", "sheep");
     let _ = index_scheduler.register(kind, None, false).unwrap();
     let kind = KindWithContent::IndexSwap {
-        swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()) }],
+        swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()), rename: false }],
     };
     let _task = index_scheduler.register(kind, None, false).unwrap();
 

crates/index-scheduler/src/queue/mod.rs

@@ -15,6 +15,7 @@ use file_store::FileStore;
 use meilisearch_types::batches::BatchId;
 use meilisearch_types::heed::{Database, Env, RoTxn, RwTxn, WithoutTls};
 use meilisearch_types::milli::{CboRoaringBitmapCodec, BEU32};
+use meilisearch_types::tasks::enterprise_edition::network::DbTaskNetwork;
 use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task};
 use roaring::RoaringBitmap;
 use time::format_description::well_known::Rfc3339;
@@ -257,7 +258,9 @@ impl Queue {
         wtxn: &mut RwTxn,
         kind: &KindWithContent,
         task_id: Option<TaskId>,
+        custom_metadata: Option<String>,
         dry_run: bool,
+        network: Option<DbTaskNetwork>,
     ) -> Result<Task> {
         let next_task_id = self.tasks.next_task_id(wtxn)?;
 
@@ -279,6 +282,8 @@ impl Queue {
             details: kind.default_details(),
             status: Status::Enqueued,
             kind: kind.clone(),
+            network,
+            custom_metadata,
         };
         // For deletion and cancelation tasks, we want to make extra sure that they
         // don't attempt to delete/cancel tasks that are newer than themselves.
@@ -309,7 +314,8 @@ impl Queue {
             | self.tasks.status.get(wtxn, &Status::Failed)?.unwrap_or_default()
             | self.tasks.status.get(wtxn, &Status::Canceled)?.unwrap_or_default();
 
-        let to_delete = RoaringBitmap::from_iter(finished.into_iter().rev().take(100_000));
+        let to_delete =
+            RoaringBitmap::from_sorted_iter(finished.into_iter().take(100_000)).unwrap();
 
         // /!\ the len must be at least 2 or else we might enter an infinite loop where we only delete
         //     the deletion tasks we enqueued ourselves.
@@ -325,7 +331,7 @@ impl Queue {
         );
 
         // it's safe to unwrap here because we checked the len above
-        let newest_task_id = to_delete.iter().last().unwrap();
+        let newest_task_id = to_delete.iter().next_back().unwrap();
         let last_task_to_delete =
             self.tasks.get_task(wtxn, newest_task_id)?.ok_or(Error::CorruptedTaskQueue)?;
 
@@ -342,7 +348,9 @@ impl Queue {
                 tasks: to_delete,
             },
             None,
+            None,
             false,
+            None,
         )?;
 
         Ok(())

crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap

@@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
-1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
-2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
+1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
+2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }}
 3 {uid: 3, batch_uid: 1, status: succeeded, details: { matched_tasks: 3, canceled_tasks: Some(2), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0, 1, 2]> }}
 ----------------------------------------------------------------------
 ### Status:
@@ -49,7 +49,7 @@ catto: { number_of_documents: 0, field_distribution: {} }
 ----------------------------------------------------------------------
 ### All Batches:
 0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", }
-1 {uid: 1, details: {"primaryKey":"sheep","matchedTasks":3,"canceledTasks":2,"originalFilter":"test_query","swaps":[{"indexes":["catto","doggo"]}]}, stats: {"totalNbTasks":3,"status":{"succeeded":1,"canceled":2},"types":{"indexCreation":1,"indexSwap":1,"taskCancelation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `taskCancelation` that cannot be batched with any other task.", }
+1 {uid: 1, details: {"primaryKey":"sheep","matchedTasks":3,"canceledTasks":2,"originalFilter":"test_query","swaps":[{"indexes":["catto","doggo"],"rename":false}]}, stats: {"totalNbTasks":3,"status":{"succeeded":1,"canceled":2},"types":{"indexCreation":1,"indexSwap":1,"taskCancelation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `taskCancelation` that cannot be batched with any other task.", }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]

crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap

@@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
-1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }}
-2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
+1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }}
+2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []

crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap

@@ -1,13 +1,12 @@
 ---
 source: crates/index-scheduler/src/queue/batches_test.rs
-snapshot_kind: text
 ---
 ### Autobatching Enabled = true
 ### Processing batch None:
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
+0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,]

crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap

@@ -1,14 +1,13 @@
 ---
 source: crates/index-scheduler/src/queue/batches_test.rs
-snapshot_kind: text
 ---
 ### Autobatching Enabled = true
 ### Processing batch None:
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
-1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }}
+0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
+1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,1,]

crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap

@@ -1,15 +1,14 @@
 ---
 source: crates/index-scheduler/src/queue/batches_test.rs
-snapshot_kind: text
 ---
 ### Autobatching Enabled = true
 ### Processing batch None:
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
-1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }}
-2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }}
+0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
+1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }}
+2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,1,2,]

crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap

@@ -7,9 +7,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs
 {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", }
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
-1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
-2 {uid: 2, status: enqueued, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
+1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
+2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [1,2,]

crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap

@@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
-1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
-2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
+1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
+2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []

crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap

@@ -1,15 +1,14 @@
 ---
 source: crates/index-scheduler/src/queue/batches_test.rs
-snapshot_kind: text
 ---
 ### Autobatching Enabled = true
 ### Processing batch None:
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
-1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
-2 {uid: 2, status: enqueued, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }}
+0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
+1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
+2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,1,2,]

crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap

@@ -6,10 +6,10 @@ source: crates/index-scheduler/src/queue/batches_test.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
-1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
-2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }}
-3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `whalo` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
+1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
+2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }}
+3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `whalo` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []
@@ -54,8 +54,8 @@ doggo: { number_of_documents: 0, field_distribution: {} }
 ### All Batches:
 0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", }
 1 {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", }
-2 {uid: 2, details: {"swaps":[{"indexes":["catto","doggo"]}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 2 of type `indexSwap` that cannot be batched with any other task.", }
-3 {uid: 3, details: {"swaps":[{"indexes":["catto","whalo"]}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 3 of type `indexSwap` that cannot be batched with any other task.", }
+2 {uid: 2, details: {"swaps":[{"indexes":["catto","doggo"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 2 of type `indexSwap` that cannot be batched with any other task.", }
+3 {uid: 3, details: {"swaps":[{"indexes":["catto","whalo"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 3 of type `indexSwap` that cannot be batched with any other task.", }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]

crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap

@@ -1,16 +1,15 @@
 ---
 source: crates/index-scheduler/src/queue/batches_test.rs
-snapshot_kind: text
 ---
 ### Autobatching Enabled = true
 ### Processing batch None:
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
-1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
-2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }}
-3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }}
+0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
+1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
+2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }}
+3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,1,2,3,]

crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap

@@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
-1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
-2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
+1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
+2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }}
 3 {uid: 3, batch_uid: 1, status: succeeded, details: { matched_tasks: 3, canceled_tasks: Some(2), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0, 1, 2]> }}
 ----------------------------------------------------------------------
 ### Status:
@@ -49,7 +49,7 @@ catto: { number_of_documents: 0, field_distribution: {} }
 ----------------------------------------------------------------------
 ### All Batches:
 0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", }
-1 {uid: 1, details: {"primaryKey":"sheep","matchedTasks":3,"canceledTasks":2,"originalFilter":"test_query","swaps":[{"indexes":["catto","doggo"]}]}, stats: {"totalNbTasks":3,"status":{"succeeded":1,"canceled":2},"types":{"indexCreation":1,"indexSwap":1,"taskCancelation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `taskCancelation` that cannot be batched with any other task.", }
+1 {uid: 1, details: {"primaryKey":"sheep","matchedTasks":3,"canceledTasks":2,"originalFilter":"test_query","swaps":[{"indexes":["catto","doggo"],"rename":false}]}, stats: {"totalNbTasks":3,"status":{"succeeded":1,"canceled":2},"types":{"indexCreation":1,"indexSwap":1,"taskCancelation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `taskCancelation` that cannot be batched with any other task.", }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]

crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap

@@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
-1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }}
-2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
+1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }}
+2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []

crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap

@@ -1,13 +1,12 @@
 ---
 source: crates/index-scheduler/src/queue/tasks_test.rs
-snapshot_kind: text
 ---
 ### Autobatching Enabled = true
 ### Processing batch None:
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
+0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,]

crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap

@@ -1,14 +1,13 @@
 ---
 source: crates/index-scheduler/src/queue/tasks_test.rs
-snapshot_kind: text
 ---
 ### Autobatching Enabled = true
 ### Processing batch None:
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
-1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }}
+0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
+1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,1,]

crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap

@@ -1,15 +1,14 @@
 ---
 source: crates/index-scheduler/src/queue/tasks_test.rs
-snapshot_kind: text
 ---
 ### Autobatching Enabled = true
 ### Processing batch None:
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
-1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }}
-2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }}
+0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
+1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }}
+2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,1,2,]

crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap

@@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
-1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
-2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
+1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
+2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []

crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap

@@ -1,15 +1,14 @@
 ---
 source: crates/index-scheduler/src/queue/tasks_test.rs
-snapshot_kind: text
 ---
 ### Autobatching Enabled = true
 ### Processing batch None:
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
-1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
-2 {uid: 2, status: enqueued, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }}
+0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
+1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
+2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,1,2,]

crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap

@@ -1,16 +1,15 @@
 ---
 source: crates/index-scheduler/src/queue/tasks_test.rs
-snapshot_kind: text
 ---
 ### Autobatching Enabled = true
 ### Processing batch None:
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
-1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
-2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }}
-3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }}
+0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
+1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }}
+2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }}
+3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,1,2,3,]

crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap

@@ -1,13 +1,12 @@
 ---
 source: crates/index-scheduler/src/queue/test.rs
-snapshot_kind: text
 ---
 ### Autobatching Enabled = true
 ### Processing batch None:
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
+0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }}
 2 {uid: 2, status: enqueued, details: { received_documents: 50, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 50, allow_index_creation: true }}
 3 {uid: 3, status: enqueued, details: { received_documents: 5000, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 5000, allow_index_creation: true }}

crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap

@@ -1,6 +1,5 @@
 ---
 source: crates/index-scheduler/src/queue/test.rs
-snapshot_kind: text
 ---
 [
   {
@@ -13,7 +12,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "enqueued",

crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap

@@ -1,6 +1,5 @@
 ---
 source: crates/index-scheduler/src/queue/test.rs
-snapshot_kind: text
 ---
 [
   {
@@ -13,7 +12,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "succeeded",
@@ -39,7 +40,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "failed",
@@ -60,7 +63,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "enqueued",
@@ -81,7 +86,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "enqueued",

crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap

@@ -1,6 +1,5 @@
 ---
 source: crates/index-scheduler/src/queue/test.rs
-snapshot_kind: text
 ---
 [
   {
@@ -13,7 +12,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "enqueued",
@@ -34,7 +35,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "enqueued",

crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap

@@ -1,6 +1,5 @@
 ---
 source: crates/index-scheduler/src/queue/test.rs
-snapshot_kind: text
 ---
 [
   {
@@ -13,7 +12,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "succeeded",
@@ -39,7 +40,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "failed",
@@ -60,7 +63,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "enqueued",
@@ -81,7 +86,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "enqueued",

crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap

@@ -1,6 +1,5 @@
 ---
 source: crates/index-scheduler/src/queue/test.rs
-snapshot_kind: text
 ---
 [
   {
@@ -13,7 +12,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "succeeded",
@@ -39,7 +40,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "failed",
@@ -60,7 +63,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "enqueued",
@@ -81,7 +86,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "enqueued",

crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap

@@ -1,6 +1,5 @@
 ---
 source: crates/index-scheduler/src/queue/test.rs
-snapshot_kind: text
 ---
 [
   {
@@ -13,7 +12,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "succeeded",
@@ -39,7 +40,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "failed",
@@ -60,7 +63,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "enqueued",
@@ -81,7 +86,9 @@ snapshot_kind: text
     "canceledBy": null,
     "details": {
       "IndexInfo": {
-        "primary_key": null
+        "primary_key": null,
+        "new_index_uid": null,
+        "old_index_uid": null
       }
     },
     "status": "enqueued",

crates/index-scheduler/src/queue/tasks.rs

@@ -97,7 +97,22 @@ impl TaskQueue {
         Ok(self.all_tasks.get(rtxn, &task_id)?)
     }
 
-    pub(crate) fn update_task(&self, wtxn: &mut RwTxn, task: &Task) -> Result<()> {
+    /// Update the inverted task indexes and write the new value of the task.
+    ///
+    /// The passed `task` object typically comes from a previous transaction, so two kinds of modification might have occurred:
+    /// 1. Modification to the `task` object after loading it from the DB (the purpose of this method is to persist these changes)
+    /// 2. Modification to the task committed by another transaction in the DB (an annoying consequence of having lost the original
+    ///    transaction from which the `task` instance was deserialized)
+    ///
+    /// When calling this function, this `task` is modified to take into account any existing `network`
+    /// that can have been added since the task was loaded into memory.
+    ///
+    /// Any other modification to the task that was committed from the DB since the parameter was pulled from the DB will be overwritten.
+    ///
+    /// # Errors
+    ///
+    /// - CorruptedTaskQueue: The task doesn't exist in the database
+    pub(crate) fn update_task(&self, wtxn: &mut RwTxn, task: &mut Task) -> Result<()> {
         let old_task = self.get_task(wtxn, task.uid)?.ok_or(Error::CorruptedTaskQueue)?;
         let reprocessing = old_task.status != Status::Enqueued;
 
@@ -157,6 +172,12 @@ impl TaskQueue {
             }
         }
 
+        task.network = match (old_task.network, task.network.take()) {
+            (None, None) => None,
+            (None, Some(network)) | (Some(network), None) => Some(network),
+            (Some(_), Some(network)) => Some(network),
+        };
+
         self.all_tasks.put(wtxn, &task.uid, task)?;
         Ok(())
     }

crates/index-scheduler/src/queue/tasks_test.rs

@@ -27,7 +27,7 @@ fn query_tasks_from_and_limit() {
     snapshot!(snapshot_index_scheduler(&index_scheduler), name: "processed_all_tasks");
 
     let rtxn = index_scheduler.env.read_txn().unwrap();
-    let processing = index_scheduler.processing_tasks.read().unwrap();
+    let processing = &index_scheduler.runtime_tasks.read().unwrap().processing;
     let query = Query { limit: Some(0), ..Default::default() };
     let (tasks, _) = index_scheduler
         .queue
@@ -304,11 +304,11 @@ fn query_tasks_special_rules() {
     let kind = index_creation_task("doggo", "sheep");
     let _task = index_scheduler.register(kind, None, false).unwrap();
     let kind = KindWithContent::IndexSwap {
-        swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()) }],
+        swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()), rename: false }],
     };
     let _task = index_scheduler.register(kind, None, false).unwrap();
     let kind = KindWithContent::IndexSwap {
-        swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "whalo".to_owned()) }],
+        swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "whalo".to_owned()), rename: false }],
     };
     let _task = index_scheduler.register(kind, None, false).unwrap();
 
@@ -317,7 +317,7 @@ fn query_tasks_special_rules() {
     handle.advance_till([Start, BatchCreated]);
 
     let rtxn = index_scheduler.env.read_txn().unwrap();
-    let proc = index_scheduler.processing_tasks.read().unwrap();
+    let proc = &index_scheduler.runtime_tasks.read().unwrap().processing;
 
     let query = Query { index_uids: Some(vec!["catto".to_owned()]), ..Default::default() };
     let (tasks, _) = index_scheduler
@@ -399,7 +399,7 @@ fn query_tasks_canceled_by() {
     let kind = index_creation_task("doggo", "sheep");
     let _ = index_scheduler.register(kind, None, false).unwrap();
     let kind = KindWithContent::IndexSwap {
-        swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()) }],
+        swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()), rename: false }],
     };
     let _task = index_scheduler.register(kind, None, false).unwrap();
 
@@ -414,7 +414,7 @@ fn query_tasks_canceled_by() {
     snapshot!(snapshot_index_scheduler(&index_scheduler), name: "start");
 
     let rtxn = index_scheduler.read_txn().unwrap();
-    let proc = index_scheduler.processing_tasks.read().unwrap();
+    let proc = &index_scheduler.runtime_tasks.read().unwrap().processing;
     let query = Query { canceled_by: Some(vec![task_cancelation.uid]), ..Query::default() };
     let (tasks, _) = index_scheduler
         .queue

crates/index-scheduler/src/queue/test.rs

@@ -203,26 +203,30 @@ fn test_disable_auto_deletion_of_tasks() {
         )
         .unwrap();
 
-    let rtxn = index_scheduler.env.read_txn().unwrap();
-    let proc = index_scheduler.processing_tasks.read().unwrap();
-    let tasks =
-        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
-    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
-    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]" }), name: "task_queue_is_full");
-    drop(rtxn);
-    drop(proc);
+    {
+        let rtxn = index_scheduler.env.read_txn().unwrap();
+        let proc = &index_scheduler.runtime_tasks.read().unwrap().processing;
+        let tasks = index_scheduler
+            .queue
+            .get_task_ids(&rtxn, &Query { ..Default::default() }, &proc)
+            .unwrap();
+        let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
+        snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]" }), name: "task_queue_is_full");
+    }
 
     // now we're above the max number of tasks
     // and if we try to advance in the tick function no new task deletion should be enqueued
     handle.advance_till([Start, BatchCreated]);
-    let rtxn = index_scheduler.env.read_txn().unwrap();
-    let proc = index_scheduler.processing_tasks.read().unwrap();
-    let tasks =
-        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
-    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
-    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "task_deletion_have_not_been_enqueued");
-    drop(rtxn);
-    drop(proc);
+    {
+        let rtxn = index_scheduler.env.read_txn().unwrap();
+        let proc = &index_scheduler.runtime_tasks.read().unwrap().processing;
+        let tasks = index_scheduler
+            .queue
+            .get_task_ids(&rtxn, &Query { ..Default::default() }, &proc)
+            .unwrap();
+        let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
+        snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "task_deletion_have_not_been_enqueued");
+    }
 }
 
 #[test]
@@ -267,59 +271,69 @@ fn test_auto_deletion_of_tasks() {
         )
         .unwrap();
 
-    let rtxn = index_scheduler.env.read_txn().unwrap();
-    let proc = index_scheduler.processing_tasks.read().unwrap();
-    let tasks =
-        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
-    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
-    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]" }), name: "task_queue_is_full");
-    drop(rtxn);
-    drop(proc);
+    {
+        let rtxn = index_scheduler.env.read_txn().unwrap();
+        let proc = &index_scheduler.runtime_tasks.read().unwrap().processing;
+        let tasks = index_scheduler
+            .queue
+            .get_task_ids(&rtxn, &Query { ..Default::default() }, &proc)
+            .unwrap();
+        let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
+        snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]" }), name: "task_queue_is_full");
+    }
 
-    // now we're above the max number of tasks
-    // and if we try to advance in the tick function a new task deletion should be enqueued
-    handle.advance_till([Start, BatchCreated]);
-    let rtxn = index_scheduler.env.read_txn().unwrap();
-    let proc = index_scheduler.processing_tasks.read().unwrap();
-    let tasks =
-        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
-    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
-    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "task_deletion_have_been_enqueued");
-    drop(rtxn);
-    drop(proc);
+    {
+        // now we're above the max number of tasks
+        // and if we try to advance in the tick function a new task deletion should be enqueued
+        handle.advance_till([Start, BatchCreated]);
+        let rtxn = index_scheduler.env.read_txn().unwrap();
+        let proc = &index_scheduler.runtime_tasks.read().unwrap().processing;
+        let tasks = index_scheduler
+            .queue
+            .get_task_ids(&rtxn, &Query { ..Default::default() }, &proc)
+            .unwrap();
+        let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
+        snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "task_deletion_have_been_enqueued");
+    }
 
-    handle.advance_till([InsideProcessBatch, ProcessBatchSucceeded, AfterProcessing]);
-    let rtxn = index_scheduler.env.read_txn().unwrap();
-    let proc = index_scheduler.processing_tasks.read().unwrap();
-    let tasks =
-        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
-    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
-    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "task_deletion_have_been_processed");
-    drop(rtxn);
-    drop(proc);
+    {
+        handle.advance_till([InsideProcessBatch, ProcessBatchSucceeded, AfterProcessing]);
+        let rtxn = index_scheduler.env.read_txn().unwrap();
+        let proc = &index_scheduler.runtime_tasks.read().unwrap().processing;
+        let tasks = index_scheduler
+            .queue
+            .get_task_ids(&rtxn, &Query { ..Default::default() }, &proc)
+            .unwrap();
+        let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
+        snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "task_deletion_have_been_processed");
+    }
 
     handle.advance_one_failed_batch();
     // a new task deletion has been enqueued
     handle.advance_one_successful_batch();
-    let rtxn = index_scheduler.env.read_txn().unwrap();
-    let proc = index_scheduler.processing_tasks.read().unwrap();
-    let tasks =
-        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
-    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
-    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "after_the_second_task_deletion");
-    drop(rtxn);
-    drop(proc);
+    {
+        let rtxn = index_scheduler.env.read_txn().unwrap();
+        let proc = &index_scheduler.runtime_tasks.read().unwrap().processing;
+        let tasks = index_scheduler
+            .queue
+            .get_task_ids(&rtxn, &Query { ..Default::default() }, &proc)
+            .unwrap();
+        let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
+        snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "after_the_second_task_deletion");
+    }
 
     handle.advance_one_failed_batch();
     handle.advance_one_successful_batch();
-    let rtxn = index_scheduler.env.read_txn().unwrap();
-    let proc = index_scheduler.processing_tasks.read().unwrap();
-    let tasks =
-        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
-    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
-    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "everything_has_been_processed");
-    drop(rtxn);
-    drop(proc);
+    {
+        let rtxn = index_scheduler.env.read_txn().unwrap();
+        let proc = &index_scheduler.runtime_tasks.read().unwrap().processing;
+        let tasks = index_scheduler
+            .queue
+            .get_task_ids(&rtxn, &Query { ..Default::default() }, &proc)
+            .unwrap();
+        let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
+        snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "everything_has_been_processed");
+    }
 }
 
 #[test]

crates/index-scheduler/src/scheduler/autobatcher.rs

@@ -68,13 +68,15 @@ impl From<KindWithContent> for AutobatchKind {
             KindWithContent::IndexCreation { .. } => AutobatchKind::IndexCreation,
             KindWithContent::IndexUpdate { .. } => AutobatchKind::IndexUpdate,
             KindWithContent::IndexSwap { .. } => AutobatchKind::IndexSwap,
-            KindWithContent::TaskCancelation { .. }
+            KindWithContent::IndexCompaction { .. }
+            | KindWithContent::TaskCancelation { .. }
             | KindWithContent::TaskDeletion { .. }
             | KindWithContent::DumpCreation { .. }
             | KindWithContent::Export { .. }
             | KindWithContent::UpgradeDatabase { .. }
+            | KindWithContent::NetworkTopologyChange(_)
             | KindWithContent::SnapshotCreation => {
-                panic!("The autobatcher should never be called with tasks that don't apply to an index.")
+                panic!("The autobatcher should never be called with tasks with special priority or that don't apply to an index.")
             }
         }
     }
@@ -288,7 +290,9 @@ impl BatchKind {
 
         match (self, autobatch_kind) {
             // We don't batch any of these operations
-            (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentEdition) => Break((this, BatchStopReason::TaskCannotBeBatched { kind, id })),
+            (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentEdition) => {
+                Break((this, BatchStopReason::TaskCannotBeBatched { kind, id }))
+            },
             // We must not batch tasks that don't have the same index creation rights if the index doesn't already exists.
             (this, kind) if !index_already_exists && this.allow_index_creation() == Some(false) && kind.allow_index_creation() == Some(true) => {
                 Break((this, BatchStopReason::IndexCreationMismatch { id }))

crates/index-scheduler/src/scheduler/autobatcher_test.rs

@@ -75,7 +75,11 @@ fn idx_create() -> KindWithContent {
 }
 
 fn idx_update() -> KindWithContent {
-    KindWithContent::IndexUpdate { index_uid: String::from("doggo"), primary_key: None }
+    KindWithContent::IndexUpdate {
+        index_uid: String::from("doggo"),
+        primary_key: None,
+        new_index_uid: None,
+    }
 }
 
 fn idx_del() -> KindWithContent {
@@ -84,7 +88,10 @@ fn idx_del() -> KindWithContent {
 
 fn idx_swap() -> KindWithContent {
     KindWithContent::IndexSwap {
-        swaps: vec![IndexSwap { indexes: (String::from("doggo"), String::from("catto")) }],
+        swaps: vec![IndexSwap {
+            indexes: (String::from("doggo"), String::from("catto")),
+            rename: false,
+        }],
     }
 }
 

crates/index-scheduler/src/scheduler/create_batch.rs

@@ -4,6 +4,7 @@ use std::io::ErrorKind;
 use meilisearch_types::heed::RoTxn;
 use meilisearch_types::milli::update::IndexDocumentsMethod;
 use meilisearch_types::settings::{Settings, Unchecked};
+use meilisearch_types::tasks::enterprise_edition::network::NetworkTopologyState;
 use meilisearch_types::tasks::{BatchStopReason, Kind, KindWithContent, Status, Task};
 use roaring::RoaringBitmap;
 use uuid::Uuid;
@@ -38,6 +39,7 @@ pub(crate) enum Batch {
     IndexUpdate {
         index_uid: String,
         primary_key: Option<String>,
+        new_index_uid: Option<String>,
         task: Task,
     },
     IndexDeletion {
@@ -54,6 +56,17 @@ pub(crate) enum Batch {
     UpgradeDatabase {
         tasks: Vec<Task>,
     },
+    IndexCompaction {
+        index_uid: String,
+        task: Task,
+    },
+    NetworkIndexBatch {
+        network_task: Task,
+        inner_batch: Box<Batch>,
+    },
+    NetworkWait {
+        task: Task,
+    },
 }
 
 #[derive(Debug)]
@@ -65,6 +78,7 @@ pub(crate) enum DocumentOperation {
 
 /// A [batch](Batch) that combines multiple tasks operating on an index.
 #[derive(Debug)]
+#[allow(clippy::large_enum_variant)]
 pub(crate) enum IndexOperation {
     DocumentOperation {
         index_uid: String,
@@ -108,7 +122,8 @@ impl Batch {
             | Batch::Dump(task)
             | Batch::IndexCreation { task, .. }
             | Batch::Export { task }
-            | Batch::IndexUpdate { task, .. } => {
+            | Batch::IndexUpdate { task, .. }
+            | Batch::IndexCompaction { task, .. } => {
                 RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap()
             }
             Batch::SnapshotCreation(tasks)
@@ -133,9 +148,14 @@ impl Batch {
                     ..
                 } => RoaringBitmap::from_iter(tasks.iter().chain(other).map(|task| task.uid)),
             },
-            Batch::IndexSwap { task } => {
+            Batch::IndexSwap { task } | Batch::NetworkWait { task } => {
                 RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap()
             }
+            Batch::NetworkIndexBatch { network_task, inner_batch } => {
+                let mut tasks = inner_batch.ids();
+                tasks.insert(network_task.uid);
+                tasks
+            }
         }
     }
 
@@ -149,11 +169,14 @@ impl Batch {
             | Dump(_)
             | Export { .. }
             | UpgradeDatabase { .. }
+            | NetworkWait { .. }
             | IndexSwap { .. } => None,
             IndexOperation { op, .. } => Some(op.index_uid()),
             IndexCreation { index_uid, .. }
             | IndexUpdate { index_uid, .. }
-            | IndexDeletion { index_uid, .. } => Some(index_uid),
+            | IndexDeletion { index_uid, .. }
+            | IndexCompaction { index_uid, .. } => Some(index_uid),
+            NetworkIndexBatch { network_task: _, inner_batch } => inner_batch.index_uid(),
         }
     }
 }
@@ -173,8 +196,11 @@ impl fmt::Display for Batch {
             Batch::IndexUpdate { .. } => f.write_str("IndexUpdate")?,
             Batch::IndexDeletion { .. } => f.write_str("IndexDeletion")?,
             Batch::IndexSwap { .. } => f.write_str("IndexSwap")?,
+            Batch::IndexCompaction { .. } => f.write_str("IndexCompaction")?,
             Batch::Export { .. } => f.write_str("Export")?,
             Batch::UpgradeDatabase { .. } => f.write_str("UpgradeDatabase")?,
+            Batch::NetworkIndexBatch { .. } => f.write_str("NetworkTopologyChange")?,
+            Batch::NetworkWait { .. } => f.write_str("NetworkTopologyChange")?,
         };
         match index_uid {
             Some(name) => f.write_fmt(format_args!(" on {name:?} from tasks: {tasks:?}")),
@@ -405,11 +431,13 @@ impl IndexScheduler {
                 let mut task =
                     self.queue.tasks.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?;
                 current_batch.processing(Some(&mut task));
-                let primary_key = match &task.kind {
-                    KindWithContent::IndexUpdate { primary_key, .. } => primary_key.clone(),
+                let (primary_key, new_index_uid) = match &task.kind {
+                    KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => {
+                        (primary_key.clone(), new_index_uid.clone())
+                    }
                     _ => unreachable!(),
                 };
-                Ok(Some(Batch::IndexUpdate { index_uid, primary_key, task }))
+                Ok(Some(Batch::IndexUpdate { index_uid, primary_key, new_index_uid, task }))
             }
             BatchKind::IndexDeletion { ids } => Ok(Some(Batch::IndexDeletion {
                 index_uid,
@@ -449,7 +477,6 @@ impl IndexScheduler {
         let mut current_batch = ProcessingBatch::new(batch_id);
 
         let enqueued = &self.queue.tasks.get_status(rtxn, Status::Enqueued)?;
-        let count_total_enqueued = enqueued.len();
         let failed = &self.queue.tasks.get_status(rtxn, Status::Failed)?;
 
         // 0. we get the last task to cancel.
@@ -508,17 +535,41 @@ impl IndexScheduler {
             return Ok(Some((Batch::TaskDeletions(tasks), current_batch)));
         }
 
-        // 3. we batch the export.
+        // 3. Check for enqueued network topology changes
+        let network_changes =
+            self.queue.tasks.get_kind(rtxn, Kind::NetworkTopologyChange)? & enqueued;
+        if let Some(task_id) = network_changes.iter().next() {
+            let task = self.queue.tasks.get_task(rtxn, task_id)?.unwrap();
+            return self.start_processing_network(rtxn, task, enqueued, current_batch);
+        }
+
+        // 4. we get the next task to compact
+        let to_compact = self.queue.tasks.get_kind(rtxn, Kind::IndexCompaction)? & enqueued;
+        if let Some(task_id) = to_compact.min() {
+            let mut task =
+                self.queue.tasks.get_task(rtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?;
+            current_batch.processing(Some(&mut task));
+            current_batch.reason(BatchStopReason::TaskCannotBeBatched {
+                kind: Kind::IndexCompaction,
+                id: task_id,
+            });
+            let index_uid =
+                task.index_uid().expect("Compaction task must have an index uid").to_owned();
+            return Ok(Some((Batch::IndexCompaction { index_uid, task }, current_batch)));
+        }
+
+        // 5. we batch the export.
         let to_export = self.queue.tasks.get_kind(rtxn, Kind::Export)? & enqueued;
         if !to_export.is_empty() {
             let task_id = to_export.iter().next().expect("There must be at least one export task");
             let mut task = self.queue.tasks.get_task(rtxn, task_id)?.unwrap();
             current_batch.processing([&mut task]);
-            current_batch.reason(BatchStopReason::TaskKindCannotBeBatched { kind: Kind::Export });
+            current_batch
+                .reason(BatchStopReason::TaskCannotBeBatched { kind: Kind::Export, id: task_id });
             return Ok(Some((Batch::Export { task }, current_batch)));
         }
 
-        // 4. we batch the snapshot.
+        // 6. we batch the snapshot.
         let to_snapshot = self.queue.tasks.get_kind(rtxn, Kind::SnapshotCreation)? & enqueued;
         if !to_snapshot.is_empty() {
             let mut tasks = self.queue.tasks.get_existing_tasks(rtxn, to_snapshot)?;
@@ -528,7 +579,7 @@ impl IndexScheduler {
             return Ok(Some((Batch::SnapshotCreation(tasks), current_batch)));
         }
 
-        // 5. we batch the dumps.
+        // 7. we batch the dumps.
         let to_dump = self.queue.tasks.get_kind(rtxn, Kind::DumpCreation)? & enqueued;
         if let Some(to_dump) = to_dump.min() {
             let mut task =
@@ -541,8 +592,25 @@ impl IndexScheduler {
             return Ok(Some((Batch::Dump(task), current_batch)));
         }
 
-        // 6. We make a batch from the unprioritised tasks. Start by taking the next enqueued task.
-        let task_id = if let Some(task_id) = enqueued.min() { task_id } else { return Ok(None) };
+        // 8. We make a batch from the unprioritised tasks. Start by taking the next enqueued task.
+        let (batch, current_batch) =
+            self.create_next_batch_unprioritized(rtxn, &enqueued, current_batch)?;
+        Ok(batch.map(|batch| (batch, current_batch)))
+    }
+
+    fn create_next_batch_unprioritized(
+        &self,
+        rtxn: &RoTxn,
+        enqueued: &RoaringBitmap,
+        mut current_batch: ProcessingBatch,
+    ) -> Result<(Option<Batch>, ProcessingBatch)> {
+        let count_total_enqueued = enqueued.len();
+
+        let task_id = if let Some(task_id) = enqueued.min() {
+            task_id
+        } else {
+            return Ok((None, current_batch));
+        };
         let mut task =
             self.queue.tasks.get_task(rtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?;
 
@@ -559,7 +627,7 @@ impl IndexScheduler {
                 kind: Kind::IndexSwap,
                 id: task.uid,
             });
-            return Ok(Some((Batch::IndexSwap { task }, current_batch)));
+            return Ok((Some(Batch::IndexSwap { task }), current_batch));
         };
 
         let index_already_exists = self.index_mapper.exists(rtxn, index_name)?;
@@ -624,7 +692,7 @@ impl IndexScheduler {
             autobatcher::autobatch(enqueued, index_already_exists, primary_key.as_deref())
         {
             current_batch.reason(autobatch_stop_reason.unwrap_or(stop_reason));
-            return Ok(self
+            let batch = self
                 .create_next_batch_index(
                     rtxn,
                     index_name.to_string(),
@@ -632,11 +700,127 @@ impl IndexScheduler {
                     &mut current_batch,
                     create_index,
                 )?
-                .map(|batch| (batch, current_batch)));
+                .map(|batch| batch);
+            return Ok((batch, current_batch));
         }
 
         // If we found no tasks then we were notified for something that got autobatched
         // somehow and there is nothing to do.
-        Ok(None)
+        Ok((None, current_batch))
+    }
+
+    fn start_processing_network(
+        &self,
+        rtxn: &RoTxn,
+        mut task: Task,
+        enqueued: &RoaringBitmap,
+        current_batch: ProcessingBatch,
+    ) -> Result<Option<(Batch, ProcessingBatch)>> {
+        let change_version =
+            task.network.as_ref().map(|network| network.network_version()).unwrap_or_default();
+        let KindWithContent::NetworkTopologyChange(network_topology_change) = &task.kind else {
+            panic!("inconsistent kind with content")
+        };
+        match network_topology_change.state() {
+            NetworkTopologyState::WaitingForOlderTasks => {
+                let mut old_tasks = RoaringBitmap::new();
+                for task_id in enqueued {
+                    let task = self
+                        .queue
+                        .tasks
+                        .get_task(rtxn, task_id)?
+                        .ok_or(Error::CorruptedTaskQueue)?;
+
+                    let has_index = task.index_uid().is_some();
+
+                    if !has_index {
+                        continue;
+                    }
+
+                    let has_older_network_version = task
+                        .network
+                        .map(|network| network.network_version() <= change_version)
+                        // if there is no version, we never retain the task
+                        .unwrap_or_default();
+
+                    if has_older_network_version {
+                        old_tasks.push(task_id);
+                    }
+                }
+
+                let res = self.create_next_batch_unprioritized(rtxn, &old_tasks, current_batch);
+                self.runtime_tasks.write().unwrap().enqueued_network.swap(old_tasks);
+
+                let (batch, mut current_batch) = res?;
+
+                current_batch.processing(Some(&mut task));
+
+                let batch = match batch {
+                    Some(batch) => {
+                        let inner_batch = Box::new(batch);
+
+                        Batch::NetworkIndexBatch { network_task: task, inner_batch }
+                    }
+                    None => Batch::NetworkWait { task },
+                };
+
+                Ok(Some((batch, current_batch)))
+            }
+            NetworkTopologyState::ImportingDocuments => {
+                let mut import_tasks = RoaringBitmap::new();
+                for task_id in enqueued {
+                    let task = self
+                        .queue
+                        .tasks
+                        .get_task(rtxn, task_id)?
+                        .ok_or(Error::CorruptedTaskQueue)?;
+
+                    let has_index = task.index_uid().is_some();
+
+                    if !has_index {
+                        continue;
+                    }
+
+                    let is_import_task = task
+                        .network
+                        .map(|network| {
+                            network.network_version() == change_version
+                                && network.import_data().is_some()
+                        })
+                        // if there is no version, we never retain the task
+                        .unwrap_or_default();
+
+                    if is_import_task {
+                        import_tasks.push(task_id);
+                    }
+                }
+
+                let res = self.create_next_batch_unprioritized(rtxn, &import_tasks, current_batch);
+                self.runtime_tasks.write().unwrap().enqueued_network.swap(import_tasks);
+
+                let (batch, mut current_batch) = res?;
+
+                current_batch.processing(Some(&mut task));
+
+                let batch = match batch {
+                    Some(batch) => {
+                        let inner_batch = Box::new(batch);
+
+                        Batch::NetworkIndexBatch { network_task: task, inner_batch }
+                    }
+                    None => Batch::NetworkWait { task },
+                };
+
+                Ok(Some((batch, current_batch)))
+            }
+            NetworkTopologyState::ExportingDocuments | NetworkTopologyState::Finished => {
+                Ok(Some((Batch::NetworkWait { task }, current_batch)))
+            }
+        }
     }
 }
+
+pub enum BatchOutcome {
+    NoTaskToProcess,
+    Batch { batch: Batch, processing: ProcessingBatch },
+}

crates/index-scheduler/src/scheduler/mod.rs

@@ -25,6 +25,7 @@ use convert_case::{Case, Casing as _};
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::heed::{Env, WithoutTls};
 use meilisearch_types::milli;
+use meilisearch_types::milli::update::S3SnapshotOptions;
 use meilisearch_types::tasks::Status;
 use process_batch::ProcessBatchInfo;
 use rayon::current_num_threads;
@@ -87,11 +88,14 @@ pub struct Scheduler {
 
     /// Snapshot compaction status.
     pub(crate) experimental_no_snapshot_compaction: bool,
+
+    /// S3 Snapshot options.
+    pub(crate) s3_snapshot_options: Option<S3SnapshotOptions>,
 }
 
 impl Scheduler {
-    pub(crate) fn private_clone(&self) -> Scheduler {
-        Scheduler {
+    pub(crate) fn private_clone(&self) -> Self {
+        Self {
             must_stop_processing: self.must_stop_processing.clone(),
             wake_up: self.wake_up.clone(),
             autobatching_enabled: self.autobatching_enabled,
@@ -103,23 +107,52 @@ impl Scheduler {
             version_file_path: self.version_file_path.clone(),
             embedding_cache_cap: self.embedding_cache_cap,
             experimental_no_snapshot_compaction: self.experimental_no_snapshot_compaction,
+            s3_snapshot_options: self.s3_snapshot_options.clone(),
         }
     }
 
     pub fn new(options: &IndexSchedulerOptions, auth_env: Env<WithoutTls>) -> Scheduler {
+        let IndexSchedulerOptions {
+            version_file_path,
+            auth_path: _,
+            tasks_path: _,
+            update_file_path: _,
+            indexes_path: _,
+            snapshots_path,
+            dumps_path,
+            cli_webhook_url: _,
+            cli_webhook_authorization: _,
+            task_db_size: _,
+            index_base_map_size: _,
+            enable_mdb_writemap: _,
+            index_growth_amount: _,
+            index_count: _,
+            indexer_config,
+            autobatching_enabled,
+            cleanup_enabled: _,
+            max_number_of_tasks: _,
+            max_number_of_batched_tasks,
+            batched_tasks_size_limit,
+            instance_features: _,
+            auto_upgrade: _,
+            embedding_cache_cap,
+            experimental_no_snapshot_compaction,
+        } = options;
+
         Scheduler {
             must_stop_processing: MustStopProcessing::default(),
             // we want to start the loop right away in case meilisearch was ctrl+Ced while processing things
             wake_up: Arc::new(SignalEvent::auto(true)),
-            autobatching_enabled: options.autobatching_enabled,
-            max_number_of_batched_tasks: options.max_number_of_batched_tasks,
-            batched_tasks_size_limit: options.batched_tasks_size_limit,
-            dumps_path: options.dumps_path.clone(),
-            snapshots_path: options.snapshots_path.clone(),
+            autobatching_enabled: *autobatching_enabled,
+            max_number_of_batched_tasks: *max_number_of_batched_tasks,
+            batched_tasks_size_limit: *batched_tasks_size_limit,
+            dumps_path: dumps_path.clone(),
+            snapshots_path: snapshots_path.clone(),
             auth_env,
-            version_file_path: options.version_file_path.clone(),
-            embedding_cache_cap: options.embedding_cache_cap,
-            experimental_no_snapshot_compaction: options.experimental_no_snapshot_compaction,
+            version_file_path: version_file_path.clone(),
+            embedding_cache_cap: *embedding_cache_cap,
+            experimental_no_snapshot_compaction: *experimental_no_snapshot_compaction,
+            s3_snapshot_options: indexer_config.s3_snapshot_options.clone(),
         }
     }
 }
@@ -167,9 +200,10 @@ impl IndexScheduler {
         // We reset the must_stop flag to be sure that we don't stop processing tasks
         self.scheduler.must_stop_processing.reset();
         let progress = self
-            .processing_tasks
+            .runtime_tasks
             .write()
             .unwrap()
+            .processing
             // We can clone the processing batch here because we don't want its modification to affect the view of the processing batches
             .start_processing(processing_batch.clone(), ids.clone());
 
@@ -268,7 +302,7 @@ impl IndexScheduler {
 
                     self.queue
                         .tasks
-                        .update_task(&mut wtxn, &task)
+                        .update_task(&mut wtxn, &mut task)
                         .map_err(|e| Error::UnrecoverableError(Box::new(e)))?;
                 }
                 if let Some(canceled_by) = canceled_by {
@@ -349,7 +383,7 @@ impl IndexScheduler {
 
                     self.queue
                         .tasks
-                        .update_task(&mut wtxn, &task)
+                        .update_task(&mut wtxn, &mut task)
                         .map_err(|e| Error::UnrecoverableError(Box::new(e)))?;
                 }
             }
@@ -420,7 +454,7 @@ impl IndexScheduler {
 
         // We should stop processing AFTER everything is processed and written to disk otherwise, a batch (which only lives in RAM) may appear in the processing task
         // and then become « not found » for some time until the commit everything is written and the final commit is made.
-        self.processing_tasks.write().unwrap().stop_processing();
+        self.runtime_tasks.write().unwrap().processing.stop_processing();
 
         // Once the tasks are committed, we should delete all the update files associated ASAP to avoid leaking files in case of a restart
         tracing::debug!("Deleting the update files");

crates/index-scheduler/src/scheduler/process_batch.rs

@@ -1,28 +1,45 @@
-use std::collections::{BTreeSet, HashMap, HashSet};
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
+use std::fs::{remove_file, File};
+use std::io::{ErrorKind, Seek, SeekFrom};
 use std::panic::{catch_unwind, AssertUnwindSafe};
 use std::sync::atomic::Ordering;
+use std::time::Duration;
 
+use bumpalo::Bump;
+use byte_unit::Byte;
 use meilisearch_types::batches::{BatchEnqueuedAt, BatchId};
+use meilisearch_types::enterprise_edition::network::Remote;
 use meilisearch_types::heed::{RoTxn, RwTxn};
-use meilisearch_types::milli::progress::{Progress, VariableNameStep};
+use meilisearch_types::milli::documents::PrimaryKey;
+use meilisearch_types::milli::heed::CompactionOption;
+use meilisearch_types::milli::progress::{EmbedderStats, Progress, VariableNameStep};
+use meilisearch_types::milli::update::new::indexer;
+use meilisearch_types::milli::update::new::indexer::enterprise_edition::sharding::Shards;
 use meilisearch_types::milli::{self, ChannelCongestion};
+use meilisearch_types::tasks::enterprise_edition::network::{NetworkTopologyState, Origin};
 use meilisearch_types::tasks::{Details, IndexSwap, Kind, KindWithContent, Status, Task};
 use meilisearch_types::versioning::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH};
 use milli::update::Settings as MilliSettings;
 use roaring::RoaringBitmap;
+use tempfile::{PersistError, TempPath};
+use time::OffsetDateTime;
 
 use super::create_batch::Batch;
 use crate::processing::{
     AtomicBatchStep, AtomicTaskStep, CreateIndexProgress, DeleteIndexProgress, FinalizingIndexStep,
-    InnerSwappingTwoIndexes, SwappingTheIndexes, TaskCancelationProgress, TaskDeletionProgress,
-    UpdateIndexProgress,
+    IndexCompaction, InnerSwappingTwoIndexes, SwappingTheIndexes, TaskCancelationProgress,
+    TaskDeletionProgress, UpdateIndexProgress,
 };
+use crate::scheduler::process_export::{ExportContext, ExportOptions, TargetInstance};
 use crate::utils::{
     self, remove_n_tasks_datetime_earlier_than, remove_task_datetime, swap_index_uid_in_task,
     ProcessingBatch,
 };
 use crate::{Error, IndexScheduler, Result, TaskId};
 
+/// The name of the copy of the data.mdb file used during compaction.
+const DATA_MDB_COPY_NAME: &str = "data.mdb.cpy";
+
 #[derive(Debug, Default)]
 pub struct ProcessBatchInfo {
     /// The write channel congestion. None when unavailable: settings update.
@@ -146,7 +163,6 @@ impl IndexScheduler {
                 };
 
                 let mut index_wtxn = index.write_txn()?;
-
                 let index_version = index.get_version(&index_wtxn)?.unwrap_or((1, 12, 0));
                 let package_version = (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH);
                 if index_version != package_version {
@@ -224,24 +240,46 @@ impl IndexScheduler {
                 self.index_mapper.create_index(wtxn, &index_uid, None)?;
 
                 self.process_batch(
-                    Batch::IndexUpdate { index_uid, primary_key, task },
+                    Batch::IndexUpdate { index_uid, primary_key, new_index_uid: None, task },
                     current_batch,
                     progress,
                 )
             }
-            Batch::IndexUpdate { index_uid, primary_key, mut task } => {
+            Batch::IndexUpdate { index_uid, primary_key, new_index_uid, mut task } => {
                 progress.update_progress(UpdateIndexProgress::UpdatingTheIndex);
+
+                // Get the index (renamed or not)
                 let rtxn = self.env.read_txn()?;
                 let index = self.index_mapper.index(&rtxn, &index_uid)?;
+                let mut index_wtxn = index.write_txn()?;
 
-                if let Some(primary_key) = primary_key.clone() {
-                    let mut index_wtxn = index.write_txn()?;
+                // Handle rename if new_index_uid is provided
+                let final_index_uid = if let Some(new_uid) = &new_index_uid {
+                    if new_uid != &index_uid {
+                        index.set_updated_at(&mut index_wtxn, &OffsetDateTime::now_utc())?;
+
+                        let mut wtxn = self.env.write_txn()?;
+                        self.apply_index_swap(
+                            &mut wtxn, &progress, task.uid, &index_uid, new_uid, true,
+                        )?;
+                        wtxn.commit()?;
+
+                        new_uid.clone()
+                    } else {
+                        new_uid.clone()
+                    }
+                } else {
+                    index_uid.clone()
+                };
+
+                // Handle primary key update if provided
+                if let Some(ref primary_key) = primary_key {
                     let mut builder = MilliSettings::new(
                         &mut index_wtxn,
                         &index,
                         self.index_mapper.indexer_config(),
                     );
-                    builder.set_primary_key(primary_key);
+                    builder.set_primary_key(primary_key.clone());
                     let must_stop_processing = self.scheduler.must_stop_processing.clone();
 
                     builder
@@ -250,15 +288,20 @@ impl IndexScheduler {
                             &progress,
                             current_batch.embedder_stats.clone(),
                         )
-                        .map_err(|e| Error::from_milli(e, Some(index_uid.to_string())))?;
-                    index_wtxn.commit()?;
+                        .map_err(|e| Error::from_milli(e, Some(final_index_uid.to_string())))?;
                 }
 
+                index_wtxn.commit()?;
                 // drop rtxn before starting a new wtxn on the same db
                 rtxn.commit()?;
 
                 task.status = Status::Succeeded;
-                task.details = Some(Details::IndexInfo { primary_key });
+                task.details = Some(Details::IndexInfo {
+                    primary_key: primary_key.clone(),
+                    new_index_uid: new_index_uid.clone(),
+                    // we only display the old index uid if a rename happened => there is a new_index_uid
+                    old_index_uid: new_index_uid.map(|_| index_uid.clone()),
+                });
 
                 // if the update processed successfully, we're going to store the new
                 // stats of the index. Since the tasks have already been processed and
@@ -268,8 +311,8 @@ impl IndexScheduler {
                     let mut wtxn = self.env.write_txn()?;
                     let index_rtxn = index.read_txn()?;
                     let stats = crate::index_mapper::IndexStats::new(&index, &index_rtxn)
-                        .map_err(|e| Error::from_milli(e, Some(index_uid.clone())))?;
-                    self.index_mapper.store_stats_of(&mut wtxn, &index_uid, &stats)?;
+                        .map_err(|e| Error::from_milli(e, Some(final_index_uid.clone())))?;
+                    self.index_mapper.store_stats_of(&mut wtxn, &final_index_uid, &stats)?;
                     wtxn.commit()?;
                     Ok(())
                 }();
@@ -330,13 +373,18 @@ impl IndexScheduler {
                     unreachable!()
                 };
                 let mut not_found_indexes = BTreeSet::new();
-                for IndexSwap { indexes: (lhs, rhs) } in swaps {
-                    for index in [lhs, rhs] {
-                        let index_exists = self.index_mapper.index_exists(&wtxn, index)?;
-                        if !index_exists {
-                            not_found_indexes.insert(index);
-                        }
+                let mut found_indexes_but_should_not = BTreeSet::new();
+                for IndexSwap { indexes: (lhs, rhs), rename } in swaps {
+                    let index_exists = self.index_mapper.index_exists(&wtxn, lhs)?;
+                    if !index_exists {
+                        not_found_indexes.insert(lhs);
                     }
+                    let index_exists = self.index_mapper.index_exists(&wtxn, rhs)?;
+                    match (index_exists, rename) {
+                        (true, true) => found_indexes_but_should_not.insert((lhs, rhs)),
+                        (false, false) => not_found_indexes.insert(rhs),
+                        (true, false) | (false, true) => true, // random value we don't read it anyway
+                    };
                 }
                 if !not_found_indexes.is_empty() {
                     if not_found_indexes.len() == 1 {
@@ -349,6 +397,23 @@ impl IndexScheduler {
                         ));
                     }
                 }
+                if !found_indexes_but_should_not.is_empty() {
+                    if found_indexes_but_should_not.len() == 1 {
+                        let (lhs, rhs) = found_indexes_but_should_not
+                            .into_iter()
+                            .next()
+                            .map(|(lhs, rhs)| (lhs.clone(), rhs.clone()))
+                            .unwrap();
+                        return Err(Error::SwapIndexFoundDuringRename(lhs, rhs));
+                    } else {
+                        return Err(Error::SwapIndexesFoundDuringRename(
+                            found_indexes_but_should_not
+                                .into_iter()
+                                .map(|(_, rhs)| rhs.to_string())
+                                .collect(),
+                        ));
+                    }
+                }
                 progress.update_progress(SwappingTheIndexes::SwappingTheIndexes);
                 for (step, swap) in swaps.iter().enumerate() {
                     progress.update_progress(VariableNameStep::<SwappingTheIndexes>::new(
@@ -362,12 +427,54 @@ impl IndexScheduler {
                         task.uid,
                         &swap.indexes.0,
                         &swap.indexes.1,
+                        swap.rename,
                     )?;
                 }
                 wtxn.commit()?;
                 task.status = Status::Succeeded;
                 Ok((vec![task], ProcessBatchInfo::default()))
             }
+            Batch::IndexCompaction { index_uid: _, mut task } => {
+                let KindWithContent::IndexCompaction { index_uid } = &task.kind else {
+                    unreachable!()
+                };
+
+                let rtxn = self.env.read_txn()?;
+                let ret = catch_unwind(AssertUnwindSafe(|| {
+                    self.apply_compaction(&rtxn, &progress, index_uid)
+                }));
+
+                let (pre_size, post_size) = match ret {
+                    Ok(Ok(stats)) => stats,
+                    Ok(Err(Error::AbortedTask)) => return Err(Error::AbortedTask),
+                    Ok(Err(e)) => return Err(e),
+                    Err(e) => {
+                        let msg = match e.downcast_ref::<&'static str>() {
+                            Some(s) => *s,
+                            None => match e.downcast_ref::<String>() {
+                                Some(s) => &s[..],
+                                None => "Box<dyn Any>",
+                            },
+                        };
+                        return Err(Error::Export(Box::new(Error::ProcessBatchPanicked(
+                            msg.to_string(),
+                        ))));
+                    }
+                };
+
+                task.status = Status::Succeeded;
+                if let Some(Details::IndexCompaction {
+                    index_uid: _,
+                    pre_compaction_size,
+                    post_compaction_size,
+                }) = task.details.as_mut()
+                {
+                    *pre_compaction_size = Some(Byte::from_u64(pre_size));
+                    *post_compaction_size = Some(Byte::from_u64(post_size));
+                }
+
+                Ok((vec![task], ProcessBatchInfo::default()))
+            }
             Batch::Export { mut task } => {
                 let KindWithContent::Export { url, api_key, payload_size, indexes } = &task.kind
                 else {
@@ -440,9 +547,295 @@ impl IndexScheduler {
 
                 Ok((tasks, ProcessBatchInfo::default()))
             }
+            Batch::NetworkIndexBatch { mut network_task, inner_batch } => {
+                let (mut tasks, info) =
+                    self.process_batch(*inner_batch, current_batch, progress)?;
+                let KindWithContent::NetworkTopologyChange(network_topology_change) =
+                    &mut network_task.kind
+                else {
+                    return Err(Error::CorruptedTaskQueue);
+                };
+                for task in &tasks {
+                    let Some(network) = task.network.as_ref() else {
+                        continue;
+                    };
+                    let Some(import) = network.import_data() else {
+                        continue;
+                    };
+                    network_topology_change.process_remote_tasks(
+                        &import.remote_name,
+                        &import.index_name,
+                        import.document_count,
+                    );
+                }
+                tasks.push(network_task);
+                Ok((tasks, info))
+            }
+            Batch::NetworkWait { mut task } => {
+                let KindWithContent::NetworkTopologyChange(network_topology_change) =
+                    &mut task.kind
+                else {
+                    tracing::error!("network topology change task has the wrong kind with content");
+                    return Err(Error::CorruptedTaskQueue);
+                };
+
+                let Some(task_network) = &task.network else {
+                    tracing::error!("network topology change task has no network");
+                    return Err(Error::CorruptedTaskQueue);
+                };
+
+                let origin;
+                let origin = match task_network.origin() {
+                    Some(origin) => origin,
+                    None => {
+                        let myself =
+                            network_topology_change.in_name().expect("origin is not the leader");
+                        origin = Origin {
+                            remote_name: myself.to_string(),
+                            task_uid: task.uid,
+                            network_version: task_network.network_version(),
+                        };
+                        &origin
+                    }
+                };
+
+                if let Some((remotes, out_name)) = network_topology_change.export_to_process() {
+                    self.balance_documents(
+                        remotes,
+                        out_name,
+                        network_topology_change.in_name(),
+                        origin,
+                        &progress,
+                        &self.scheduler.must_stop_processing,
+                    )?;
+                }
+                network_topology_change.update_state();
+                if network_topology_change.state() == NetworkTopologyState::Finished {
+                    task.status = Status::Succeeded;
+                }
+                Ok((vec![task], Default::default()))
+            }
         }
     }
 
+    fn balance_documents(
+        &self,
+        remotes: &BTreeMap<String, Remote>,
+        out_name: &str,
+        in_name: Option<&str>,
+        network_change_origin: &Origin,
+        progress: &Progress,
+        must_stop_processing: &crate::scheduler::MustStopProcessing,
+    ) -> crate::Result<()> {
+        let new_shards = Shards::from_remotes_local(remotes.keys().map(String::as_str), in_name);
+
+        // TECHDEBT: this spawns a `ureq` agent additionally to `reqwest`. We probably want to harmonize all of this.
+        let agent = ureq::AgentBuilder::new().timeout(Duration::from_secs(5)).build();
+
+        let mut indexer_alloc = Bump::new();
+
+        let scheduler_rtxn = self.env.read_txn()?;
+
+        let index_count = self.index_mapper.index_count(&scheduler_rtxn)?;
+
+        // process by batches of 20MiB. Allow for compression? Don't forget about embeddings
+        let _: Vec<()> = self.index_mapper.try_for_each_index(
+            &scheduler_rtxn,
+            |index_uid, index| -> crate::Result<()> {
+                indexer_alloc.reset();
+                let err = |err| Error::from_milli(err, Some(index_uid.to_string()));
+                let index_rtxn = index.read_txn()?;
+                let all_docids = index.external_documents_ids();
+                let mut documents_to_move_to: HashMap<String, RoaringBitmap> = HashMap::new();
+                let mut documents_to_delete = RoaringBitmap::new();
+
+                for res in all_docids.iter(&index_rtxn)? {
+                    let (external_docid, docid) = res?;
+                    match new_shards.processing_shard(external_docid) {
+                        Some(shard) if shard.is_own => continue,
+                        Some(shard) => {
+                            documents_to_move_to
+                                .entry(shard.name.clone())
+                                .or_default()
+                                .insert(docid);
+                        }
+                        None => {
+                            documents_to_delete.insert(docid);
+                        }
+                    }
+                }
+
+                let fields_ids_map = index.fields_ids_map(&index_rtxn)?;
+
+                for (remote, documents_to_move) in documents_to_move_to {
+                    /// TODO: justify the unwrap
+                    let remote = remotes.get(&remote).unwrap();
+
+                    let target = TargetInstance {
+                        base_url: &remote.url,
+                        api_key: remote.write_api_key.as_deref(),
+                    };
+                    let options = ExportOptions {
+                        index_uid,
+                        payload_size: None,
+                        override_settings: false,
+                        export_mode: super::process_export::ExportMode::NetworkBalancing {
+                            index_count,
+                            export_old_remote_name: out_name,
+                            network_change_origin,
+                        },
+                    };
+                    let ctx = ExportContext {
+                        index,
+                        index_rtxn: &index_rtxn,
+                        universe: &documents_to_move,
+                        progress,
+                        agent: &agent,
+                        must_stop_processing,
+                    };
+
+                    self.export_one_index(target, options, ctx)?;
+
+                    documents_to_delete |= documents_to_move;
+                }
+
+                if documents_to_delete.is_empty() {
+                    return Ok(());
+                }
+
+                let mut new_fields_ids_map = fields_ids_map.clone();
+
+                // candidates not empty => index not empty => a primary key is set
+                let primary_key = index.primary_key(&index_rtxn)?.unwrap();
+
+                let primary_key = PrimaryKey::new_or_insert(primary_key, &mut new_fields_ids_map)
+                    .map_err(milli::Error::from)
+                    .map_err(err)?;
+
+                let mut index_wtxn = index.write_txn()?;
+
+                let mut indexer = indexer::DocumentDeletion::new();
+                indexer.delete_documents_by_docids(documents_to_delete);
+                let document_changes = indexer.into_changes(&indexer_alloc, primary_key);
+                let embedders = index
+                    .embedding_configs()
+                    .embedding_configs(&index_wtxn)
+                    .map_err(milli::Error::from)
+                    .map_err(err)?;
+                let embedders = self.embedders(index_uid.to_string(), embedders)?;
+                let indexer_config = self.index_mapper.indexer_config();
+                let pool = &indexer_config.thread_pool;
+
+                indexer::index(
+                    &mut index_wtxn,
+                    index,
+                    pool,
+                    indexer_config.grenad_parameters(),
+                    &fields_ids_map,
+                    new_fields_ids_map,
+                    None, // document deletion never changes primary key
+                    &document_changes,
+                    embedders,
+                    &|| must_stop_processing.get(),
+                    &progress,
+                    &EmbedderStats::default(),
+                )
+                .map_err(err)?;
+
+                index_wtxn.commit()?;
+
+                Ok(())
+            },
+        )?;
+        Ok(())
+    }
+
+    fn apply_compaction(
+        &self,
+        rtxn: &RoTxn,
+        progress: &Progress,
+        index_uid: &str,
+    ) -> Result<(u64, u64)> {
+        // 1. Verify that the index exists
+        if !self.index_mapper.index_exists(rtxn, index_uid)? {
+            return Err(Error::IndexNotFound(index_uid.to_owned()));
+        }
+
+        // 2. We retrieve the index and create a temporary file in the index directory
+        progress.update_progress(IndexCompaction::RetrieveTheIndex);
+        let index = self.index_mapper.index(rtxn, index_uid)?;
+
+        // the index operation can take a long time, so save this handle to make it available to the search for the duration of the tick
+        self.index_mapper
+            .set_currently_updating_index(Some((index_uid.to_string(), index.clone())));
+
+        progress.update_progress(IndexCompaction::CreateTemporaryFile);
+        let src_path = index.path().join("data.mdb");
+        let pre_size = std::fs::metadata(&src_path)?.len();
+
+        let dst_path = TempPath::from_path(index.path().join(DATA_MDB_COPY_NAME));
+        let file = File::create(&dst_path)?;
+        let mut file = tempfile::NamedTempFile::from_parts(file, dst_path);
+
+        // 3. We copy the index data to the temporary file
+        progress.update_progress(IndexCompaction::CopyAndCompactTheIndex);
+        index
+            .copy_to_file(file.as_file_mut(), CompactionOption::Enabled)
+            .map_err(|error| Error::Milli { error, index_uid: Some(index_uid.to_string()) })?;
+        // ...and reset the file position as specified in the documentation
+        file.seek(SeekFrom::Start(0))?;
+
+        // 4. We replace the index data file with the temporary file
+        progress.update_progress(IndexCompaction::PersistTheCompactedIndex);
+        match file.persist(src_path) {
+            Ok(file) => file.sync_all()?,
+            // TODO see if we have a _resource busy_ error and probably handle this by:
+            //      1. closing the index, 2. replacing and 3. reopening it
+            Err(PersistError { error, file: _ }) => return Err(Error::IoError(error)),
+        };
+
+        // 5. Prepare to close the index
+        progress.update_progress(IndexCompaction::CloseTheIndex);
+
+        // unmark that the index is the processing one so we don't keep a handle to it, preventing its closing
+        self.index_mapper.set_currently_updating_index(None);
+
+        self.index_mapper.close_index(rtxn, index_uid)?;
+        drop(index);
+
+        progress.update_progress(IndexCompaction::ReopenTheIndex);
+        // 6. Reopen the index
+        // The index will use the compacted data file when being reopened
+        let index = self.index_mapper.index(rtxn, index_uid)?;
+
+        // if the update processed successfully, we're going to store the new
+        // stats of the index. Since the tasks have already been processed and
+        // this is a non-critical operation. If it fails, we should not fail
+        // the entire batch.
+        let res = || -> Result<_> {
+            let mut wtxn = self.env.write_txn()?;
+            let index_rtxn = index.read_txn()?;
+            let stats = crate::index_mapper::IndexStats::new(&index, &index_rtxn)
+                .map_err(|e| Error::from_milli(e, Some(index_uid.to_string())))?;
+            self.index_mapper.store_stats_of(&mut wtxn, index_uid, &stats)?;
+            wtxn.commit()?;
+            Ok(stats.database_size)
+        }();
+
+        let post_size = match res {
+            Ok(post_size) => post_size,
+            Err(e) => {
+                tracing::error!(
+                    error = &e as &dyn std::error::Error,
+                    "Could not write the stats of the index"
+                );
+                0
+            }
+        };
+
+        Ok((pre_size, post_size))
+    }
+
     /// Swap the index `lhs` with the index `rhs`.
     fn apply_index_swap(
         &self,
@@ -451,6 +844,7 @@ impl IndexScheduler {
         task_id: u32,
         lhs: &str,
         rhs: &str,
+        rename: bool,
     ) -> Result<()> {
         progress.update_progress(InnerSwappingTwoIndexes::RetrieveTheTasks);
         // 1. Verify that both lhs and rhs are existing indexes
@@ -458,16 +852,23 @@ impl IndexScheduler {
         if !index_lhs_exists {
             return Err(Error::IndexNotFound(lhs.to_owned()));
         }
-        let index_rhs_exists = self.index_mapper.index_exists(wtxn, rhs)?;
-        if !index_rhs_exists {
-            return Err(Error::IndexNotFound(rhs.to_owned()));
+        if !rename {
+            let index_rhs_exists = self.index_mapper.index_exists(wtxn, rhs)?;
+            if !index_rhs_exists {
+                return Err(Error::IndexNotFound(rhs.to_owned()));
+            }
         }
 
         // 2. Get the task set for index = name that appeared before the index swap task
         let mut index_lhs_task_ids = self.queue.tasks.index_tasks(wtxn, lhs)?;
         index_lhs_task_ids.remove_range(task_id..);
-        let mut index_rhs_task_ids = self.queue.tasks.index_tasks(wtxn, rhs)?;
-        index_rhs_task_ids.remove_range(task_id..);
+        let index_rhs_task_ids = if rename {
+            let mut index_rhs_task_ids = self.queue.tasks.index_tasks(wtxn, rhs)?;
+            index_rhs_task_ids.remove_range(task_id..);
+            index_rhs_task_ids
+        } else {
+            RoaringBitmap::new()
+        };
 
         // 3. before_name -> new_name in the task's KindWithContent
         progress.update_progress(InnerSwappingTwoIndexes::UpdateTheTasks);
@@ -496,7 +897,11 @@ impl IndexScheduler {
         })?;
 
         // 6. Swap in the index mapper
-        self.index_mapper.swap(wtxn, lhs, rhs)?;
+        if rename {
+            self.index_mapper.rename(wtxn, lhs, rhs)?;
+        } else {
+            self.index_mapper.swap(wtxn, lhs, rhs)?;
+        }
 
         Ok(())
     }
@@ -514,7 +919,7 @@ impl IndexScheduler {
 
         // 1. Remove from this list the tasks that we are not allowed to delete
         let enqueued_tasks = self.queue.tasks.get_status(wtxn, Status::Enqueued)?;
-        let processing_tasks = &self.processing_tasks.read().unwrap().processing.clone();
+        let processing_tasks = &self.runtime_tasks.read().unwrap().processing.processing.clone();
 
         let all_task_ids = self.queue.tasks.all_task_ids(wtxn)?;
         let mut to_delete_tasks = all_task_ids & matched_tasks;
@@ -718,9 +1123,10 @@ impl IndexScheduler {
 
         let enqueued_tasks = &self.queue.tasks.get_status(rtxn, Status::Enqueued)?;
 
-        // 0. Check if any upgrade task was matched.
+        // 0. Check if any upgrade or compaction tasks were matched.
         //    If so, we cancel all the failed or enqueued upgrade tasks.
         let upgrade_tasks = &self.queue.tasks.get_kind(rtxn, Kind::UpgradeDatabase)?;
+        let compaction_tasks = &self.queue.tasks.get_kind(rtxn, Kind::IndexCompaction)?;
         let is_canceling_upgrade = !matched_tasks.is_disjoint(upgrade_tasks);
         if is_canceling_upgrade {
             let failed_tasks = self.queue.tasks.get_status(rtxn, Status::Failed)?;
@@ -785,7 +1191,33 @@ impl IndexScheduler {
             }
         }
 
-        // 3. We now have a list of tasks to cancel, cancel them
+        // 3. If we are cancelling a compaction task, remove the tempfiles after incomplete compactions
+        for compaction_task in &tasks_to_cancel & compaction_tasks {
+            progress.update_progress(TaskCancelationProgress::CleaningCompactionLeftover);
+            let task = self.queue.tasks.get_task(rtxn, compaction_task)?.unwrap();
+            let Some(Details::IndexCompaction {
+                index_uid,
+                pre_compaction_size: _,
+                post_compaction_size: _,
+            }) = task.details
+            else {
+                unreachable!("wrong details for compaction task {compaction_task}")
+            };
+
+            let index_path = match self.index_mapper.index_mapping.get(rtxn, &index_uid)? {
+                Some(index_uuid) => self.index_mapper.index_path(index_uuid),
+                None => continue,
+            };
+
+            if let Err(e) = remove_file(index_path.join(DATA_MDB_COPY_NAME)) {
+                match e.kind() {
+                    ErrorKind::NotFound => (),
+                    _ => return Err(Error::IoError(e)),
+                }
+            }
+        }
+
+        // 4. We now have a list of tasks to cancel, cancel them
         let (task_progress, progress_obj) = AtomicTaskStep::new(tasks_to_cancel.len() as u32);
         progress.update_progress(progress_obj);
 

crates/index-scheduler/src/scheduler/process_export.rs

@@ -7,6 +7,7 @@ use backoff::ExponentialBackoff;
 use byte_unit::Byte;
 use flate2::write::GzEncoder;
 use flate2::Compression;
+use meilisearch_types::error::Code;
 use meilisearch_types::index_uid_pattern::IndexUidPattern;
 use meilisearch_types::milli::constants::RESERVED_VECTORS_FIELD_NAME;
 use meilisearch_types::milli::index::EmbeddingsWithMetadata;
@@ -15,7 +16,11 @@ use meilisearch_types::milli::update::{request_threads, Setting};
 use meilisearch_types::milli::vector::parsed_vectors::{ExplicitVectors, VectorOrArrayOfVectors};
 use meilisearch_types::milli::{self, obkv_to_json, Filter, InternalError};
 use meilisearch_types::settings::{self, SecretPolicy};
+use meilisearch_types::tasks::enterprise_edition::network::{
+    headers, ImportData, ImportMetadata, Origin,
+};
 use meilisearch_types::tasks::{DetailsExportIndexSettings, ExportIndexSettings};
+use roaring::RoaringBitmap;
 use serde::Deserialize;
 use ureq::{json, Response};
 
@@ -50,6 +55,7 @@ impl IndexScheduler {
         let agent = ureq::AgentBuilder::new().timeout(Duration::from_secs(5)).build();
         let must_stop_processing = self.scheduler.must_stop_processing.clone();
         for (i, (_pattern, uid, export_settings)) in indexes.iter().enumerate() {
+            let err = |err| Error::from_milli(err, Some(uid.to_string()));
             if must_stop_processing.get() {
                 return Err(Error::AbortedTask);
             }
@@ -61,104 +67,31 @@ impl IndexScheduler {
             ));
 
             let ExportIndexSettings { filter, override_settings } = export_settings;
+
             let index = self.index(uid)?;
             let index_rtxn = index.read_txn()?;
-            let bearer = api_key.map(|api_key| format!("Bearer {api_key}"));
-
-            // First, check if the index already exists
-            let url = format!("{base_url}/indexes/{uid}");
-            let response = retry(&must_stop_processing, || {
-                let mut request = agent.get(&url);
-                if let Some(bearer) = &bearer {
-                    request = request.set("Authorization", bearer);
-                }
-
-                request.send_bytes(Default::default()).map_err(into_backoff_error)
-            });
-            let index_exists = match response {
-                Ok(response) => response.status() == 200,
-                Err(Error::FromRemoteWhenExporting { code, .. }) if code == "index_not_found" => {
-                    false
-                }
-                Err(e) => return Err(e),
-            };
-
-            let primary_key = index
-                .primary_key(&index_rtxn)
-                .map_err(|e| Error::from_milli(e.into(), Some(uid.to_string())))?;
-
-            // Create the index
-            if !index_exists {
-                let url = format!("{base_url}/indexes");
-                retry(&must_stop_processing, || {
-                    let mut request = agent.post(&url);
-                    if let Some(bearer) = &bearer {
-                        request = request.set("Authorization", bearer);
-                    }
-                    let index_param = json!({ "uid": uid, "primaryKey": primary_key });
-                    request.send_json(&index_param).map_err(into_backoff_error)
-                })?;
-            }
-
-            // Patch the index primary key
-            if index_exists && *override_settings {
-                let url = format!("{base_url}/indexes/{uid}");
-                retry(&must_stop_processing, || {
-                    let mut request = agent.patch(&url);
-                    if let Some(bearer) = &bearer {
-                        request = request.set("Authorization", bearer);
-                    }
-                    let index_param = json!({ "primaryKey": primary_key });
-                    request.send_json(&index_param).map_err(into_backoff_error)
-                })?;
-            }
-
-            // Send the index settings
-            if !index_exists || *override_settings {
-                let mut settings =
-                    settings::settings(&index, &index_rtxn, SecretPolicy::RevealSecrets)
-                        .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
-                // Remove the experimental chat setting if not enabled
-                if self.features().check_chat_completions("exporting chat settings").is_err() {
-                    settings.chat = Setting::NotSet;
-                }
-                // Retry logic for sending settings
-                let url = format!("{base_url}/indexes/{uid}/settings");
-                retry(&must_stop_processing, || {
-                    let mut request = agent.patch(&url);
-                    if let Some(bearer) = bearer.as_ref() {
-                        request = request.set("Authorization", bearer);
-                    }
-                    request.send_json(settings.clone()).map_err(into_backoff_error)
-                })?;
-            }
-
-            let filter = filter
-                .as_ref()
-                .map(Filter::from_json)
-                .transpose()
-                .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?
-                .flatten();
-
-            let filter_universe = filter
-                .map(|f| f.evaluate(&index_rtxn, &index))
-                .transpose()
-                .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
-            let whole_universe = index
-                .documents_ids(&index_rtxn)
-                .map_err(|e| Error::from_milli(e.into(), Some(uid.to_string())))?;
+            let filter = filter.as_ref().map(Filter::from_json).transpose().map_err(err)?.flatten();
+            let filter_universe =
+                filter.map(|f| f.evaluate(&index_rtxn, &index)).transpose().map_err(err)?;
+            let whole_universe =
+                index.documents_ids(&index_rtxn).map_err(milli::Error::from).map_err(err)?;
             let universe = filter_universe.unwrap_or(whole_universe);
-
-            let fields_ids_map = index.fields_ids_map(&index_rtxn)?;
-            let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
-
-            // We don't need to keep this one alive as we will
-            // spawn many threads to process the documents
-            drop(index_rtxn);
-
-            let total_documents = universe.len() as u32;
-            let (step, progress_step) = AtomicDocumentStep::new(total_documents);
-            progress.update_progress(progress_step);
+            let target = TargetInstance { base_url, api_key };
+            let ctx = ExportContext {
+                index: &index,
+                index_rtxn: &index_rtxn,
+                universe: &universe,
+                progress: &progress,
+                agent: &agent,
+                must_stop_processing: &must_stop_processing,
+            };
+            let options = ExportOptions {
+                index_uid: uid,
+                payload_size,
+                override_settings: *override_settings,
+                export_mode: ExportMode::ExportRoute,
+            };
+            let total_documents = self.export_one_index(target, options, ctx)?;
 
             output.insert(
                 IndexUidPattern::new_unchecked(uid.clone()),
@@ -167,155 +100,307 @@ impl IndexScheduler {
                     matched_documents: Some(total_documents as u64),
                 },
             );
-
-            let limit = payload_size.map(|ps| ps.as_u64() as usize).unwrap_or(20 * 1024 * 1024); // defaults to 20 MiB
-            let documents_url = format!("{base_url}/indexes/{uid}/documents");
-
-            let results = request_threads()
-                .broadcast(|ctx| {
-                    let index_rtxn = index
-                        .read_txn()
-                        .map_err(|e| Error::from_milli(e.into(), Some(uid.to_string())))?;
-
-                    let mut buffer = Vec::new();
-                    let mut tmp_buffer = Vec::new();
-                    let mut compressed_buffer = Vec::new();
-                    for (i, docid) in universe.iter().enumerate() {
-                        if i % ctx.num_threads() != ctx.index() {
-                            continue;
-                        }
-
-                        let document = index
-                            .document(&index_rtxn, docid)
-                            .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
-
-                        let mut document = obkv_to_json(&all_fields, &fields_ids_map, document)
-                            .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
-
-                        // TODO definitely factorize this code
-                        'inject_vectors: {
-                            let embeddings = index
-                                .embeddings(&index_rtxn, docid)
-                                .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
-
-                            if embeddings.is_empty() {
-                                break 'inject_vectors;
-                            }
-
-                            let vectors = document
-                                .entry(RESERVED_VECTORS_FIELD_NAME)
-                                .or_insert(serde_json::Value::Object(Default::default()));
-
-                            let serde_json::Value::Object(vectors) = vectors else {
-                                return Err(Error::from_milli(
-                                    milli::Error::UserError(
-                                        milli::UserError::InvalidVectorsMapType {
-                                            document_id: {
-                                                if let Ok(Some(Ok(index))) = index
-                                                    .external_id_of(
-                                                        &index_rtxn,
-                                                        std::iter::once(docid),
-                                                    )
-                                                    .map(|it| it.into_iter().next())
-                                                {
-                                                    index
-                                                } else {
-                                                    format!("internal docid={docid}")
-                                                }
-                                            },
-                                            value: vectors.clone(),
-                                        },
-                                    ),
-                                    Some(uid.to_string()),
-                                ));
-                            };
-
-                            for (
-                                embedder_name,
-                                EmbeddingsWithMetadata { embeddings, regenerate, has_fragments },
-                            ) in embeddings
-                            {
-                                let embeddings = ExplicitVectors {
-                                    embeddings: Some(
-                                        VectorOrArrayOfVectors::from_array_of_vectors(embeddings),
-                                    ),
-                                    regenerate: regenerate &&
-                                    // Meilisearch does not handle well dumps with fragments, because as the fragments
-                                    // are marked as user-provided,
-                                    // all embeddings would be regenerated on any settings change or document update.
-                                    // To prevent this, we mark embeddings has non regenerate in this case.
-                                    !has_fragments,
-                                };
-                                vectors.insert(
-                                    embedder_name,
-                                    serde_json::to_value(embeddings).unwrap(),
-                                );
-                            }
-                        }
-
-                        tmp_buffer.clear();
-                        serde_json::to_writer(&mut tmp_buffer, &document)
-                            .map_err(milli::InternalError::from)
-                            .map_err(|e| Error::from_milli(e.into(), Some(uid.to_string())))?;
-
-                        // Make sure we put at least one document in the buffer even
-                        // though we might go above the buffer limit before sending
-                        if !buffer.is_empty() && buffer.len() + tmp_buffer.len() > limit {
-                            // We compress the documents before sending them
-                            let mut encoder =
-                                GzEncoder::new(&mut compressed_buffer, Compression::default());
-                            encoder
-                                .write_all(&buffer)
-                                .map_err(|e| Error::from_milli(e.into(), Some(uid.clone())))?;
-                            encoder
-                                .finish()
-                                .map_err(|e| Error::from_milli(e.into(), Some(uid.clone())))?;
-
-                            retry(&must_stop_processing, || {
-                                let mut request = agent.post(&documents_url);
-                                request = request.set("Content-Type", "application/x-ndjson");
-                                request = request.set("Content-Encoding", "gzip");
-                                if let Some(bearer) = &bearer {
-                                    request = request.set("Authorization", bearer);
-                                }
-                                request.send_bytes(&compressed_buffer).map_err(into_backoff_error)
-                            })?;
-                            buffer.clear();
-                            compressed_buffer.clear();
-                        }
-                        buffer.extend_from_slice(&tmp_buffer);
-
-                        if i > 0 && i % 100 == 0 {
-                            step.fetch_add(100, atomic::Ordering::Relaxed);
-                        }
-                    }
-
-                    retry(&must_stop_processing, || {
-                        let mut request = agent.post(&documents_url);
-                        request = request.set("Content-Type", "application/x-ndjson");
-                        if let Some(bearer) = &bearer {
-                            request = request.set("Authorization", bearer);
-                        }
-                        request.send_bytes(&buffer).map_err(into_backoff_error)
-                    })?;
-
-                    Ok(())
-                })
-                .map_err(|e| {
-                    Error::from_milli(
-                        milli::Error::InternalError(InternalError::PanicInThreadPool(e)),
-                        Some(uid.to_string()),
-                    )
-                })?;
-            for result in results {
-                result?;
-            }
-
-            step.store(total_documents, atomic::Ordering::Relaxed);
         }
 
         Ok(output)
     }
+
+    pub(super) fn export_one_index(
+        &self,
+        target: TargetInstance<'_>,
+        options: ExportOptions<'_>,
+        ctx: ExportContext<'_>,
+    ) -> Result<u64, Error> {
+        let err = |err| Error::from_milli(err, Some(options.index_uid.to_string()));
+
+        let bearer = target.api_key.map(|api_key| format!("Bearer {api_key}"));
+        let url = format!(
+            "{base_url}/indexes/{index_uid}",
+            base_url = target.base_url,
+            index_uid = options.index_uid
+        );
+        let response = retry(ctx.must_stop_processing, || {
+            let mut request = ctx.agent.get(&url);
+            if let Some(bearer) = &bearer {
+                request = request.set("Authorization", bearer);
+            }
+
+            request.send_bytes(Default::default()).map_err(into_backoff_error)
+        });
+        let index_exists = match response {
+            Ok(response) => response.status() == 200,
+            Err(Error::FromRemoteWhenExporting { code, .. })
+                if code == Code::IndexNotFound.name() =>
+            {
+                false
+            }
+            Err(e) => return Err(e),
+        };
+        let primary_key =
+            ctx.index.primary_key(&ctx.index_rtxn).map_err(milli::Error::from).map_err(err)?;
+        if !index_exists {
+            let url = format!("{base_url}/indexes", base_url = target.base_url);
+            retry(ctx.must_stop_processing, || {
+                let mut request = ctx.agent.post(&url);
+                if let Some(bearer) = &bearer {
+                    request = request.set("Authorization", bearer);
+                }
+                let index_param = json!({ "uid": options.index_uid, "primaryKey": primary_key });
+                request.send_json(&index_param).map_err(into_backoff_error)
+            })?;
+        }
+        if index_exists && options.override_settings {
+            retry(ctx.must_stop_processing, || {
+                let mut request = ctx.agent.patch(&url);
+                if let Some(bearer) = &bearer {
+                    request = request.set("Authorization", bearer);
+                }
+                let index_param = json!({ "primaryKey": primary_key });
+                request.send_json(&index_param).map_err(into_backoff_error)
+            })?;
+        }
+        if !index_exists || options.override_settings {
+            let mut settings =
+                settings::settings(&ctx.index, &ctx.index_rtxn, SecretPolicy::RevealSecrets)
+                    .map_err(err)?;
+            // Remove the experimental chat setting if not enabled
+            if self.features().check_chat_completions("exporting chat settings").is_err() {
+                settings.chat = Setting::NotSet;
+            }
+            // Retry logic for sending settings
+            let url = format!(
+                "{base_url}/indexes/{index_uid}/settings",
+                base_url = target.base_url,
+                index_uid = options.index_uid
+            );
+            retry(ctx.must_stop_processing, || {
+                let mut request = ctx.agent.patch(&url);
+                if let Some(bearer) = bearer.as_ref() {
+                    request = request.set("Authorization", bearer);
+                }
+                request.send_json(settings.clone()).map_err(into_backoff_error)
+            })?;
+        }
+
+        let fields_ids_map = ctx.index.fields_ids_map(&ctx.index_rtxn)?;
+        let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
+        let total_documents = ctx.universe.len() as u32;
+        let (step, progress_step) = AtomicDocumentStep::new(total_documents);
+        ctx.progress.update_progress(progress_step);
+
+        let limit = options.payload_size.map(|ps| ps.as_u64() as usize).unwrap_or(20 * 1024 * 1024);
+        let documents_url = format!(
+            "{base_url}/indexes/{index_uid}/documents",
+            base_url = target.base_url,
+            index_uid = options.index_uid
+        );
+        let results = request_threads()
+            .broadcast(|broadcast| {
+                let mut task_network = if let ExportMode::NetworkBalancing {
+                    index_count,
+                    export_old_remote_name,
+                    network_change_origin,
+                } = options.export_mode
+                {
+                    Some((
+                        ImportData {
+                            remote_name: export_old_remote_name.to_string(),
+                            index_name: options.index_uid.to_string(),
+                            document_count: 0,
+                        },
+                        network_change_origin.clone(),
+                        ImportMetadata {
+                            index_count,
+                            task_key: 0,
+                            total_index_documents: ctx.universe.len(),
+                        },
+                    ))
+                } else {
+                    None
+                };
+
+                let index_rtxn = ctx.index.read_txn().map_err(milli::Error::from).map_err(err)?;
+
+                let mut buffer = Vec::new();
+                let mut tmp_buffer = Vec::new();
+                let mut compressed_buffer = Vec::new();
+                for (i, docid) in ctx.universe.iter().enumerate() {
+                    if i % broadcast.num_threads() != broadcast.index() {
+                        continue;
+                    }
+                    if let Some((import_data, _, metadata)) = &mut task_network {
+                        import_data.document_count += 1;
+                        metadata.task_key = docid;
+                    }
+
+                    let document = ctx.index.document(&index_rtxn, docid).map_err(err)?;
+
+                    let mut document =
+                        obkv_to_json(&all_fields, &fields_ids_map, document).map_err(err)?;
+
+                    // TODO definitely factorize this code
+                    'inject_vectors: {
+                        let embeddings = ctx.index.embeddings(&index_rtxn, docid).map_err(err)?;
+
+                        if embeddings.is_empty() {
+                            break 'inject_vectors;
+                        }
+
+                        let vectors = document
+                            .entry(RESERVED_VECTORS_FIELD_NAME)
+                            .or_insert(serde_json::Value::Object(Default::default()));
+
+                        let serde_json::Value::Object(vectors) = vectors else {
+                            return Err(err(milli::Error::UserError(
+                                milli::UserError::InvalidVectorsMapType {
+                                    document_id: {
+                                        if let Ok(Some(Ok(index))) = ctx
+                                            .index
+                                            .external_id_of(&index_rtxn, std::iter::once(docid))
+                                            .map(|it| it.into_iter().next())
+                                        {
+                                            index
+                                        } else {
+                                            format!("internal docid={docid}")
+                                        }
+                                    },
+                                    value: vectors.clone(),
+                                },
+                            )));
+                        };
+
+                        for (
+                            embedder_name,
+                            EmbeddingsWithMetadata { embeddings, regenerate, has_fragments },
+                        ) in embeddings
+                        {
+                            let embeddings = ExplicitVectors {
+                                embeddings: Some(VectorOrArrayOfVectors::from_array_of_vectors(
+                                    embeddings,
+                                )),
+                                regenerate: regenerate &&
+                                // Meilisearch does not handle well dumps with fragments, because as the fragments
+                                // are marked as user-provided,
+                                // all embeddings would be regenerated on any settings change or document update.
+                                // To prevent this, we mark embeddings has non regenerate in this case.
+                                !has_fragments,
+                            };
+                            vectors
+                                .insert(embedder_name, serde_json::to_value(embeddings).unwrap());
+                        }
+                    }
+
+                    tmp_buffer.clear();
+                    serde_json::to_writer(&mut tmp_buffer, &document)
+                        .map_err(milli::InternalError::from)
+                        .map_err(milli::Error::from)
+                        .map_err(err)?;
+
+                    // Make sure we put at least one document in the buffer even
+                    // though we might go above the buffer limit before sending
+                    if !buffer.is_empty() && buffer.len() + tmp_buffer.len() > limit {
+                        // We compress the documents before sending them
+                        let mut encoder =
+                            GzEncoder::new(&mut compressed_buffer, Compression::default());
+                        encoder.write_all(&buffer).map_err(milli::Error::from).map_err(err)?;
+                        encoder.finish().map_err(milli::Error::from).map_err(err)?;
+
+                        match retry(ctx.must_stop_processing, || {
+                            let mut request = ctx.agent.post(&documents_url);
+                            request = request.set("Content-Type", "application/x-ndjson");
+                            request = request.set("Content-Encoding", "gzip");
+                            if let Some(bearer) = &bearer {
+                                request = request.set("Authorization", bearer);
+                            }
+                            if let Some((import_data, origin, metadata)) = &task_network {
+                                request = set_network_ureq_headers(
+                                    request,
+                                    import_data,
+                                    origin,
+                                    metadata,
+                                );
+                            }
+                            request.send_bytes(&compressed_buffer).map_err(into_backoff_error)
+                        }) {
+                            Ok(_response) => {}
+                            Err(Error::FromRemoteWhenExporting { code, .. })
+                                if code == Code::ImportTaskAlreadyReceived.name() =>
+                            {
+                                continue;
+                            }
+                            Err(Error::FromRemoteWhenExporting { code, message, .. })
+                                if code == Code::ImportTaskUnknownRemote.name() =>
+                            {
+                                tracing::warn!("remote answered with: {message}");
+                                break;
+                            }
+                            Err(Error::FromRemoteWhenExporting { code, message, .. })
+                                if code == Code::ImportTaskWithoutNetworkTask.name() =>
+                            {
+                                tracing::warn!("remote answered with: {message}");
+                                break;
+                            }
+                            Err(e) => return Err(e),
+                        }
+                        buffer.clear();
+                        compressed_buffer.clear();
+                        if let Some((import_data, _, metadata)) = &mut task_network {
+                            import_data.document_count = 0;
+                            metadata.task_key = 0;
+                        }
+                    }
+                    buffer.extend_from_slice(&tmp_buffer);
+
+                    if i > 0 && i % 100 == 0 {
+                        step.fetch_add(100, atomic::Ordering::Relaxed);
+                    }
+                }
+
+                retry(ctx.must_stop_processing, || {
+                    let mut request = ctx.agent.post(&documents_url);
+                    request = request.set("Content-Type", "application/x-ndjson");
+                    if let Some((import_data, origin, metadata)) = &task_network {
+                        request = set_network_ureq_headers(request, import_data, origin, metadata);
+                    }
+                    if let Some(bearer) = &bearer {
+                        request = request.set("Authorization", bearer);
+                    }
+                    request.send_bytes(&buffer).map_err(into_backoff_error)
+                })?;
+
+                Ok(())
+            })
+            .map_err(|e| err(milli::Error::InternalError(InternalError::PanicInThreadPool(e))))?;
+        for result in results {
+            result?;
+        }
+        step.store(total_documents, atomic::Ordering::Relaxed);
+        Ok(total_documents as u64)
+    }
+}
+
+fn set_network_ureq_headers(
+    request: ureq::Request,
+    import_data: &ImportData,
+    origin: &Origin,
+    metadata: &ImportMetadata,
+) -> ureq::Request {
+    request
+        .set(headers::PROXY_ORIGIN_REMOTE_HEADER, &origin.remote_name)
+        .set(headers::PROXY_ORIGIN_TASK_UID_HEADER, &origin.task_uid.to_string())
+        .set(
+            headers::PROXY_ORIGIN_NETWORK_VERSION_HEADER,
+            &origin.network_version.to_u128_le().to_string(),
+        )
+        .set(headers::PROXY_IMPORT_REMOTE_HEADER, &import_data.remote_name)
+        .set(headers::PROXY_IMPORT_INDEX_HEADER, &import_data.index_name)
+        .set(headers::PROXY_IMPORT_TASK_KEY_HEADER, &metadata.task_key.to_string())
+        .set(headers::PROXY_IMPORT_DOCS_HEADER, &import_data.document_count.to_string())
+        .set(headers::PROXY_IMPORT_INDEX_COUNT_HEADER, &metadata.index_count.to_string())
+        .set(
+            headers::PROXY_IMPORT_TOTAL_INDEX_DOCS_HEADER,
+            &metadata.total_index_documents.to_string(),
+        )
 }
 
 fn retry<F>(must_stop_processing: &MustStopProcessing, send_request: F) -> Result<ureq::Response>
@@ -370,8 +455,41 @@ fn ureq_error_into_error(error: ureq::Error) -> Error {
             }
             Err(e) => e.into(),
         },
-        ureq::Error::Transport(transport) => io::Error::new(io::ErrorKind::Other, transport).into(),
+        ureq::Error::Transport(transport) => io::Error::other(transport).into(),
     }
 }
 
+// export_one_index arguments
+pub(super) struct TargetInstance<'a> {
+    pub(super) base_url: &'a str,
+    pub(super) api_key: Option<&'a str>,
+}
+
+pub(super) struct ExportOptions<'a> {
+    pub(super) index_uid: &'a str,
+    pub(super) payload_size: Option<&'a Byte>,
+    pub(super) override_settings: bool,
+    pub(super) export_mode: ExportMode<'a>,
+}
+
+pub(super) struct ExportContext<'a> {
+    pub(super) index: &'a meilisearch_types::milli::Index,
+    pub(super) index_rtxn: &'a milli::heed::RoTxn<'a>,
+    pub(super) universe: &'a RoaringBitmap,
+    pub(super) progress: &'a Progress,
+    pub(super) agent: &'a ureq::Agent,
+    pub(super) must_stop_processing: &'a MustStopProcessing,
+}
+
+pub(super) enum ExportMode<'a> {
+    ExportRoute,
+    NetworkBalancing {
+        index_count: u64,
+
+        export_old_remote_name: &'a str,
+        network_change_origin: &'a Origin,
+    },
+}
+
+// progress related
 enum ExportIndex {}

crates/index-scheduler/src/scheduler/process_index_operation.rs

@@ -66,6 +66,11 @@ impl IndexScheduler {
             }
             IndexOperation::DocumentOperation { index_uid, primary_key, operations, mut tasks } => {
                 progress.update_progress(DocumentOperationProgress::RetrievingConfig);
+
+                let network = self.network();
+
+                let shards = network.shards();
+
                 // TODO: at some point, for better efficiency we might want to reuse the bumpalo for successive batches.
                 // this is made difficult by the fact we're doing private clones of the index scheduler and sending it
                 // to a fresh thread.
@@ -130,6 +135,7 @@ impl IndexScheduler {
                         &mut new_fields_ids_map,
                         &|| must_stop_processing.get(),
                         progress.clone(),
+                        shards.as_ref(),
                     )
                     .map_err(|e| Error::from_milli(e, Some(index_uid.clone())))?;
 

crates/index-scheduler/src/scheduler/process_snapshot_creation.rs

@@ -12,6 +12,8 @@ use crate::processing::{AtomicUpdateFileStep, SnapshotCreationProgress};
 use crate::queue::TaskQueue;
 use crate::{Error, IndexScheduler, Result};
 
+const UPDATE_FILES_DIR_NAME: &str = "update_files";
+
 /// # Safety
 ///
 /// See [`EnvOpenOptions::open`].
@@ -78,10 +80,32 @@ impl IndexScheduler {
     pub(super) fn process_snapshot(
         &self,
         progress: Progress,
-        mut tasks: Vec<Task>,
+        tasks: Vec<Task>,
     ) -> Result<Vec<Task>> {
         progress.update_progress(SnapshotCreationProgress::StartTheSnapshotCreation);
 
+        match self.scheduler.s3_snapshot_options.clone() {
+            Some(options) => {
+                #[cfg(not(unix))]
+                {
+                    let _ = options;
+                    panic!("Non-unix platform does not support S3 snapshotting");
+                }
+                #[cfg(unix)]
+                self.runtime
+                    .as_ref()
+                    .expect("Runtime not initialized")
+                    .block_on(self.process_snapshot_to_s3(progress, options, tasks))
+            }
+            None => self.process_snapshots_to_disk(progress, tasks),
+        }
+    }
+
+    fn process_snapshots_to_disk(
+        &self,
+        progress: Progress,
+        mut tasks: Vec<Task>,
+    ) -> Result<Vec<Task>, Error> {
         fs::create_dir_all(&self.scheduler.snapshots_path)?;
         let temp_snapshot_dir = tempfile::tempdir()?;
 
@@ -128,7 +152,7 @@ impl IndexScheduler {
         let rtxn = self.env.read_txn()?;
 
         // 2.4 Create the update files directory
-        let update_files_dir = temp_snapshot_dir.path().join("update_files");
+        let update_files_dir = temp_snapshot_dir.path().join(UPDATE_FILES_DIR_NAME);
         fs::create_dir_all(&update_files_dir)?;
 
         // 2.5 Only copy the update files of the enqueued tasks
@@ -140,7 +164,7 @@ impl IndexScheduler {
             let task =
                 self.queue.tasks.get_task(&rtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?;
             if let Some(content_uuid) = task.content_uuid() {
-                let src = self.queue.file_store.get_update_path(content_uuid);
+                let src = self.queue.file_store.update_path(content_uuid);
                 let dst = update_files_dir.join(content_uuid.to_string());
                 fs::copy(src, dst)?;
             }
@@ -206,4 +230,403 @@ impl IndexScheduler {
 
         Ok(tasks)
     }
+
+    #[cfg(unix)]
+    pub(super) async fn process_snapshot_to_s3(
+        &self,
+        progress: Progress,
+        opts: meilisearch_types::milli::update::S3SnapshotOptions,
+        mut tasks: Vec<Task>,
+    ) -> Result<Vec<Task>> {
+        use meilisearch_types::milli::update::S3SnapshotOptions;
+
+        let S3SnapshotOptions {
+            s3_bucket_url,
+            s3_bucket_region,
+            s3_bucket_name,
+            s3_snapshot_prefix,
+            s3_access_key,
+            s3_secret_key,
+            s3_max_in_flight_parts,
+            s3_compression_level: level,
+            s3_signature_duration,
+            s3_multipart_part_size,
+        } = opts;
+
+        let must_stop_processing = self.scheduler.must_stop_processing.clone();
+        let retry_backoff = backoff::ExponentialBackoff::default();
+        let db_name = {
+            let mut base_path = self.env.path().to_owned();
+            base_path.pop();
+            base_path.file_name().and_then(OsStr::to_str).unwrap_or("data.ms").to_string()
+        };
+
+        let (reader, writer) = std::io::pipe()?;
+        let uploader_task = tokio::spawn(multipart_stream_to_s3(
+            s3_bucket_url,
+            s3_bucket_region,
+            s3_bucket_name,
+            s3_snapshot_prefix,
+            s3_access_key,
+            s3_secret_key,
+            s3_max_in_flight_parts,
+            s3_signature_duration,
+            s3_multipart_part_size,
+            must_stop_processing,
+            retry_backoff,
+            db_name,
+            reader,
+        ));
+
+        let index_scheduler = IndexScheduler::private_clone(self);
+        let builder_task = tokio::task::spawn_blocking(move || {
+            stream_tarball_into_pipe(progress, level, writer, index_scheduler)
+        });
+
+        let (uploader_result, builder_result) = tokio::join!(uploader_task, builder_task);
+
+        // Check uploader result first to early return on task abortion.
+        // safety: JoinHandle can return an error if the task was aborted, cancelled, or panicked.
+        uploader_result.unwrap()?;
+        builder_result.unwrap()?;
+
+        for task in &mut tasks {
+            task.status = Status::Succeeded;
+        }
+
+        Ok(tasks)
+    }
+}
+
+/// Streams a tarball of the database content into a pipe.
+#[cfg(unix)]
+fn stream_tarball_into_pipe(
+    progress: Progress,
+    level: u32,
+    writer: std::io::PipeWriter,
+    index_scheduler: IndexScheduler,
+) -> std::result::Result<(), Error> {
+    use std::io::Write as _;
+    use std::path::Path;
+
+    let writer = flate2::write::GzEncoder::new(writer, flate2::Compression::new(level));
+    let mut tarball = tar::Builder::new(writer);
+
+    // 1. Snapshot the version file
+    tarball
+        .append_path_with_name(&index_scheduler.scheduler.version_file_path, VERSION_FILE_NAME)?;
+
+    // 2. Snapshot the index scheduler LMDB env
+    progress.update_progress(SnapshotCreationProgress::SnapshotTheIndexScheduler);
+    let tasks_env_file = index_scheduler.env.try_clone_inner_file()?;
+    let path = Path::new("tasks").join("data.mdb");
+    append_file_to_tarball(&mut tarball, path, tasks_env_file)?;
+
+    // 2.3 Create a read transaction on the index-scheduler
+    let rtxn = index_scheduler.env.read_txn()?;
+
+    // 2.4 Create the update files directory
+    //     And only copy the update files of the enqueued tasks
+    progress.update_progress(SnapshotCreationProgress::SnapshotTheUpdateFiles);
+    let enqueued = index_scheduler.queue.tasks.get_status(&rtxn, Status::Enqueued)?;
+    let (atomic, update_file_progress) = AtomicUpdateFileStep::new(enqueued.len() as u32);
+    progress.update_progress(update_file_progress);
+
+    // We create the update_files directory so that it
+    // always exists even if there are no update files
+    let update_files_dir = Path::new(UPDATE_FILES_DIR_NAME);
+    let src_update_files_dir = {
+        let mut path = index_scheduler.env.path().to_path_buf();
+        path.pop();
+        path.join(UPDATE_FILES_DIR_NAME)
+    };
+    tarball.append_dir(update_files_dir, src_update_files_dir)?;
+
+    for task_id in enqueued {
+        let task = index_scheduler
+            .queue
+            .tasks
+            .get_task(&rtxn, task_id)?
+            .ok_or(Error::CorruptedTaskQueue)?;
+        if let Some(content_uuid) = task.content_uuid() {
+            use std::fs::File;
+
+            let src = index_scheduler.queue.file_store.update_path(content_uuid);
+            let mut update_file = File::open(src)?;
+            let path = update_files_dir.join(content_uuid.to_string());
+            tarball.append_file(path, &mut update_file)?;
+        }
+        atomic.fetch_add(1, Ordering::Relaxed);
+    }
+
+    // 3. Snapshot every indexes
+    progress.update_progress(SnapshotCreationProgress::SnapshotTheIndexes);
+    let index_mapping = index_scheduler.index_mapper.index_mapping;
+    let nb_indexes = index_mapping.len(&rtxn)? as u32;
+    let indexes_dir = Path::new("indexes");
+    let indexes_references: Vec<_> = index_scheduler
+        .index_mapper
+        .index_mapping
+        .iter(&rtxn)?
+        .map(|res| res.map_err(Error::from).map(|(name, uuid)| (name.to_string(), uuid)))
+        .collect::<Result<_, Error>>()?;
+
+    // It's prettier to use a for loop instead of the IndexMapper::try_for_each_index
+    // method, especially when we need to access the UUID, local path and index number.
+    for (i, (name, uuid)) in indexes_references.into_iter().enumerate() {
+        progress.update_progress(VariableNameStep::<SnapshotCreationProgress>::new(
+            &name, i as u32, nb_indexes,
+        ));
+        let path = indexes_dir.join(uuid.to_string()).join("data.mdb");
+        let index = index_scheduler.index_mapper.index(&rtxn, &name)?;
+        let index_file = index.try_clone_inner_file()?;
+        tracing::trace!("Appending index file for {name} in {}", path.display());
+        append_file_to_tarball(&mut tarball, path, index_file)?;
+    }
+
+    drop(rtxn);
+
+    // 4. Snapshot the auth LMDB env
+    progress.update_progress(SnapshotCreationProgress::SnapshotTheApiKeys);
+    let auth_env_file = index_scheduler.scheduler.auth_env.try_clone_inner_file()?;
+    let path = Path::new("auth").join("data.mdb");
+    append_file_to_tarball(&mut tarball, path, auth_env_file)?;
+
+    let mut gzencoder = tarball.into_inner()?;
+    gzencoder.flush()?;
+    gzencoder.try_finish()?;
+    let mut writer = gzencoder.finish()?;
+    writer.flush()?;
+
+    Result::<_, Error>::Ok(())
+}
+
+#[cfg(unix)]
+fn append_file_to_tarball<W, P>(
+    tarball: &mut tar::Builder<W>,
+    path: P,
+    mut auth_env_file: fs::File,
+) -> Result<(), Error>
+where
+    W: std::io::Write,
+    P: AsRef<std::path::Path>,
+{
+    use std::io::{Seek as _, SeekFrom};
+
+    // Note: A previous snapshot operation may have left the cursor
+    //       at the end of the file so we need to seek to the start.
+    auth_env_file.seek(SeekFrom::Start(0))?;
+    tarball.append_file(path, &mut auth_env_file)?;
+    Ok(())
+}
+
+/// Streams the content read from the given reader to S3.
+#[cfg(unix)]
+#[allow(clippy::too_many_arguments)]
+async fn multipart_stream_to_s3(
+    s3_bucket_url: String,
+    s3_bucket_region: String,
+    s3_bucket_name: String,
+    s3_snapshot_prefix: String,
+    s3_access_key: String,
+    s3_secret_key: String,
+    s3_max_in_flight_parts: std::num::NonZero<usize>,
+    s3_signature_duration: std::time::Duration,
+    s3_multipart_part_size: u64,
+    must_stop_processing: super::MustStopProcessing,
+    retry_backoff: backoff::exponential::ExponentialBackoff<backoff::SystemClock>,
+    db_name: String,
+    reader: std::io::PipeReader,
+) -> Result<(), Error> {
+    use std::{collections::VecDeque, os::fd::OwnedFd, path::PathBuf};
+
+    use bytes::{Bytes, BytesMut};
+    use reqwest::{Client, Response};
+    use rusty_s3::S3Action as _;
+    use rusty_s3::{actions::CreateMultipartUpload, Bucket, BucketError, Credentials, UrlStyle};
+    use tokio::task::JoinHandle;
+
+    let reader = OwnedFd::from(reader);
+    let reader = tokio::net::unix::pipe::Receiver::from_owned_fd(reader)?;
+    let s3_snapshot_prefix = PathBuf::from(s3_snapshot_prefix);
+    let url =
+        s3_bucket_url.parse().map_err(BucketError::ParseError).map_err(Error::S3BucketError)?;
+    let bucket = Bucket::new(url, UrlStyle::Path, s3_bucket_name, s3_bucket_region)
+        .map_err(Error::S3BucketError)?;
+    let credential = Credentials::new(s3_access_key, s3_secret_key);
+
+    // Note for the future (rust 1.91+): use with_added_extension, it's prettier
+    let object_path = s3_snapshot_prefix.join(format!("{db_name}.snapshot"));
+    // Note: It doesn't work on Windows and if a port to this platform is needed,
+    //       use the slash-path crate or similar to get the correct path separator.
+    let object = object_path.display().to_string();
+
+    let action = bucket.create_multipart_upload(Some(&credential), &object);
+    let url = action.sign(s3_signature_duration);
+
+    let client = Client::new();
+    let resp = client.post(url).send().await.map_err(Error::S3HttpError)?;
+    let status = resp.status();
+
+    let body = match resp.error_for_status_ref() {
+        Ok(_) => resp.text().await.map_err(Error::S3HttpError)?,
+        Err(_) => {
+            return Err(Error::S3Error { status, body: resp.text().await.unwrap_or_default() })
+        }
+    };
+
+    let multipart =
+        CreateMultipartUpload::parse_response(&body).map_err(|e| Error::S3XmlError(Box::new(e)))?;
+    tracing::debug!("Starting the upload of the snapshot to {object}");
+
+    // We use this bumpalo for etags strings.
+    let bump = bumpalo::Bump::new();
+    let mut etags = Vec::<&str>::new();
+    let mut in_flight = VecDeque::<(JoinHandle<reqwest::Result<Response>>, Bytes)>::with_capacity(
+        s3_max_in_flight_parts.get(),
+    );
+
+    // Part numbers start at 1 and cannot be larger than 10k
+    for part_number in 1u16.. {
+        if must_stop_processing.get() {
+            return Err(Error::AbortedTask);
+        }
+
+        let part_upload =
+            bucket.upload_part(Some(&credential), &object, part_number, multipart.upload_id());
+        let url = part_upload.sign(s3_signature_duration);
+
+        // Wait for a buffer to be ready if there are in-flight parts that landed
+        let mut buffer = if in_flight.len() >= s3_max_in_flight_parts.get() {
+            let (handle, buffer) = in_flight.pop_front().expect("At least one in flight request");
+            let resp = join_and_map_error(handle).await?;
+            extract_and_append_etag(&bump, &mut etags, resp.headers())?;
+
+            let mut buffer = match buffer.try_into_mut() {
+                Ok(buffer) => buffer,
+                Err(_) => unreachable!("All bytes references were consumed in the task"),
+            };
+            buffer.clear();
+            buffer
+        } else {
+            BytesMut::with_capacity(s3_multipart_part_size as usize)
+        };
+
+        // If we successfully read enough bytes,
+        // we can continue and send the buffer/part
+        while buffer.len() < (s3_multipart_part_size as usize / 2) {
+            // Wait for the pipe to be readable
+
+            use std::io;
+            reader.readable().await?;
+
+            match reader.try_read_buf(&mut buffer) {
+                Ok(0) => break,
+                // We read some bytes but maybe not enough
+                Ok(_) => continue,
+                // The readiness event is a false positive.
+                Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => continue,
+                Err(e) => return Err(e.into()),
+            }
+        }
+
+        if buffer.is_empty() {
+            // Break the loop if the buffer is
+            // empty after we tried to read bytes
+            break;
+        }
+
+        let body = buffer.freeze();
+        tracing::trace!("Sending part {part_number}");
+        let task = tokio::spawn({
+            let client = client.clone();
+            let body = body.clone();
+            backoff::future::retry(retry_backoff.clone(), move || {
+                let client = client.clone();
+                let url = url.clone();
+                let body = body.clone();
+                async move {
+                    match client.put(url).body(body).send().await {
+                        Ok(resp) if resp.status().is_client_error() => {
+                            resp.error_for_status().map_err(backoff::Error::Permanent)
+                        }
+                        Ok(resp) => Ok(resp),
+                        Err(e) => Err(backoff::Error::transient(e)),
+                    }
+                }
+            })
+        });
+        in_flight.push_back((task, body));
+    }
+
+    for (handle, _buffer) in in_flight {
+        let resp = join_and_map_error(handle).await?;
+        extract_and_append_etag(&bump, &mut etags, resp.headers())?;
+    }
+
+    tracing::debug!("Finalizing the multipart upload");
+
+    let action = bucket.complete_multipart_upload(
+        Some(&credential),
+        &object,
+        multipart.upload_id(),
+        etags.iter().map(AsRef::as_ref),
+    );
+    let url = action.sign(s3_signature_duration);
+    let body = action.body();
+    let resp = backoff::future::retry(retry_backoff, move || {
+        let client = client.clone();
+        let url = url.clone();
+        let body = body.clone();
+        async move {
+            match client.post(url).body(body).send().await {
+                Ok(resp) if resp.status().is_client_error() => {
+                    resp.error_for_status().map_err(backoff::Error::Permanent)
+                }
+                Ok(resp) => Ok(resp),
+                Err(e) => Err(backoff::Error::transient(e)),
+            }
+        }
+    })
+    .await
+    .map_err(Error::S3HttpError)?;
+
+    let status = resp.status();
+    let body = resp.text().await.map_err(|e| Error::S3Error { status, body: e.to_string() })?;
+    if status.is_success() {
+        Ok(())
+    } else {
+        Err(Error::S3Error { status, body })
+    }
+}
+
+#[cfg(unix)]
+async fn join_and_map_error(
+    join_handle: tokio::task::JoinHandle<Result<reqwest::Response, reqwest::Error>>,
+) -> Result<reqwest::Response> {
+    // safety: Panic happens if the task (JoinHandle) was aborted, cancelled, or panicked
+    let request = join_handle.await.unwrap();
+    let resp = request.map_err(Error::S3HttpError)?;
+    match resp.error_for_status_ref() {
+        Ok(_) => Ok(resp),
+        Err(_) => Err(Error::S3Error {
+            status: resp.status(),
+            body: resp.text().await.unwrap_or_default(),
+        }),
+    }
+}
+
+#[cfg(unix)]
+fn extract_and_append_etag<'b>(
+    bump: &'b bumpalo::Bump,
+    etags: &mut Vec<&'b str>,
+    headers: &reqwest::header::HeaderMap,
+) -> Result<()> {
+    use reqwest::header::ETAG;
+
+    let etag = headers.get(ETAG).ok_or_else(|| Error::S3XmlError("Missing ETag header".into()))?;
+    let etag = etag.to_str().map_err(|e| Error::S3XmlError(Box::new(e)))?;
+    etags.push(bump.alloc_str(etag));
+
+    Ok(())
 }

crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap

@@ -6,9 +6,9 @@ source: crates/index-scheduler/src/scheduler/test.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }}
-1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }}
-2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "girafos", primary_key: None }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }}
+1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }}
+2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "girafos", primary_key: None }}
 3 {uid: 3, batch_uid: 3, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }}
 4 {uid: 4, batch_uid: 4, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "cattos" }}
 5 {uid: 5, batch_uid: 5, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "girafos" }}

crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }}
 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }}
 ----------------------------------------------------------------------

crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }}
 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(0) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 2 {uid: 2, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: IndexDeletion { index_uid: "doggos" }}
 ----------------------------------------------------------------------

crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap

@@ -1,13 +1,12 @@
 ---
 source: crates/index-scheduler/src/scheduler/test.rs
-snapshot_kind: text
 ---
 ### Autobatching Enabled = true
 ### Processing batch None:
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }}
+0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,]

crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap

@@ -1,13 +1,12 @@
 ---
 source: crates/index-scheduler/src/scheduler/test.rs
-snapshot_kind: text
 ---
 ### Autobatching Enabled = true
 ### Processing batch None:
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }}
+0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }}
 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status: