Merge #5376

5376: Support dumpless upgrade for all v1.13 patches r=Kerollmops a=dureuill

# Pull Request

## Related issue
Fixes #5373 

## What does this PR do?
- ensure v1.13 versions are known to the index scheduler upgrading code
- Forbid opening a db of v1.13.x from v1.13.y
- Keep old stat format to make sure the number of documents is available in stats during dumpless upgrade


Co-authored-by: Kerollmops <Kerollmops@users.noreply.github.com>
Co-authored-by: Louis Dureuil <louis@meilisearch.com>

.github/ISSUE_TEMPLATE/sprint_issue.md

@@ -2,14 +2,13 @@
 name: New sprint issue
 about: ⚠️ Should only be used by the engine team ⚠️
 title: ''
-labels: ''
+labels: 'missing usage in PRD, impacts docs'
 assignees: ''
 
 ---
 
 Related product team resources: [PRD]() (_internal only_)
 Related product discussion:
-Related spec: WIP
 
 ## Motivation
 
@@ -21,11 +20,7 @@ Related spec: WIP
 
 ## TODO
 
-<!---Feel free to adapt this list with more technical/product steps-->
-
-- [ ] Release a prototype
-- [ ] If prototype validated, merge changes into `main`
-- [ ] Update the spec
+<!---If necessary, create a list with technical/product steps-->
 
 ### Reminders when modifying the Setting API
 

.github/workflows/bench-manual.yml

@@ -18,11 +18,9 @@ jobs:
         timeout-minutes: 180 # 3h
         steps:
             - uses: actions/checkout@v3
-            - uses: actions-rs/toolchain@v1
+            - uses: dtolnay/rust-toolchain@1.81
               with:
                 profile: minimal
-                toolchain: stable
-                override: true
 
             - name: Run benchmarks - workload ${WORKLOAD_NAME} - branch ${{ github.ref }} - commit ${{ github.sha }}
               run: |

.github/workflows/bench-pr.yml

@@ -16,6 +16,37 @@ jobs:
       runs-on: benchmarks
       timeout-minutes: 180 # 3h
       steps:
+        - name: Check permissions
+          id: permission
+          env:
+            PR_AUTHOR: ${{github.event.issue.user.login }}
+            COMMENT_AUTHOR: ${{github.event.comment.user.login }}
+            REPOSITORY: ${{github.repository}}
+            PR_ID: ${{github.event.issue.number}}
+          run: |
+            PR_REPOSITORY=$(gh api /repos/"$REPOSITORY"/pulls/"$PR_ID" --jq .head.repo.full_name)
+            if $(gh api /repos/"$REPOSITORY"/collaborators/"$PR_AUTHOR"/permission --jq .user.permissions.push)
+            then
+              echo "::notice title=Authentication success::PR author authenticated"
+            else
+              echo "::error title=Authentication error::PR author doesn't have push permission on this repository"
+              exit 1
+            fi
+            if $(gh api /repos/"$REPOSITORY"/collaborators/"$COMMENT_AUTHOR"/permission --jq .user.permissions.push)
+            then
+              echo "::notice title=Authentication success::Comment author authenticated"
+            else
+              echo "::error title=Authentication error::Comment author doesn't have push permission on this repository"
+              exit 1
+            fi
+            if [ "$PR_REPOSITORY" = "$REPOSITORY" ]
+            then
+              echo "::notice title=Authentication success::PR started from main repository"
+            else
+              echo "::error title=Authentication error::PR started from a fork"
+              exit 1
+            fi
+
         - name: Check for Command
           id: command
           uses: xt0rted/slash-command-action@v2
@@ -24,7 +55,7 @@ jobs:
               reaction-type: "rocket"
               repo-token: ${{ env.GH_TOKEN }}
 
-        - uses: xt0rted/pull-request-comment-branch@v2
+        - uses: xt0rted/pull-request-comment-branch@v3
           id: comment-branch
           with:
             repo_token: ${{ env.GH_TOKEN }}
@@ -35,12 +66,17 @@ jobs:
             fetch-depth: 0 # fetch full history to be able to get main commit sha
             ref: ${{ steps.comment-branch.outputs.head_ref }}
 
-        - uses: actions-rs/toolchain@v1
+        - uses: dtolnay/rust-toolchain@1.81
           with:
             profile: minimal
-            toolchain: stable
-            override: true
 
         - name: Run benchmarks on PR ${{ github.event.issue.id }}
           run: |
-            cargo xtask bench --api-key "${{ secrets.BENCHMARK_API_KEY }}" --dashboard-url "${{ vars.BENCHMARK_DASHBOARD_URL }}" --reason "[Comment](${{ github.event.comment.url }}) on [#${{github.event.issue.id}}](${{ github.event.issue.url }})" -- ${{ steps.command.outputs.command-arguments }}
+            cargo xtask bench --api-key "${{ secrets.BENCHMARK_API_KEY }}" \
+               --dashboard-url "${{ vars.BENCHMARK_DASHBOARD_URL }}" \
+               --reason "[Comment](${{ github.event.comment.html_url }}) on [#${{ github.event.issue.number }}](${{ github.event.issue.html_url }})" \
+               -- ${{ steps.command.outputs.command-arguments }} > benchlinks.txt
+
+        - name: Send comment in PR
+          run: |
+            gh pr comment ${{github.event.issue.number}} --body-file benchlinks.txt

.github/workflows/bench-push-indexing.yml

@@ -12,11 +12,9 @@ jobs:
         timeout-minutes: 180 # 3h
         steps:
           - uses: actions/checkout@v3
-          - uses: actions-rs/toolchain@v1
+          - uses: dtolnay/rust-toolchain@1.81
             with:
               profile: minimal
-              toolchain: stable
-              override: true
 
           # Run benchmarks
           - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch main - Commit ${{ github.sha }}

.github/workflows/benchmarks-manual.yml

@@ -18,11 +18,9 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: dtolnay/rust-toolchain@1.81
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       # Set variables
       - name: Set current branch name
@@ -45,7 +43,7 @@ jobs:
       # Run benchmarks
       - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
         run: |
-          cd benchmarks
+          cd crates/benchmarks
           cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
 
       # Generate critcmp files

.github/workflows/benchmarks-pr.yml

@@ -13,11 +13,40 @@ jobs:
     runs-on: benchmarks
     timeout-minutes: 4320 # 72h
     steps:
-      - uses: actions-rs/toolchain@v1
+      - name: Check permissions
+        id: permission
+        env:
+          PR_AUTHOR: ${{github.event.issue.user.login }}
+          COMMENT_AUTHOR: ${{github.event.comment.user.login }}
+          REPOSITORY: ${{github.repository}}
+          PR_ID: ${{github.event.issue.number}}
+        run: |
+          PR_REPOSITORY=$(gh api /repos/"$REPOSITORY"/pulls/"$PR_ID" --jq .head.repo.full_name)
+          if $(gh api /repos/"$REPOSITORY"/collaborators/"$PR_AUTHOR"/permission --jq .user.permissions.push)
+          then
+            echo "::notice title=Authentication success::PR author authenticated"
+          else
+            echo "::error title=Authentication error::PR author doesn't have push permission on this repository"
+            exit 1
+          fi
+          if $(gh api /repos/"$REPOSITORY"/collaborators/"$COMMENT_AUTHOR"/permission --jq .user.permissions.push)
+          then
+            echo "::notice title=Authentication success::Comment author authenticated"
+          else
+            echo "::error title=Authentication error::Comment author doesn't have push permission on this repository"
+            exit 1
+          fi
+          if [ "$PR_REPOSITORY" = "$REPOSITORY" ]
+          then
+            echo "::notice title=Authentication success::PR started from main repository"
+          else
+            echo "::error title=Authentication error::PR started from a fork"
+            exit 1
+          fi
+
+      - uses: dtolnay/rust-toolchain@1.81
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       - name: Check for Command
         id: command
@@ -27,7 +56,7 @@ jobs:
           reaction-type: "eyes"
           repo-token: ${{ env.GH_TOKEN }}
 
-      - uses: xt0rted/pull-request-comment-branch@v2
+      - uses: xt0rted/pull-request-comment-branch@v3
         id: comment-branch
         with:
           repo_token: ${{ env.GH_TOKEN }}
@@ -59,7 +88,7 @@ jobs:
       # Run benchmarks
       - name: Run benchmarks - Dataset ${{ steps.command.outputs.command-arguments }} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
         run: |
-          cd benchmarks
+          cd crates/benchmarks
           cargo bench --bench ${{ steps.command.outputs.command-arguments }} -- --save-baseline ${{ steps.file.outputs.basename }}
 
       # Generate critcmp files

.github/workflows/benchmarks-push-indexing.yml

@@ -16,11 +16,9 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: dtolnay/rust-toolchain@1.81
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       # Set variables
       - name: Set current branch name
@@ -43,7 +41,7 @@ jobs:
       # Run benchmarks
       - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
         run: |
-          cd benchmarks
+          cd crates/benchmarks
           cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
 
       # Generate critcmp files

.github/workflows/benchmarks-push-search-geo.yml

@@ -15,11 +15,9 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: dtolnay/rust-toolchain@1.81
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       # Set variables
       - name: Set current branch name
@@ -42,7 +40,7 @@ jobs:
       # Run benchmarks
       - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
         run: |
-          cd benchmarks
+          cd crates/benchmarks
           cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
 
       # Generate critcmp files

.github/workflows/benchmarks-push-search-songs.yml

@@ -15,11 +15,9 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: dtolnay/rust-toolchain@1.81
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       # Set variables
       - name: Set current branch name
@@ -42,7 +40,7 @@ jobs:
       # Run benchmarks
       - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
         run: |
-          cd benchmarks
+          cd crates/benchmarks
           cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
 
       # Generate critcmp files

.github/workflows/benchmarks-push-search-wiki.yml

@@ -15,11 +15,9 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: dtolnay/rust-toolchain@1.81
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       # Set variables
       - name: Set current branch name
@@ -42,7 +40,7 @@ jobs:
       # Run benchmarks
       - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
         run: |
-          cd benchmarks
+          cd crates/benchmarks
           cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
 
       # Generate critcmp files

.github/workflows/flaky-tests.yml

@@ -1,4 +1,5 @@
 name: Look for flaky tests
+
 on:
   workflow_dispatch:
   schedule:
@@ -8,25 +9,22 @@ jobs:
   flaky:
     runs-on: ubuntu-latest
     container:
-      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
-      image: ubuntu:18.04
+      # Use ubuntu-20.04 to compile with glibc 2.28
+      image: ubuntu:20.04
     steps:
     - uses: actions/checkout@v3
     - name: Install needed dependencies
       run: |
         apt-get update && apt-get install -y curl
         apt-get install build-essential -y
-    - uses: actions-rs/toolchain@v1
-      with:
-        toolchain: stable
-        override: true
+    - uses: dtolnay/rust-toolchain@1.81
     - name: Install cargo-flaky
       run: cargo install cargo-flaky
     - name: Run cargo flaky in the dumps
-      run: cd dump; cargo flaky -i 100 --release
+      run: cd crates/dump; cargo flaky -i 100 --release
     - name: Run cargo flaky in the index-scheduler
-      run: cd index-scheduler; cargo flaky -i 100 --release
+      run: cd crates/index-scheduler; cargo flaky -i 100 --release
     - name: Run cargo flaky in the auth
-      run: cd meilisearch-auth; cargo flaky -i 100 --release
+      run: cd crates/meilisearch-auth; cargo flaky -i 100 --release
     - name: Run cargo flaky in meilisearch
-      run: cd meilisearch; cargo flaky -i 100 --release
+      run: cd crates/meilisearch; cargo flaky -i 100 --release

.github/workflows/fuzzer-indexing.yml

@@ -12,11 +12,9 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: dtolnay/rust-toolchain@1.81
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       # Run benchmarks
       - name: Run the fuzzer

.github/workflows/milestone-workflow.yml

@@ -110,6 +110,44 @@ jobs:
             --milestone $MILESTONE_VERSION \
             --assignee curquiza
 
+  create-update-version-issue:
+    needs: get-release-version
+    # Create the update-version issue even if the release is a patch release
+    if: github.event.action == 'created'
+    runs-on: ubuntu-latest
+    env:
+      ISSUE_TEMPLATE: issue-template.md
+    steps:
+      - uses: actions/checkout@v3
+      - name: Download the issue template
+        run: curl -s https://raw.githubusercontent.com/meilisearch/engine-team/main/issue-templates/update-version-issue.md > $ISSUE_TEMPLATE
+      - name: Create the issue
+        run: |
+          gh issue create \
+            --title "Update version in Cargo.toml for $MILESTONE_VERSION" \
+            --label 'maintenance' \
+            --body-file $ISSUE_TEMPLATE \
+            --milestone $MILESTONE_VERSION
+
+  create-update-openapi-issue:
+    needs: get-release-version
+    # Create the openAPI issue if the release is not only a patch release
+    if: github.event.action == 'created' && needs.get-release-version.outputs.is-patch == 'false'
+    runs-on: ubuntu-latest
+    env:
+      ISSUE_TEMPLATE: issue-template.md
+    steps:
+      - uses: actions/checkout@v3
+      - name: Download the issue template
+        run: curl -s https://raw.githubusercontent.com/meilisearch/engine-team/main/issue-templates/update-openapi-issue.md > $ISSUE_TEMPLATE
+      - name: Create the issue
+        run: |
+          gh issue create \
+            --title "Update Open API file for $MILESTONE_VERSION" \
+            --label 'maintenance' \
+            --body-file $ISSUE_TEMPLATE \
+            --milestone $MILESTONE_VERSION
+
 # ----------------
 # MILESTONE CLOSED
 # ----------------

.github/workflows/publish-apt-brew-pkg.yml

@@ -18,17 +18,14 @@ jobs:
     runs-on: ubuntu-latest
     needs: check-version
     container:
-      # Use ubuntu-18.04 to compile with glibc 2.27
-      image: ubuntu:18.04
+      # Use ubuntu-20.04 to compile with glibc 2.28
+      image: ubuntu:20.04
     steps:
     - name: Install needed dependencies
       run: |
         apt-get update && apt-get install -y curl
         apt-get install build-essential -y
-    - uses: actions-rs/toolchain@v1
-      with:
-        toolchain: stable
-        override: true
+    - uses: dtolnay/rust-toolchain@1.81
     - name: Install cargo-deb
       run: cargo install cargo-deb
     - uses: actions/checkout@v3

.github/workflows/publish-binaries.yml

@@ -37,18 +37,15 @@ jobs:
     runs-on: ubuntu-latest
     needs: check-version
     container:
-      # Use ubuntu-18.04 to compile with glibc 2.27
-      image: ubuntu:18.04
+      # Use ubuntu-20.04 to compile with glibc 2.28
+      image: ubuntu:20.04
     steps:
     - uses: actions/checkout@v3
     - name: Install needed dependencies
       run: |
         apt-get update && apt-get install -y curl
         apt-get install build-essential -y
-    - uses: actions-rs/toolchain@v1
-      with:
-        toolchain: stable
-        override: true
+    - uses: dtolnay/rust-toolchain@1.81
     - name: Build
       run: cargo build --release --locked
     # No need to upload binaries for dry run (cron)
@@ -68,9 +65,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-12, windows-2022]
+        os: [macos-13, windows-2022]
         include:
-          - os: macos-12
+          - os: macos-13
             artifact_name: meilisearch
             asset_name: meilisearch-macos-amd64
           - os: windows-2022
@@ -78,10 +75,7 @@ jobs:
             asset_name: meilisearch-windows-amd64.exe
     steps:
     - uses: actions/checkout@v3
-    - uses: actions-rs/toolchain@v1
-      with:
-        toolchain: stable
-        override: true
+    - uses: dtolnay/rust-toolchain@1.81
     - name: Build
       run: cargo build --release --locked
     # No need to upload binaries for dry run (cron)
@@ -96,7 +90,7 @@ jobs:
 
   publish-macos-apple-silicon:
     name: Publish binary for macOS silicon
-    runs-on: macos-12
+    runs-on: macos-13
     needs: check-version
     strategy:
       matrix:
@@ -107,12 +101,10 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
       - name: Installing Rust toolchain
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@1.81
         with:
-          toolchain: stable
           profile: minimal
           target: ${{ matrix.target }}
-          override: true
       - name: Cargo build
         uses: actions-rs/cargo@v1
         with:
@@ -132,9 +124,11 @@ jobs:
     name: Publish binary for aarch64
     runs-on: ubuntu-latest
     needs: check-version
+    env:
+      DEBIAN_FRONTEND: noninteractive
     container:
-      # Use ubuntu-18.04 to compile with glibc 2.27
-      image: ubuntu:18.04
+      # Use ubuntu-20.04 to compile with glibc 2.28
+      image: ubuntu:20.04
     strategy:
       matrix:
         include:
@@ -154,12 +148,10 @@ jobs:
           add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
           apt-get update -y && apt-get install -y docker-ce
       - name: Installing Rust toolchain
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@1.81
         with:
-          toolchain: stable
           profile: minimal
           target: ${{ matrix.target }}
-          override: true
       - name: Configure target aarch64 GNU
         ## Environment variable is not passed using env:
         ## LD gold won't work with MUSL
@@ -170,6 +162,9 @@ jobs:
           echo '[target.aarch64-unknown-linux-gnu]' >> ~/.cargo/config
           echo 'linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config
           echo 'JEMALLOC_SYS_WITH_LG_PAGE=16' >> $GITHUB_ENV
+      - name: Install a default toolchain that will be used to build cargo cross
+        run: |
+          rustup default stable
       - name: Cargo build
         uses: actions-rs/cargo@v1
         with:

.github/workflows/publish-docker-images.yml

@@ -80,10 +80,11 @@ jobs:
             type=ref,event=tag
             type=raw,value=nightly,enable=${{ github.event_name != 'push' }}
             type=semver,pattern=v{{major}}.{{minor}},enable=${{ steps.check-tag-format.outputs.stable == 'true' }}
+            type=semver,pattern=v{{major}},enable=${{ steps.check-tag-format.outputs.stable == 'true' }}
             type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable == 'true' && steps.check-tag-format.outputs.latest == 'true' }}
 
       - name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           push: true
           platforms: linux/amd64,linux/arm64

.github/workflows/sdks-tests.yml

@@ -52,7 +52,7 @@ jobs:
       - name: Setup .NET Core
         uses: actions/setup-dotnet@v4
         with:
-          dotnet-version: "6.0.x"
+          dotnet-version: "8.0.x"
       - name: Install dependencies
         run: dotnet restore
       - name: Build

.github/workflows/test-suite.yml

@@ -19,11 +19,11 @@ env:
 
 jobs:
   test-linux:
-    name: Tests on ubuntu-18.04
+    name: Tests on ubuntu-20.04
     runs-on: ubuntu-latest
     container:
-      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
-      image: ubuntu:18.04
+      # Use ubuntu-20.04 to compile with glibc 2.28
+      image: ubuntu:20.04
     steps:
       - uses: actions/checkout@v3
       - name: Install needed dependencies
@@ -31,12 +31,9 @@ jobs:
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
       - name: Setup test with Rust stable
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+        uses: dtolnay/rust-toolchain@1.81
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.7.1
+        uses: Swatinem/rust-cache@v2.7.7
       - name: Run cargo check without any default features
         uses: actions-rs/cargo@v1
         with:
@@ -54,15 +51,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-12, windows-2022]
+        os: [macos-13, windows-2022]
     steps:
       - uses: actions/checkout@v3
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.7.1
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+        uses: Swatinem/rust-cache@v2.7.7
+      - uses: dtolnay/rust-toolchain@1.81
       - name: Run cargo check without any default features
         uses: actions-rs/cargo@v1
         with:
@@ -78,8 +72,8 @@ jobs:
     name: Tests almost all features
     runs-on: ubuntu-latest
     container:
-      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
-      image: ubuntu:18.04
+      # Use ubuntu-20.04 to compile with glibc 2.28
+      image: ubuntu:20.04
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     steps:
       - uses: actions/checkout@v3
@@ -87,10 +81,7 @@ jobs:
         run: |
           apt-get update
           apt-get install --assume-yes build-essential curl
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+      - uses: dtolnay/rust-toolchain@1.81
       - name: Run cargo build with almost all features
         run: |
           cargo build --workspace --locked --release --features "$(cargo xtask list-features --exclude-feature cuda)"
@@ -102,7 +93,7 @@ jobs:
     name: Test disabled tokenization
     runs-on: ubuntu-latest
     container:
-      image: ubuntu:18.04
+      image: ubuntu:20.04
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     steps:
       - uses: actions/checkout@v3
@@ -110,13 +101,10 @@ jobs:
         run: |
           apt-get update
           apt-get install --assume-yes build-essential curl
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+      - uses: dtolnay/rust-toolchain@1.81
       - name: Run cargo tree without default features and check lindera is not present
         run: |
-          if cargo tree -f '{p} {f}' -e normal --no-default-features | grep -vqz lindera; then
+          if cargo tree -f '{p} {f}' -e normal --no-default-features | grep -qz lindera; then
             echo "lindera has been found in the sources and it shouldn't"
             exit 1
           fi
@@ -129,20 +117,17 @@ jobs:
     name: Run tests in debug
     runs-on: ubuntu-latest
     container:
-      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
-      image: ubuntu:18.04
+      # Use ubuntu-20.04 to compile with glibc 2.28
+      image: ubuntu:20.04
     steps:
       - uses: actions/checkout@v3
       - name: Install needed dependencies
         run: |
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+      - uses: dtolnay/rust-toolchain@1.81
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.7.1
+        uses: Swatinem/rust-cache@v2.7.7
       - name: Run tests in debug
         uses: actions-rs/cargo@v1
         with:
@@ -154,14 +139,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: dtolnay/rust-toolchain@1.81
         with:
           profile: minimal
-          toolchain: 1.75.0
-          override: true
           components: clippy
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.7.1
+        uses: Swatinem/rust-cache@v2.7.7
       - name: Run cargo clippy
         uses: actions-rs/cargo@v1
         with:
@@ -173,18 +156,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: dtolnay/rust-toolchain@1.81
         with:
           profile: minimal
-          toolchain: nightly
+          toolchain: nightly-2024-07-09
           override: true
           components: rustfmt
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.7.1
+        uses: Swatinem/rust-cache@v2.7.7
       - name: Run cargo fmt
         # Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file.
         # Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate
         # we are going to create an empty file where rustfmt expects it.
         run: |
-          echo -ne "\n" > benchmarks/benches/datasets_paths.rs
+          echo -ne "\n" > crates/benchmarks/benches/datasets_paths.rs
           cargo fmt --all -- --check

.github/workflows/update-cargo-toml-version.yml

@@ -18,11 +18,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: dtolnay/rust-toolchain@1.81
         with:
           profile: minimal
-          toolchain: stable
-          override: true
       - name: Install sd
         run: cargo install sd
       - name: Update Cargo.toml file

.gitignore

@@ -5,12 +5,12 @@
 **/*.json_lines
 **/*.rs.bk
 /*.mdb
-/query-history.txt
 /data.ms
 /snapshots
 /dumps
 /bench
 /_xtask_benchmark.ms
+/benchmarks
 
 # Snapshots
 ## ... large
@@ -19,4 +19,4 @@
 *.snap.new
 
 # Fuzzcheck data for the facet indexing fuzz test
-milli/fuzz/update::facet::incremental::fuzz::fuzz/
+crates/milli/fuzz/update::facet::incremental::fuzz::fuzz/

BENCHMARKS.md

@@ -0,0 +1,392 @@
+# Benchmarks
+
+Currently this repository hosts two kinds of benchmarks:
+
+1. The older "milli benchmarks", that use [criterion](https://github.com/bheisler/criterion.rs) and live in the "benchmarks" directory.
+2. The newer "bench" that are workload-based and so split between the [`workloads`](./workloads/) directory and the [`xtask::bench`](./xtask/src/bench/) module.
+
+This document describes the newer "bench" benchmarks. For more details on the "milli benchmarks", see [benchmarks/README.md](./benchmarks/README.md).
+
+## Design philosophy for the benchmarks
+
+The newer "bench" benchmarks are **integration** benchmarks, in the sense that they spawn an actual Meilisearch server and measure its performance end-to-end, including HTTP request overhead.
+
+Since this is prone to fluctuating, the benchmarks regain a bit of precision by measuring the runtime of the individual spans using the [logging machinery](./CONTRIBUTING.md#logging) of Meilisearch.
+
+A span roughly translates to a function call. The benchmark runner collects all the spans by name using the [logs route](https://github.com/orgs/meilisearch/discussions/721) and sums their runtime. The processed results are then sent to the [benchmark dashboard](https://bench.meilisearch.dev), which is in charge of storing and presenting the data.
+
+## Running the benchmarks
+
+Benchmarks can run locally or in CI.
+
+### Locally
+
+#### With a local benchmark dashboard
+
+The benchmarks dashboard lives in its [own repository](https://github.com/meilisearch/benchboard). We provide binaries for Ubuntu/Debian, but you can build from source for other platforms (MacOS should work as it was developed under that platform).
+
+Run the `benchboard` binary to create a fresh database of results. By default it will serve the results and the API to gather results on `http://localhost:9001`.
+
+From the Meilisearch repository, you can then run benchmarks with:
+
+```sh
+cargo xtask bench -- workloads/my_workload_1.json ..
+```
+
+This command will build and run Meilisearch locally on port 7700, so make sure that this port is available.
+To run benchmarks on a different commit, just use the usual git command to get back to the desired commit.
+
+#### Without a local benchmark dashboard
+
+To work with the raw results, you can also skip using a local benchmark dashboard.
+
+Run:
+
+```sh
+cargo xtask bench --no-dashboard -- workloads/my_workload_1.json workloads/my_workload_2.json ..
+```
+
+For processing the results, look at [Looking at benchmark results/Without dashboard](#without-dashboard).
+
+#### Sending a workload by hand
+
+Sometimes you want to visualize the metrics of a worlkoad that comes from a custom report.
+It is not quite easy to trick the benchboard in thinking that your report is legitimate but here are the commands you can run to upload your firefox report on a running benchboard.
+
+```bash
+# Name this hostname whatever you want
+echo '{ "hostname": "the-best-place" }' | xh PUT 'http://127.0.0.1:9001/api/v1/machine'
+
+# You'll receive an UUID from this command that we will call $invocation_uuid
+echo '{ "commit": { "sha1": "1234567", "commit_date": "2024-09-05 12:00:12.0 +00:00:00", "message": "A cool message" }, "machine_hostname": "the-best-place", "max_workloads": 1 }' | xh PUT 'http://127.0.0.1:9001/api/v1/invocation'
+
+# Just use UUID from the previous command
+# and you'll receive another UUID that we will call $workload_uuid
+echo '{ "invocation_uuid": "$invocation_uuid", "name": "toto", "max_runs": 1 }' | xh PUT 'http://127.0.0.1:9001/api/v1/workload'
+
+# And now use your $workload_uuid and the content of your firefox report
+# but don't forget to convert your firefox report from JSONLines into an object
+echo '{ "workload_uuid": "$workload_uuid", "data": $REPORT_JSON_DATA }' | xh PUT 'http://127.0.0.1:9001/api/v1/run'
+```
+
+### In CI
+
+We have dedicated runners to run workloads on CI. Currently, there are three ways of running the CI:
+
+1. Automatically, on every push to `main`.
+2. Manually, by clicking the [`Run workflow`](https://github.com/meilisearch/meilisearch/actions/workflows/bench-manual.yml) button and specifying the target reference (tag, commit or branch) as well as one or multiple workloads to run. The workloads must exist in the Meilisearch repository (conventionally, in the [`workloads`](./workloads/) directory) on the target reference. Globbing (e.g., `workloads/*.json`) works.
+3. Manually on a PR, by posting a comment containing a `/bench` command, followed by one or multiple workloads to run. Globbing works. The workloads must exist in the Meilisearch repository in the branch of the PR.
+  ```
+  /bench workloads/movies*.json /hackernews_1M.json
+  ```
+
+## Looking at benchmark results
+
+### On the dashboard
+
+Results are available on the global dashboard used by CI at <https://bench.meilisearch.dev> or on your [local dashboard](#with-a-local-benchmark-dashboard).
+
+The dashboard homepage presents three sections:
+
+1. The latest invocations (a call to `cargo xtask bench`, either local or by CI) with their reason (generally set to some helpful link in CI) and their status.
+2. The latest workloads ran on `main`.
+3. The latest workloads ran on other references.
+
+By default, the workload shows the total runtime delta with the latest applicable commit on `main`. The latest applicable commit is the latest commit for workload invocations that do not originate on `main`, and the latest previous commit for workload invocations that originate on `main`.
+
+You can explicitly request a detailed comparison by span with the `main` branch, the branch or origin, or any previous commit, by clicking the links at the bottom of the workload invocation.
+
+In the detailed comparison view, the spans are sorted by improvements, regressions, stable (no statistically significant change) and unstable (the span runtime is comparable to its standard deviation).
+
+You can click on the name of any span to get a box plot comparing the target commit with multiple commits of the selected branch.
+
+### Without dashboard
+
+After the workloads are done running, the reports will live in the Meilisearch repository, in the `bench/reports` directory (by default).
+
+You can then convert these reports into other formats.
+
+- To [Firefox profiler](https://profiler.firefox.com) format. Run:
+  ```sh
+  cd bench/reports
+  cargo run --release --bin trace-to-firefox -- my_workload_1-0-trace.json
+  ```
+  You can then upload the resulting `firefox-my_workload_1-0-trace.json` file to the online profiler.
+
+
+## Designing benchmark workloads
+
+Benchmark workloads conventionally live in the `workloads` directory of the Meilisearch repository.
+
+They are JSON files with the following structure (comments are not actually supported, to make your own, remove them or copy some existing workload file):
+
+```jsonc
+{
+  // Name of the workload. Must be unique to the workload, as it will be used to group results on the dashboard.
+  "name": "hackernews.ndjson_1M,no-threads",
+  // Number of consecutive runs of the commands that should be performed.
+  // Each run uses a fresh instance of Meilisearch and a fresh database.
+  // Each run produces its own report file.
+  "run_count": 3,
+  // List of arguments to add to the Meilisearch command line.
+  "extra_cli_args": ["--max-indexing-threads=1"],
+  // An expression that can be parsed as a comma-separated list of targets and levels
+  // as described in [tracing_subscriber's documentation](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/targets/struct.Targets.html#examples).
+  // The expression is used to filter the spans that are measured for profiling purposes.
+  // Optional, defaults to "indexing::=trace" (for indexing workloads), common other values is
+  // "search::=trace"
+  "target": "indexing::=trace",
+  // List of named assets that can be used in the commands.
+  "assets": {
+    // name of the asset.
+    // Must be unique at the workload level.
+    // For better results, the same asset (same sha256) should have the same name accross workloads.
+    // Having multiple assets with the same name and distinct hashes is supported accross workloads,
+    // but will lead to superfluous downloads.
+    //
+    // Assets are stored in the `bench/assets/` directory by default.
+    "hackernews-100_000.ndjson": {
+      // If the assets exists in the local filesystem (Meilisearch repository or for your local workloads)
+      // Its file path can be specified here.
+      // `null` if the asset should be downloaded from a remote location.
+      "local_location": null,
+      // URL of the remote location where the asset can be downloaded.
+      // Use the `--assets-key` of the runner to pass an API key in the `Authorization: Bearer` header of the download requests.
+      // `null` if the asset should be imported from a local location.
+      // if both local and remote locations are specified, then the local one is tried first, then the remote one
+      // if the file is locally missing or its hash differs.
+      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-100_000.ndjson",
+      // SHA256 of the asset.
+      // Optional, the `sha256` of the asset will be displayed during a run of the workload if it is missing.
+      // If present, the hash of the asset in the `bench/assets/` directory will be compared against this hash before
+      // running the workload. If the hashes differ, the asset will be downloaded anew.
+      "sha256": "60ecd23485d560edbd90d9ca31f0e6dba1455422f2a44e402600fbb5f7f1b213",
+      // Optional, one of "Auto", "Json", "NdJson" or "Raw".
+      // If missing, assumed to be "Auto".
+      // If "Auto", the format will be determined from the extension in the asset name.
+      "format": "NdJson"
+    },
+    "hackernews-200_000.ndjson": {
+      "local_location": null,
+      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-200_000.ndjson",
+      "sha256": "785b0271fdb47cba574fab617d5d332276b835c05dd86e4a95251cf7892a1685"
+    },
+    "hackernews-300_000.ndjson": {
+      "local_location": null,
+      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-300_000.ndjson",
+      "sha256": "de73c7154652eddfaf69cdc3b2f824d5c452f095f40a20a1c97bb1b5c4d80ab2"
+    },
+    "hackernews-400_000.ndjson": {
+      "local_location": null,
+      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-400_000.ndjson",
+      "sha256": "c1b00a24689110f366447e434c201c086d6f456d54ed1c4995894102794d8fe7"
+    },
+    "hackernews-500_000.ndjson": {
+      "local_location": null,
+      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-500_000.ndjson",
+      "sha256": "ae98f9dbef8193d750e3e2dbb6a91648941a1edca5f6e82c143e7996f4840083"
+    },
+    "hackernews-600_000.ndjson": {
+      "local_location": null,
+      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-600_000.ndjson",
+      "sha256": "b495fdc72c4a944801f786400f22076ab99186bee9699f67cbab2f21f5b74dbe"
+    },
+    "hackernews-700_000.ndjson": {
+      "local_location": null,
+      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-700_000.ndjson",
+      "sha256": "4b2c63974f3dabaa4954e3d4598b48324d03c522321ac05b0d583f36cb78a28b"
+    },
+    "hackernews-800_000.ndjson": {
+      "local_location": null,
+      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-800_000.ndjson",
+      "sha256": "cb7b6afe0e6caa1be111be256821bc63b0771b2a0e1fad95af7aaeeffd7ba546"
+    },
+    "hackernews-900_000.ndjson": {
+      "local_location": null,
+      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-900_000.ndjson",
+      "sha256": "e1154ddcd398f1c867758a93db5bcb21a07b9e55530c188a2917fdef332d3ba9"
+    },
+    "hackernews-1_000_000.ndjson": {
+      "local_location": null,
+      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-1_000_000.ndjson",
+      "sha256": "27e25efd0b68b159b8b21350d9af76938710cb29ce0393fa71b41c4f3c630ffe"
+    }
+  },
+  // Core of the workload.
+  // A list of commands to run sequentially.
+  // Optional: A precommand is a request to the Meilisearch instance that is executed before the profiling runs.
+  "precommands": [
+    {
+      // Meilisearch route to call. `http://localhost:7700/` will be prepended.
+      "route": "indexes/movies/settings",
+      // HTTP method to call.
+      "method": "PATCH",
+      // If applicable, body of the request.
+      // Optional, if missing, the body will be empty.
+      "body": {
+        // One of "empty", "inline" or "asset".
+        // If using "empty", you can skip the entire "body" key.
+        "inline": {
+          // when "inline" is used, the body is the JSON object that is the value of the `"inline"` key.
+          "displayedAttributes": [
+            "title",
+            "by",
+            "score",
+            "time"
+          ],
+          "searchableAttributes": [
+            "title"
+          ],
+          "filterableAttributes": [
+            "by"
+          ],
+          "sortableAttributes": [
+            "score",
+            "time"
+          ]
+        }
+      },
+      // Whether to wait before running the next request.
+      // One of:
+      // - DontWait: run the next command without waiting the response to this one.
+      // - WaitForResponse: run the next command as soon as the response from the server is received.
+      // - WaitForTask: run the next command once **all** the Meilisearch tasks created up to now have finished processing.
+      "synchronous": "WaitForTask"
+    }
+  ],
+  // A command is a request to the Meilisearch instance that is executed while the profiling runs.
+  "commands": [
+    {
+      "route": "indexes/movies/documents",
+      "method": "POST",
+      "body": {
+        // When using "asset", use the name of an asset as value to use the content of that asset as body.
+        // the content type is derived of the format of the asset:
+        // "NdJson" => "application/x-ndjson"
+        // "Json" => "application/json"
+        // "Raw" => "application/octet-stream"
+        // See [AssetFormat::to_content_type](https://github.com/meilisearch/meilisearch/blob/7b670a4afadb132ac4a01b6403108700501a391d/xtask/src/bench/assets.rs#L30)
+        // for details and up-to-date list.
+        "asset": "hackernews-100_000.ndjson"
+      },
+      "synchronous": "WaitForTask"
+    },
+    {
+      "route": "indexes/movies/documents",
+      "method": "POST",
+      "body": {
+        "asset": "hackernews-200_000.ndjson"
+      },
+      "synchronous": "WaitForResponse"
+    },
+    {
+      "route": "indexes/movies/documents",
+      "method": "POST",
+      "body": {
+        "asset": "hackernews-300_000.ndjson"
+      },
+      "synchronous": "WaitForResponse"
+    },
+    {
+      "route": "indexes/movies/documents",
+      "method": "POST",
+      "body": {
+        "asset": "hackernews-400_000.ndjson"
+      },
+      "synchronous": "WaitForResponse"
+    },
+    {
+      "route": "indexes/movies/documents",
+      "method": "POST",
+      "body": {
+        "asset": "hackernews-500_000.ndjson"
+      },
+      "synchronous": "WaitForResponse"
+    },
+    {
+      "route": "indexes/movies/documents",
+      "method": "POST",
+      "body": {
+        "asset": "hackernews-600_000.ndjson"
+      },
+      "synchronous": "WaitForResponse"
+    },
+    {
+      "route": "indexes/movies/documents",
+      "method": "POST",
+      "body": {
+        "asset": "hackernews-700_000.ndjson"
+      },
+      "synchronous": "WaitForResponse"
+    },
+    {
+      "route": "indexes/movies/documents",
+      "method": "POST",
+      "body": {
+        "asset": "hackernews-800_000.ndjson"
+      },
+      "synchronous": "WaitForResponse"
+    },
+    {
+      "route": "indexes/movies/documents",
+      "method": "POST",
+      "body": {
+        "asset": "hackernews-900_000.ndjson"
+      },
+      "synchronous": "WaitForResponse"
+    },
+    {
+      "route": "indexes/movies/documents",
+      "method": "POST",
+      "body": {
+        "asset": "hackernews-1_000_000.ndjson"
+      },
+      "synchronous": "WaitForTask"
+    }
+  ]
+}
+```
+
+### Adding new assets
+
+Assets reside in our DigitalOcean S3 space. Assuming you have team access to the DigitalOcean S3 space:
+
+1. go to <https://cloud.digitalocean.com/spaces/milli-benchmarks?i=d1c552&path=bench%2Fdatasets%2F>
+2. upload your dataset:
+   1. if your dataset is a single file, upload that single file using the "upload" button,
+   2. otherwise, create a folder using the "create folder" button, then inside that folder upload your individual files.
+
+## Upgrading `https://bench.meilisearch.dev`
+
+The URL of the server is in our password manager (look for "benchboard").
+
+1. Make the needed modifications on the [benchboard repository](https://github.com/meilisearch/benchboard) and merge them to main.
+2. Publish a new release to produce the Ubuntu/Debian binary.
+3. Download the binary locally, send it to the server:
+  ```
+  scp -6 ~/Downloads/benchboard root@\[<ipv6-address>\]:/bench/new-benchboard
+  ```
+  Note that the ipv6 must be between escaped square brackets for SCP.
+4. SSH to the server:
+  ```
+  ssh root@<ipv6-address>
+  ```
+  Note the ipv6 must **NOT** be between escaped square brackets for SSH 🥲
+5. On the server, set the correct permissions for the new binary:
+   ```
+   chown bench:bench /bench/new-benchboard
+   chmod 700 /bench/new-benchboard
+   ```
+6. On the server, move the new binary to the location of the running binary (if unsure, start by making a backup of the running binary):
+  ```
+  mv /bench/{new-,}benchboard
+  ```
+7. Restart the benchboard service.
+  ```
+  systemctl restart benchboard
+  ```
+8. Check that the service runs correctly.
+  ```
+  systemctl status benchboard
+  ```
+9. Check the availability of the service by going to <https://bench.meilisearch.dev> on your browser.

CONTRIBUTING.md

@@ -4,7 +4,7 @@ First, thank you for contributing to Meilisearch! The goal of this document is t
 
 Remember that there are many ways to contribute other than writing code: writing [tutorials or blog posts](https://github.com/meilisearch/awesome-meilisearch), improving [the documentation](https://github.com/meilisearch/documentation), submitting [bug reports](https://github.com/meilisearch/meilisearch/issues/new?assignees=&labels=&template=bug_report.md&title=) and [feature requests](https://github.com/meilisearch/product/discussions/categories/feedback-feature-proposal)...
 
-The code in this repository is only concerned with managing multiple indexes, handling the update store, and exposing an HTTP API. Search and indexation are the domain of our core engine, [`milli`](https://github.com/meilisearch/milli), while tokenization is handled by [our `charabia` library](https://github.com/meilisearch/charabia/).
+Meilisearch can manage multiple indexes, handle the update store, and expose an HTTP API. Search and indexation are the domain of our core engine, [`milli`](https://github.com/meilisearch/meilisearch/tree/main/milli), while tokenization is handled by [our `charabia` library](https://github.com/meilisearch/charabia/).
 
 If Meilisearch does not offer optimized support for your language, please consider contributing to `charabia` by following the [CONTRIBUTING.md file](https://github.com/meilisearch/charabia/blob/main/CONTRIBUTING.md) and integrating your intended normalizer/segmenter.
 
@@ -52,6 +52,20 @@ cargo test
 
 This command will be triggered to each PR as a requirement for merging it.
 
+#### Faster build
+
+You can set the `LINDERA_CACHE` environment variable to speed up your successive builds by up to 2 minutes.
+It'll store some built artifacts in the directory of your choice.
+
+We recommend using the standard `$HOME/.cache/lindera` directory:
+```sh
+export LINDERA_CACHE=$HOME/.cache/lindera
+```
+
+Furthermore, you can improve incremental compilation by setting the `MEILI_NO_VERGEN` environment variable.
+Setting this variable will prevent the Meilisearch binary from being rebuilt each time the directory that hosts the Meilisearch repository changes.
+Do not enable this environment variable for production builds (as it will break the `version` route, among other things).
+
 #### Snapshot-based tests
 
 We are using [insta](https://insta.rs) to perform snapshot-based testing.
@@ -63,7 +77,7 @@ Furthermore, we provide some macros on top of insta, notably a way to use snapsh
 
 To effectively debug snapshot-based hashes, we recommend you export the `MEILI_TEST_FULL_SNAPS` environment variable so that snapshot are fully created locally:
 
-```
+```sh
 export MEILI_TEST_FULL_SNAPS=true # add this to your .bashrc, .zshrc, ...
 ```
 
@@ -81,6 +95,30 @@ Meilisearch follows the [cargo xtask](https://github.com/matklad/cargo-xtask) wo
 
 Run `cargo xtask --help` from the root of the repository to find out what is available.
 
+### Logging
+
+Meilisearch uses [`tracing`](https://lib.rs/crates/tracing) for logging purposes. Tracing logs are structured and can be displayed as JSON to the end user, so prefer passing arguments as fields rather than interpolating them in the message.
+
+Refer to the [documentation](https://docs.rs/tracing/0.1.40/tracing/index.html#using-the-macros) for the syntax of the spans and events.
+
+Logging spans are used for 3 distinct purposes:
+
+1. Regular logging
+2. Profiling
+3. Benchmarking
+
+As a result, the spans should follow some rules:
+
+- They should not be put on functions that are called too often. That is because opening and closing a span causes some overhead. For regular logging, avoid putting spans on functions that are taking less than a few hundred nanoseconds. For profiling or benchmarking, avoid putting spans on functions that are taking less than a few microseconds.
+- For profiling and benchmarking, use the `TRACE` level.
+- For profiling and benchmarking, use the following `target` prefixes:
+  - `indexing::` for spans meant when profiling the indexing operations.
+  - `search::` for spans meant when profiling the search operations.
+
+### Benchmarking
+
+See [BENCHMARKS.md](./BENCHMARKS.md)
+
 ## Git Guidelines
 
 ### Git Branches

Cargo.toml

@@ -1,27 +1,28 @@
 [workspace]
 resolver = "2"
 members = [
-    "meilisearch",
-    "meilitool",
-    "meilisearch-types",
-    "meilisearch-auth",
-    "meili-snap",
-    "index-scheduler",
-    "dump",
-    "file-store",
-    "permissive-json-pointer",
-    "milli",
-    "filter-parser",
-    "flatten-serde-json",
-    "json-depth-checker",
-    "benchmarks",
-    "fuzzers",
-    "tracing-trace",
-    "xtask", "build-info",
+    "crates/meilisearch",
+    "crates/meilitool",
+    "crates/meilisearch-types",
+    "crates/meilisearch-auth",
+    "crates/meili-snap",
+    "crates/index-scheduler",
+    "crates/dump",
+    "crates/file-store",
+    "crates/permissive-json-pointer",
+    "crates/milli",
+    "crates/filter-parser",
+    "crates/flatten-serde-json",
+    "crates/json-depth-checker",
+    "crates/benchmarks",
+    "crates/fuzzers",
+    "crates/tracing-trace",
+    "crates/xtask",
+    "crates/build-info",
 ]
 
 [workspace.package]
-version = "1.7.1"
+version = "1.13.2"
 authors = [
     "Quentin de Quelen <quentin@dequelen.me>",
     "Clément Renault <clement@meilisearch.com>",
@@ -42,24 +43,3 @@ opt-level = 3
 opt-level = 3
 [profile.dev.package.roaring]
 opt-level = 3
-
-[profile.dev.package.lindera-ipadic-builder]
-opt-level = 3
-[profile.dev.package.encoding]
-opt-level = 3
-[profile.dev.package.yada]
-opt-level = 3
-
-[profile.release.package.lindera-ipadic-builder]
-opt-level = 3
-[profile.release.package.encoding]
-opt-level = 3
-[profile.release.package.yada]
-opt-level = 3
-
-[profile.bench.package.lindera-ipadic-builder]
-opt-level = 3
-[profile.bench.package.encoding]
-opt-level = 3
-[profile.bench.package.yada]
-opt-level = 3

Dockerfile

@@ -1,7 +1,7 @@
 # Compile
-FROM    rust:1.75.0-alpine3.18 AS compiler
+FROM    rust:1.81.0-alpine3.20 AS compiler
 
-RUN     apk add -q --update-cache --no-cache build-base openssl-dev
+RUN     apk add -q --no-cache build-base openssl-dev
 
 WORKDIR /
 
@@ -20,13 +20,13 @@ RUN     set -eux; \
         cargo build --release -p meilisearch -p meilitool
 
 # Run
-FROM    alpine:3.16
+FROM    alpine:3.20
+LABEL   org.opencontainers.image.source="https://github.com/meilisearch/meilisearch"
 
 ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
 ENV     MEILI_SERVER_PROVIDER docker
 
-RUN     apk update --quiet \
-        && apk add -q --no-cache libgcc tini curl
+RUN     apk add -q --no-cache libgcc tini curl
 
 # add meilisearch and meilitool to the `/bin` so you can run it from anywhere
 # and it's easy to find.

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019-2024 Meili SAS
+Copyright (c) 2019-2025 Meili SAS
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -25,7 +25,7 @@
 
 <p align="center">⚡ A lightning-fast search engine that fits effortlessly into your apps, websites, and workflow 🔍</p>
 
-Meilisearch helps you shape a delightful search experience in a snap, offering features that work out-of-the-box to speed up your workflow.
+[Meilisearch](https://www.meilisearch.com?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=intro) helps you shape a delightful search experience in a snap, offering features that work out of the box to speed up your workflow.
 
 <p align="center" name="demo">
   <a href="https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-gif#gh-light-mode-only" target="_blank">
@@ -36,34 +36,42 @@ Meilisearch helps you shape a delightful search experience in a snap, offering f
   </a>
 </p>
 
-🔥 [**Try it!**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-link) 🔥
+## 🖥 Examples
+
+- [**Movies**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=organization) — An application to help you find streaming platforms to watch movies using [hybrid search](https://www.meilisearch.com/solutions/hybrid-search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos).
+- [**Ecommerce**](https://ecommerce.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Ecommerce website using disjunctive [facets](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos), range and rating filtering, and pagination.
+- [**Songs**](https://music.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search through 47 million of songs.
+- [**SaaS**](https://saas.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search for contacts, deals, and companies in this [multi-tenant](https://www.meilisearch.com/docs/learn/security/multitenancy_tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) CRM application.
+
+See the list of all our example apps in our [demos repository](https://github.com/meilisearch/demos).
 
 ## ✨ Features
-
-- **Search-as-you-type:** find search results in less than 50 milliseconds
-- **[Typo tolerance](https://www.meilisearch.com/docs/learn/configuration/typo_tolerance?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** get relevant matches even when queries contain typos and misspellings
+- **Hybrid search:** Combine the best of both [semantic](https://www.meilisearch.com/docs/learn/experimental/vector_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features) & full-text search to get the most relevant results
+- **Search-as-you-type:** Find & display results in less than 50 milliseconds to provide an intuitive experience
+- **[Typo tolerance](https://www.meilisearch.com/docs/learn/relevancy/typo_tolerance_settings?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** get relevant matches even when queries contain typos and misspellings
 - **[Filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features) and [faceted search](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** enhance your users' search experience with custom filters and build a faceted search interface in a few lines of code
 - **[Sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** sort results based on price, date, or pretty much anything else your users need
-- **[Synonym support](https://www.meilisearch.com/docs/learn/configuration/synonyms?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** configure synonyms to include more relevant content in your search results
+- **[Synonym support](https://www.meilisearch.com/docs/learn/relevancy/synonyms?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** configure synonyms to include more relevant content in your search results
 - **[Geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** filter and sort documents based on geographic data
 - **[Extensive language support](https://www.meilisearch.com/docs/learn/what_is_meilisearch/language?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** search datasets in any language, with optimized support for Chinese, Japanese, Hebrew, and languages using the Latin alphabet
 - **[Security management](https://www.meilisearch.com/docs/learn/security/master_api_keys?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** control which users can access what data with API keys that allow fine-grained permissions handling
-- **[Multi-Tenancy](https://www.meilisearch.com/docs/learn/security/tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** personalize search results for any number of application tenants
+- **[Multi-Tenancy](https://www.meilisearch.com/docs/learn/security/multitenancy_tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** personalize search results for any number of application tenants
 - **Highly Customizable:** customize Meilisearch to your specific needs or use our out-of-the-box and hassle-free presets
 - **[RESTful API](https://www.meilisearch.com/docs/reference/api/overview?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** integrate Meilisearch in your technical stack with our plugins and SDKs
+- **AI-ready:** works out of the box with [langchain](https://www.meilisearch.com/with/langchain) and the [model context protocol](https://github.com/meilisearch/meilisearch-mcp)
 - **Easy to install, deploy, and maintain**
 
 ## 📖 Documentation
 
-You can consult Meilisearch's documentation at [https://www.meilisearch.com/docs](https://www.meilisearch.com/docs/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=docs).
+You can consult Meilisearch's documentation at [meilisearch.com/docs](https://www.meilisearch.com/docs/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=docs).
 
 ## 🚀 Getting started
 
-For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [Quick Start](https://www.meilisearch.com/docs/learn/getting_started/quick_start?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) guide.
+For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [documentation](https://www.meilisearch.com/docs?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) guide.
 
-## ⚡ Supercharge your Meilisearch experience
+## 🌍 Supercharge your Meilisearch experience
 
-Say goodbye to server deployment and manual updates with [Meilisearch Cloud](https://www.meilisearch.com/cloud?utm_campaign=oss&utm_source=github&utm_medium=meilisearch). No credit card required.
+Say goodbye to server deployment and manual updates with [Meilisearch Cloud](https://www.meilisearch.com/cloud?utm_campaign=oss&utm_source=github&utm_medium=meilisearch). Additional features include analytics & monitoring in many regions around the world. No credit card is required.
 
 ## 🧰 SDKs & integration tools
 
@@ -83,15 +91,15 @@ Finally, for more in-depth information, refer to our articles explaining fundame
 
 ## 📊 Telemetry
 
-Meilisearch collects **anonymized** data from users to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) whenever you want.
+Meilisearch collects **anonymized** user data to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) whenever you want.
 
-To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Don't forget to include your `Instance UID` in the message, as this helps us quickly find and delete your data.
+To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Remember to include your `Instance UID` in the message, as this helps us quickly find and delete your data.
 
 If you want to know more about the kind of data we collect and what we use it for, check the [telemetry section](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) of our documentation.
 
 ## 📫 Get in touch!
 
-Meilisearch is a search engine created by [Meili](https://www.welcometothejungle.com/en/companies/meilisearch), a software development company based in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact)
+Meilisearch is a search engine created by [Meili]([https://www.welcometothejungle.com/en/companies/meilisearch](https://www.meilisearch.com/careers)), a software development company headquartered in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact)
 
 🗞 [Subscribe to our newsletter](https://meilisearch.us2.list-manage.com/subscribe?u=27870f7b71c908a8b359599fb&id=79582d828e) if you don't want to miss any updates! We promise we won't clutter your mailbox: we only send one edition every two months.
 
@@ -105,11 +113,11 @@ Thank you for your support!
 
 ## 👩‍💻 Contributing
 
-Meilisearch is, and will always be, open-source! If you want to contribute to the project, please take a look at [our contribution guidelines](CONTRIBUTING.md).
+Meilisearch is, and will always be, open-source! If you want to contribute to the project, please look at [our contribution guidelines](CONTRIBUTING.md).
 
 ## 📦 Versioning
 
-Meilisearch releases and their associated binaries are available [in this GitHub page](https://github.com/meilisearch/meilisearch/releases).
+Meilisearch releases and their associated binaries are available on the project's [releases page](https://github.com/meilisearch/meilisearch/releases).
 
 The binaries are versioned following [SemVer conventions](https://semver.org/). To know more, read our [versioning policy](https://github.com/meilisearch/engine-team/blob/main/resources/versioning-policy.md).
 

bors.toml

@@ -1,6 +1,6 @@
 status = [
-    'Tests on ubuntu-18.04',
-    'Tests on macos-12',
+    'Tests on ubuntu-20.04',
+    'Tests on macos-13',
     'Tests on windows-2022',
     'Run Clippy',
     'Run Rustfmt',

crates/benchmarks/Cargo.toml

@@ -11,24 +11,27 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.79"
-csv = "1.3.0"
+anyhow = "1.0.95"
+bumpalo = "3.16.0"
+csv = "1.3.1"
+memmap2 = "0.9.5"
 milli = { path = "../milli" }
-mimalloc = { version = "0.1.39", default-features = false }
-serde_json = { version = "1.0.111", features = ["preserve_order"] }
+mimalloc = { version = "0.1.43", default-features = false }
+serde_json = { version = "1.0.135", features = ["preserve_order"] }
+tempfile = "3.15.0"
 
 [dev-dependencies]
 criterion = { version = "0.5.1", features = ["html_reports"] }
 rand = "0.8.5"
 rand_chacha = "0.3.1"
-roaring = "0.10.2"
+roaring = "0.10.10"
 
 [build-dependencies]
-anyhow = "1.0.79"
-bytes = "1.5.0"
+anyhow = "1.0.95"
+bytes = "1.9.0"
 convert_case = "0.6.0"
-flate2 = "1.0.28"
-reqwest = { version = "0.11.23", features = ["blocking", "rustls-tls"], default-features = false }
+flate2 = "1.0.35"
+reqwest = { version = "0.12.12", features = ["blocking", "rustls-tls"], default-features = false }
 
 [features]
 default = ["milli/all-tokenizations"]

crates/benchmarks/benches/search_geo.rs

@@ -5,6 +5,7 @@ use criterion::{criterion_group, criterion_main};
 use milli::update::Settings;
 use utils::Conf;
 
+#[cfg(not(windows))]
 #[global_allocator]
 static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
 

crates/benchmarks/benches/search_songs.rs

@@ -5,6 +5,7 @@ use criterion::{criterion_group, criterion_main};
 use milli::update::Settings;
 use utils::Conf;
 
+#[cfg(not(windows))]
 #[global_allocator]
 static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
 

crates/benchmarks/benches/search_wiki.rs

@@ -5,6 +5,7 @@ use criterion::{criterion_group, criterion_main};
 use milli::update::Settings;
 use utils::Conf;
 
+#[cfg(not(windows))]
 #[global_allocator]
 static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
 

crates/benchmarks/benches/utils.rs

@@ -1,17 +1,19 @@
 #![allow(dead_code)]
 
 use std::fs::{create_dir_all, remove_dir_all, File};
-use std::io::{self, BufRead, BufReader, Cursor, Read, Seek};
-use std::num::ParseFloatError;
+use std::io::{self, BufReader, BufWriter, Read};
 use std::path::Path;
-use std::str::FromStr;
+use std::str::FromStr as _;
 
+use anyhow::Context;
+use bumpalo::Bump;
 use criterion::BenchmarkId;
-use milli::documents::{DocumentsBatchBuilder, DocumentsBatchReader};
+use memmap2::Mmap;
 use milli::heed::EnvOpenOptions;
-use milli::update::{
-    IndexDocuments, IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig, Settings,
-};
+use milli::progress::Progress;
+use milli::update::new::indexer;
+use milli::update::{IndexDocumentsMethod, IndexerConfig, Settings};
+use milli::vector::EmbeddingConfigs;
 use milli::{Criterion, Filter, Index, Object, TermsMatchingStrategy};
 use serde_json::Value;
 
@@ -65,8 +67,8 @@ pub fn base_setup(conf: &Conf) -> Index {
 
     let mut options = EnvOpenOptions::new();
     options.map_size(100 * 1024 * 1024 * 1024); // 100 GB
-    options.max_readers(10);
-    let index = Index::new(options, conf.database_name).unwrap();
+    options.max_readers(100);
+    let index = Index::new(options, conf.database_name, true).unwrap();
 
     let config = IndexerConfig::default();
     let mut wtxn = index.write_txn().unwrap();
@@ -92,18 +94,44 @@ pub fn base_setup(conf: &Conf) -> Index {
 
     let config = IndexerConfig::default();
     let mut wtxn = index.write_txn().unwrap();
-    let indexing_config = IndexDocumentsConfig {
-        autogenerate_docids: conf.primary_key.is_none(),
-        update_method: IndexDocumentsMethod::ReplaceDocuments,
-        ..Default::default()
-    };
-    let builder =
-        IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| (), || false).unwrap();
+    let rtxn = index.read_txn().unwrap();
+    let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
+    let mut new_fields_ids_map = db_fields_ids_map.clone();
+
     let documents = documents_from(conf.dataset, conf.dataset_format);
-    let (builder, user_error) = builder.add_documents(documents).unwrap();
-    user_error.unwrap();
-    builder.execute().unwrap();
+    let mut indexer = indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+    indexer.add_documents(&documents).unwrap();
+
+    let indexer_alloc = Bump::new();
+    let (document_changes, _operation_stats, primary_key) = indexer
+        .into_changes(
+            &indexer_alloc,
+            &index,
+            &rtxn,
+            None,
+            &mut new_fields_ids_map,
+            &|| false,
+            Progress::default(),
+        )
+        .unwrap();
+
+    indexer::index(
+        &mut wtxn,
+        &index,
+        &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
+        config.grenad_parameters(),
+        &db_fields_ids_map,
+        new_fields_ids_map,
+        primary_key,
+        &document_changes,
+        EmbeddingConfigs::default(),
+        &|| false,
+        &Progress::default(),
+    )
+    .unwrap();
+
     wtxn.commit().unwrap();
+    drop(rtxn);
 
     index
 }
@@ -140,49 +168,96 @@ pub fn run_benches(c: &mut criterion::Criterion, confs: &[Conf]) {
     }
 }
 
-pub fn documents_from(filename: &str, filetype: &str) -> DocumentsBatchReader<impl BufRead + Seek> {
-    let reader = File::open(filename)
-        .unwrap_or_else(|_| panic!("could not find the dataset in: {}", filename));
-    let reader = BufReader::new(reader);
-    let documents = match filetype {
-        "csv" => documents_from_csv(reader).unwrap(),
-        "json" => documents_from_json(reader).unwrap(),
-        "jsonl" => documents_from_jsonl(reader).unwrap(),
-        otherwise => panic!("invalid update format {:?}", otherwise),
-    };
-    DocumentsBatchReader::from_reader(Cursor::new(documents)).unwrap()
+pub fn documents_from(filename: &str, filetype: &str) -> Mmap {
+    let file = File::open(filename)
+        .unwrap_or_else(|_| panic!("could not find the dataset in: {filename}"));
+    match filetype {
+        "csv" => documents_from_csv(file).unwrap(),
+        "json" => documents_from_json(file).unwrap(),
+        "jsonl" => documents_from_jsonl(file).unwrap(),
+        otherwise => panic!("invalid update format {otherwise:?}"),
+    }
 }
 
-fn documents_from_jsonl(reader: impl BufRead) -> anyhow::Result<Vec<u8>> {
-    let mut documents = DocumentsBatchBuilder::new(Vec::new());
+fn documents_from_jsonl(file: File) -> anyhow::Result<Mmap> {
+    unsafe { Mmap::map(&file).map_err(Into::into) }
+}
 
-    for result in serde_json::Deserializer::from_reader(reader).into_iter::<Object>() {
-        let object = result?;
-        documents.append_json_object(&object)?;
+fn documents_from_json(file: File) -> anyhow::Result<Mmap> {
+    let reader = BufReader::new(file);
+    let documents: Vec<milli::Object> = serde_json::from_reader(reader)?;
+    let mut output = tempfile::tempfile().map(BufWriter::new)?;
+
+    for document in documents {
+        serde_json::to_writer(&mut output, &document)?;
     }
 
-    documents.into_inner().map_err(Into::into)
+    let file = output.into_inner()?;
+    unsafe { Mmap::map(&file).map_err(Into::into) }
 }
 
-fn documents_from_json(reader: impl BufRead) -> anyhow::Result<Vec<u8>> {
-    let mut documents = DocumentsBatchBuilder::new(Vec::new());
+fn documents_from_csv(file: File) -> anyhow::Result<Mmap> {
+    let output = tempfile::tempfile()?;
+    let mut output = BufWriter::new(output);
+    let mut reader = csv::ReaderBuilder::new().from_reader(file);
 
-    documents.append_json_array(reader)?;
+    let headers = reader.headers().context("while retrieving headers")?.clone();
+    let typed_fields: Vec<_> = headers.iter().map(parse_csv_header).collect();
+    let mut object: serde_json::Map<_, _> =
+        typed_fields.iter().map(|(k, _)| (k.to_string(), Value::Null)).collect();
 
-    documents.into_inner().map_err(Into::into)
-}
+    let mut line = 0;
+    let mut record = csv::StringRecord::new();
+    while reader.read_record(&mut record).context("while reading a record")? {
+        // We increment here and not at the end of the loop
+        // to take the header offset into account.
+        line += 1;
 
-fn documents_from_csv(reader: impl BufRead) -> anyhow::Result<Vec<u8>> {
-    let csv = csv::Reader::from_reader(reader);
+        // Reset the document values
+        object.iter_mut().for_each(|(_, v)| *v = Value::Null);
 
-    let mut documents = DocumentsBatchBuilder::new(Vec::new());
-    documents.append_csv(csv)?;
+        for (i, (name, atype)) in typed_fields.iter().enumerate() {
+            let value = &record[i];
+            let trimmed_value = value.trim();
+            let value = match atype {
+                AllowedType::Number if trimmed_value.is_empty() => Value::Null,
+                AllowedType::Number => {
+                    match trimmed_value.parse::<i64>() {
+                        Ok(integer) => Value::from(integer),
+                        Err(_) => match trimmed_value.parse::<f64>() {
+                            Ok(float) => Value::from(float),
+                            Err(error) => {
+                                anyhow::bail!("document format error on line {line}: {error}. For value: {value}")
+                            }
+                        },
+                    }
+                }
+                AllowedType::Boolean if trimmed_value.is_empty() => Value::Null,
+                AllowedType::Boolean => match trimmed_value.parse::<bool>() {
+                    Ok(bool) => Value::from(bool),
+                    Err(error) => {
+                        anyhow::bail!(
+                            "document format error on line {line}: {error}. For value: {value}"
+                        )
+                    }
+                },
+                AllowedType::String if value.is_empty() => Value::Null,
+                AllowedType::String => Value::from(value),
+            };
 
-    documents.into_inner().map_err(Into::into)
+            *object.get_mut(name).expect("encountered an unknown field") = value;
+        }
+
+        serde_json::to_writer(&mut output, &object).context("while writing to disk")?;
+    }
+
+    let output = output.into_inner()?;
+    unsafe { Mmap::map(&output).map_err(Into::into) }
 }
 
 enum AllowedType {
     String,
+    Boolean,
     Number,
 }
 
@@ -191,8 +266,9 @@ fn parse_csv_header(header: &str) -> (String, AllowedType) {
     match header.rsplit_once(':') {
         Some((field_name, field_type)) => match field_type {
             "string" => (field_name.to_string(), AllowedType::String),
+            "boolean" => (field_name.to_string(), AllowedType::Boolean),
             "number" => (field_name.to_string(), AllowedType::Number),
-            // we may return an error in this case.
+            // if the pattern isn't recognized, we keep the whole field.
             _otherwise => (header.to_string(), AllowedType::String),
         },
         None => (header.to_string(), AllowedType::String),
@@ -230,10 +306,13 @@ impl<R: Read> Iterator for CSVDocumentDeserializer<R> {
                 for ((field_name, field_type), value) in
                     self.headers.iter().zip(csv_document.into_iter())
                 {
-                    let parsed_value: Result<Value, ParseFloatError> = match field_type {
+                    let parsed_value: anyhow::Result<Value> = match field_type {
                         AllowedType::Number => {
                             value.parse::<f64>().map(Value::from).map_err(Into::into)
                         }
+                        AllowedType::Boolean => {
+                            value.parse::<bool>().map(Value::from).map_err(Into::into)
+                        }
                         AllowedType::String => Ok(Value::String(value.to_string())),
                     };
 

crates/build-info/Cargo.toml

@@ -11,8 +11,8 @@ license.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-time = { version = "0.3.34", features = ["parsing"] }
+time = { version = "0.3.37", features = ["parsing"] }
 
 [build-dependencies]
-anyhow = "1.0.80"
-vergen-git2 = "1.0.0-beta.2"
+anyhow = "1.0.95"
+vergen-git2 = "1.0.2"

crates/build-info/build.rs

@@ -5,6 +5,13 @@ fn main() {
 }
 
 fn emit_git_variables() -> anyhow::Result<()> {
+    println!("cargo::rerun-if-env-changed=MEILI_NO_VERGEN");
+
+    let has_vergen =
+        !matches!(std::env::var_os("MEILI_NO_VERGEN"), Some(x) if x != "false" && x != "0");
+
+    anyhow::ensure!(has_vergen, "disabled via `MEILI_NO_VERGEN`");
+
     // Note: any code that needs VERGEN_ environment variables should take care to define them manually in the Dockerfile and pass them
     // in the corresponding GitHub workflow (publish_docker.yml).
     // This is due to the Dockerfile building the binary outside of the git directory.

crates/dump/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+name = "dump"
+publish = false
+
+version.workspace = true
+authors.workspace = true
+description.workspace = true
+edition.workspace = true
+homepage.workspace = true
+readme.workspace = true
+license.workspace = true
+
+[dependencies]
+anyhow = "1.0.95"
+flate2 = "1.0.35"
+http = "1.2.0"
+meilisearch-types = { path = "../meilisearch-types" }
+once_cell = "1.20.2"
+regex = "1.11.1"
+roaring = { version = "0.10.10", features = ["serde"] }
+serde = { version = "1.0.217", features = ["derive"] }
+serde_json = { version = "1.0.135", features = ["preserve_order"] }
+tar = "0.4.43"
+tempfile = "3.15.0"
+thiserror = "2.0.9"
+time = { version = "0.3.37", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+tracing = "0.1.41"
+uuid = { version = "1.11.0", features = ["serde", "v4"] }
+
+[dev-dependencies]
+big_s = "1.0.2"
+maplit = "1.0.2"
+meili-snap = { path = "../meili-snap" }
+meilisearch-types = { path = "../meilisearch-types" }

crates/dump/README.md

@@ -10,8 +10,10 @@ dump
 ├── instance-uid.uuid
 ├── keys.jsonl
 ├── metadata.json
-└── tasks
-    ├── update_files
-    │   └── [task_id].jsonl
+├── tasks
+│   ├── update_files
+│   │   └── [task_id].jsonl
+│   └── queue.jsonl
+└── batches
     └── queue.jsonl
-```
+```

crates/dump/src/lib.rs

@@ -1,6 +1,7 @@
 #![allow(clippy::type_complexity)]
 #![allow(clippy::wrong_self_convention)]
 
+use meilisearch_types::batches::BatchId;
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::keys::Key;
 use meilisearch_types::milli::update::IndexDocumentsMethod;
@@ -57,6 +58,9 @@ pub enum Version {
 #[serde(rename_all = "camelCase")]
 pub struct TaskDump {
     pub uid: TaskId,
+    // The batch ID were introduced in v1.12, everything prior to this version will be `None`.
+    #[serde(default)]
+    pub batch_uid: Option<BatchId>,
     #[serde(default)]
     pub index_uid: Option<String>,
     pub status: Status,
@@ -104,6 +108,11 @@ pub enum KindDump {
     DocumentDeletionByFilter {
         filter: serde_json::Value,
     },
+    DocumentEdition {
+        filter: Option<serde_json::Value>,
+        context: Option<serde_json::Map<String, serde_json::Value>>,
+        function: String,
+    },
     Settings {
         settings: Box<meilisearch_types::settings::Settings<Unchecked>>,
         is_deletion: bool,
@@ -132,12 +141,16 @@ pub enum KindDump {
         instance_uid: Option<InstanceUid>,
     },
     SnapshotCreation,
+    UpgradeDatabase {
+        from: (u32, u32, u32),
+    },
 }
 
 impl From<Task> for TaskDump {
     fn from(task: Task) -> Self {
         TaskDump {
             uid: task.uid,
+            batch_uid: task.batch_uid,
             index_uid: task.index_uid().map(|uid| uid.to_string()),
             status: task.status,
             kind: task.kind.into(),
@@ -172,6 +185,9 @@ impl From<KindWithContent> for KindDump {
             KindWithContent::DocumentDeletionByFilter { filter_expr, .. } => {
                 KindDump::DocumentDeletionByFilter { filter: filter_expr }
             }
+            KindWithContent::DocumentEdition { filter_expr, context, function, .. } => {
+                KindDump::DocumentEdition { filter: filter_expr, context, function }
+            }
             KindWithContent::DocumentClear { .. } => KindDump::DocumentClear,
             KindWithContent::SettingsUpdate {
                 new_settings,
@@ -197,6 +213,9 @@ impl From<KindWithContent> for KindDump {
                 KindDump::DumpCreation { keys, instance_uid }
             }
             KindWithContent::SnapshotCreation => KindDump::SnapshotCreation,
+            KindWithContent::UpgradeDatabase { from: version } => {
+                KindDump::UpgradeDatabase { from: version }
+            }
         }
     }
 }
@@ -209,14 +228,16 @@ pub(crate) mod test {
 
     use big_s::S;
     use maplit::{btreemap, btreeset};
+    use meilisearch_types::batches::{Batch, BatchEnqueuedAt, BatchStats};
     use meilisearch_types::facet_values_sort::FacetValuesSort;
-    use meilisearch_types::features::RuntimeTogglableFeatures;
+    use meilisearch_types::features::{Network, Remote, RuntimeTogglableFeatures};
     use meilisearch_types::index_uid_pattern::IndexUidPattern;
     use meilisearch_types::keys::{Action, Key};
     use meilisearch_types::milli;
     use meilisearch_types::milli::update::Setting;
     use meilisearch_types::settings::{Checked, FacetingSettings, Settings};
-    use meilisearch_types::tasks::{Details, Status};
+    use meilisearch_types::task_view::DetailsView;
+    use meilisearch_types::tasks::{Details, Kind, Status};
     use serde_json::{json, Map, Value};
     use time::macros::datetime;
     use uuid::Uuid;
@@ -256,8 +277,8 @@ pub(crate) mod test {
 
     pub fn create_test_settings() -> Settings<Checked> {
         let settings = Settings {
-            displayed_attributes: Setting::Set(vec![S("race"), S("name")]),
-            searchable_attributes: Setting::Set(vec![S("name"), S("race")]),
+            displayed_attributes: Setting::Set(vec![S("race"), S("name")]).into(),
+            searchable_attributes: Setting::Set(vec![S("name"), S("race")]).into(),
             filterable_attributes: Setting::Set(btreeset! { S("race"), S("age") }),
             sortable_attributes: Setting::Set(btreeset! { S("age") }),
             ranking_rules: Setting::NotSet,
@@ -277,16 +298,45 @@ pub(crate) mod test {
             }),
             pagination: Setting::NotSet,
             embedders: Setting::NotSet,
+            search_cutoff_ms: Setting::NotSet,
+            localized_attributes: Setting::NotSet,
+            facet_search: Setting::NotSet,
+            prefix_search: Setting::NotSet,
             _kind: std::marker::PhantomData,
         };
         settings.check()
     }
 
+    pub fn create_test_batches() -> Vec<Batch> {
+        vec![Batch {
+            uid: 0,
+            details: DetailsView {
+                received_documents: Some(12),
+                indexed_documents: Some(Some(10)),
+                ..DetailsView::default()
+            },
+            progress: None,
+            stats: BatchStats {
+                total_nb_tasks: 1,
+                status: maplit::btreemap! { Status::Succeeded => 1 },
+                types: maplit::btreemap! { Kind::DocumentAdditionOrUpdate => 1 },
+                index_uids: maplit::btreemap! { "doggo".to_string() => 1 },
+            },
+            enqueued_at: Some(BatchEnqueuedAt {
+                earliest: datetime!(2022-11-11 0:00 UTC),
+                oldest: datetime!(2022-11-11 0:00 UTC),
+            }),
+            started_at: datetime!(2022-11-20 0:00 UTC),
+            finished_at: Some(datetime!(2022-11-21 0:00 UTC)),
+        }]
+    }
+
     pub fn create_test_tasks() -> Vec<(TaskDump, Option<Vec<Document>>)> {
         vec![
             (
                 TaskDump {
                     uid: 0,
+                    batch_uid: Some(0),
                     index_uid: Some(S("doggo")),
                     status: Status::Succeeded,
                     kind: KindDump::DocumentImport {
@@ -310,6 +360,7 @@ pub(crate) mod test {
             (
                 TaskDump {
                     uid: 1,
+                    batch_uid: None,
                     index_uid: Some(S("doggo")),
                     status: Status::Enqueued,
                     kind: KindDump::DocumentImport {
@@ -336,6 +387,7 @@ pub(crate) mod test {
             (
                 TaskDump {
                     uid: 5,
+                    batch_uid: None,
                     index_uid: Some(S("catto")),
                     status: Status::Enqueued,
                     kind: KindDump::IndexDeletion,
@@ -401,6 +453,15 @@ pub(crate) mod test {
         index.flush().unwrap();
         index.settings(&settings).unwrap();
 
+        // ========== pushing the batch queue
+        let batches = create_test_batches();
+
+        let mut batch_queue = dump.create_batches_queue().unwrap();
+        for batch in &batches {
+            batch_queue.push_batch(batch).unwrap();
+        }
+        batch_queue.flush().unwrap();
+
         // ========== pushing the task queue
         let tasks = create_test_tasks();
 
@@ -429,6 +490,10 @@ pub(crate) mod test {
 
         dump.create_experimental_features(features).unwrap();
 
+        // ========== network
+        let network = create_test_network();
+        dump.create_network(network).unwrap();
+
         // create the dump
         let mut file = tempfile::tempfile().unwrap();
         dump.persist_to(&mut file).unwrap();
@@ -438,7 +503,14 @@ pub(crate) mod test {
     }
 
     fn create_test_features() -> RuntimeTogglableFeatures {
-        RuntimeTogglableFeatures { vector_store: true, ..Default::default() }
+        RuntimeTogglableFeatures::default()
+    }
+
+    fn create_test_network() -> Network {
+        Network {
+            local: Some("myself".to_string()),
+            remotes: maplit::btreemap! {"other".to_string() => Remote { url: "http://test".to_string(), search_api_key: Some("apiKey".to_string()) }},
+        }
     }
 
     #[test]
@@ -489,5 +561,9 @@ pub(crate) mod test {
         // ==== checking the features
         let expected = create_test_features();
         assert_eq!(dump.features().unwrap().unwrap(), expected);
+
+        // ==== checking the network
+        let expected = create_test_network();
+        assert_eq!(&expected, dump.network().unwrap().unwrap());
     }
 }

crates/dump/src/reader/compat/v2_to_v3.rs

@@ -425,7 +425,7 @@ pub(crate) mod test {
         let mut dump = v2::V2Reader::open(dir).unwrap().to_v3();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

crates/dump/src/reader/compat/v3_to_v4.rs

@@ -358,7 +358,7 @@ pub(crate) mod test {
         let mut dump = v3::V3Reader::open(dir).unwrap().to_v4();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

crates/dump/src/reader/compat/v4_to_v5.rs

@@ -394,8 +394,8 @@ pub(crate) mod test {
         let mut dump = v4::V4Reader::open(dir).unwrap().to_v5();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
+        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

crates/dump/src/reader/compat/v5_to_v6.rs

@@ -70,6 +70,7 @@ impl CompatV5ToV6 {
 
                 let task = v6::Task {
                     uid: task_view.uid,
+                    batch_uid: None,
                     index_uid: task_view.index_uid,
                     status: match task_view.status {
                         v5::Status::Enqueued => v6::Status::Enqueued,
@@ -195,6 +196,10 @@ impl CompatV5ToV6 {
     pub fn features(&self) -> Result<Option<v6::RuntimeTogglableFeatures>> {
         Ok(None)
     }
+
+    pub fn network(&self) -> Result<Option<&v6::Network>> {
+        Ok(None)
+    }
 }
 
 pub enum CompatIndexV5ToV6 {
@@ -315,8 +320,8 @@ impl From<v5::ResponseError> for v6::ResponseError {
 impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
     fn from(settings: v5::Settings<T>) -> Self {
         v6::Settings {
-            displayed_attributes: settings.displayed_attributes.into(),
-            searchable_attributes: settings.searchable_attributes.into(),
+            displayed_attributes: v6::Setting::from(settings.displayed_attributes).into(),
+            searchable_attributes: v6::Setting::from(settings.searchable_attributes).into(),
             filterable_attributes: settings.filterable_attributes.into(),
             sortable_attributes: settings.sortable_attributes.into(),
             ranking_rules: {
@@ -379,6 +384,10 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
                 v5::Setting::NotSet => v6::Setting::NotSet,
             },
             embedders: v6::Setting::NotSet,
+            localized_attributes: v6::Setting::NotSet,
+            search_cutoff_ms: v6::Setting::NotSet,
+            facet_search: v6::Setting::NotSet,
+            prefix_search: v6::Setting::NotSet,
             _kind: std::marker::PhantomData,
         }
     }
@@ -441,13 +450,13 @@ pub(crate) mod test {
         let mut dump = v5::V5Reader::open(dir).unwrap().to_v6();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
+        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
         let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"41f91d3a94911b2735ec41b07540df5c");
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"4b03e23e740b27bfb9d2a1faffe512e2");
         assert_eq!(update_files.len(), 22);
         assert!(update_files[0].is_none()); // the dump creation
         assert!(update_files[1].is_some()); // the enqueued document addition

crates/dump/src/reader/mod.rs

@@ -23,6 +23,7 @@ mod v6;
 pub type Document = serde_json::Map<String, serde_json::Value>;
 pub type UpdateFile = dyn Iterator<Item = Result<Document>>;
 
+#[allow(clippy::large_enum_variant)]
 pub enum DumpReader {
     Current(V6Reader),
     Compat(CompatV5ToV6),
@@ -101,6 +102,13 @@ impl DumpReader {
         }
     }
 
+    pub fn batches(&mut self) -> Result<Box<dyn Iterator<Item = Result<v6::Batch>> + '_>> {
+        match self {
+            DumpReader::Current(current) => Ok(current.batches()),
+            DumpReader::Compat(_compat) => Ok(Box::new(std::iter::empty())),
+        }
+    }
+
     pub fn keys(&mut self) -> Result<Box<dyn Iterator<Item = Result<v6::Key>> + '_>> {
         match self {
             DumpReader::Current(current) => Ok(current.keys()),
@@ -114,6 +122,13 @@ impl DumpReader {
             DumpReader::Compat(compat) => compat.features(),
         }
     }
+
+    pub fn network(&self) -> Result<Option<&v6::Network>> {
+        match self {
+            DumpReader::Current(current) => Ok(current.network()),
+            DumpReader::Compat(compat) => compat.network(),
+        }
+    }
 }
 
 impl From<V6Reader> for DumpReader {
@@ -197,19 +212,161 @@ pub(crate) mod test {
     use super::*;
     use crate::reader::v6::RuntimeTogglableFeatures;
 
+    #[test]
+    fn import_dump_v6_with_vectors() {
+        // dump containing two indexes
+        //
+        // "vector", configured with an embedder
+        // contains:
+        // - one document with an overriden vector,
+        // - one document with a natural vector
+        // - one document with a _vectors map containing one additional embedder name and a natural vector
+        // - one document with a _vectors map containing one additional embedder name and an overriden vector
+        //
+        // "novector", no embedder
+        // contains:
+        // - a document without vector
+        // - a document with a random _vectors field
+        let dump = File::open("tests/assets/v6-with-vectors.dump").unwrap();
+        let mut dump = DumpReader::open(dump).unwrap();
+
+        // top level infos
+        insta::assert_snapshot!(dump.date().unwrap(), @"2024-05-16 15:51:34.151044 +00:00:00");
+        insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @"None");
+
+        // batches didn't exists at the time
+        let batches = dump.batches().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        meili_snap::snapshot!(meili_snap::json_string!(batches), @"[]");
+
+        // tasks
+        let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"2b8a72d6bc6ba79980491966437daaf9");
+        assert_eq!(update_files.len(), 10);
+        assert!(update_files[0].is_none()); // the dump creation
+        assert!(update_files[1].is_none());
+        assert!(update_files[2].is_none());
+        assert!(update_files[3].is_none());
+        assert!(update_files[4].is_none());
+        assert!(update_files[5].is_none());
+        assert!(update_files[6].is_none());
+        assert!(update_files[7].is_none());
+        assert!(update_files[8].is_none());
+        assert!(update_files[9].is_none());
+
+        // indexes
+        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        // the index are not ordered in any way by default
+        indexes.sort_by_key(|index| index.metadata().uid.to_string());
+
+        let mut vector_index = indexes.pop().unwrap();
+        let mut novector_index = indexes.pop().unwrap();
+        assert!(indexes.is_empty());
+
+        // vector
+
+        insta::assert_json_snapshot!(vector_index.metadata(), @r###"
+        {
+          "uid": "vector",
+          "primaryKey": "id",
+          "createdAt": "2024-05-16T15:33:17.240962Z",
+          "updatedAt": "2024-05-16T15:40:55.723052Z"
+        }
+        "###);
+
+        insta::assert_json_snapshot!(vector_index.settings().unwrap());
+
+        {
+            let documents: Result<Vec<_>> = vector_index.documents().unwrap().collect();
+            let mut documents = documents.unwrap();
+            assert_eq!(documents.len(), 4);
+
+            documents.sort_by_key(|doc| doc.get("id").unwrap().to_string());
+
+            {
+                let document = documents.pop().unwrap();
+                insta::assert_json_snapshot!(document);
+            }
+
+            {
+                let document = documents.pop().unwrap();
+                insta::assert_json_snapshot!(document);
+            }
+
+            {
+                let document = documents.pop().unwrap();
+                insta::assert_json_snapshot!(document);
+            }
+
+            {
+                let document = documents.pop().unwrap();
+                insta::assert_json_snapshot!(document);
+            }
+        }
+
+        // novector
+
+        insta::assert_json_snapshot!(novector_index.metadata(), @r###"
+        {
+          "uid": "novector",
+          "primaryKey": "id",
+          "createdAt": "2024-05-16T15:33:03.568055Z",
+          "updatedAt": "2024-05-16T15:33:07.530217Z"
+        }
+        "###);
+
+        insta::assert_json_snapshot!(novector_index.settings().unwrap().embedders, @"null");
+
+        {
+            let documents: Result<Vec<_>> = novector_index.documents().unwrap().collect();
+            let mut documents = documents.unwrap();
+            assert_eq!(documents.len(), 2);
+
+            documents.sort_by_key(|doc| doc.get("id").unwrap().to_string());
+
+            {
+                let document = documents.pop().unwrap();
+                insta::assert_json_snapshot!(document, @r###"
+                {
+                  "id": "e1",
+                  "other": "random1",
+                  "_vectors": "toto"
+                }
+                "###);
+            }
+
+            {
+                let document = documents.pop().unwrap();
+                insta::assert_json_snapshot!(document, @r###"
+                {
+                  "id": "e0",
+                  "other": "random0"
+                }
+                "###);
+            }
+        }
+
+        assert_eq!(dump.features().unwrap().unwrap(), RuntimeTogglableFeatures::default());
+        assert_eq!(dump.network().unwrap(), None);
+    }
+
     #[test]
     fn import_dump_v6_experimental() {
         let dump = File::open("tests/assets/v6-with-experimental.dump").unwrap();
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2023-07-06 7:10:27.21958 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2023-07-06 7:10:27.21958 +00:00:00");
         insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @"None");
 
+        // batches didn't exists at the time
+        let batches = dump.batches().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        meili_snap::snapshot!(meili_snap::json_string!(batches), @"[]");
+
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
         let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"d45cd8571703e58ae53c7bd7ce3f5c22");
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"3ddf6169b0a3703c5d770971f036fc5d");
         assert_eq!(update_files.len(), 2);
         assert!(update_files[0].is_none()); // the dump creation
         assert!(update_files[1].is_none()); // the processed document addition
@@ -237,10 +394,28 @@ pub(crate) mod test {
 
         assert_eq!(test.documents().unwrap().count(), 1);
 
-        assert_eq!(
-            dump.features().unwrap().unwrap(),
-            RuntimeTogglableFeatures { vector_store: true, ..Default::default() }
-        );
+        assert_eq!(dump.features().unwrap().unwrap(), RuntimeTogglableFeatures::default());
+    }
+
+    #[test]
+    fn import_dump_v6_network() {
+        let dump = File::open("tests/assets/v6-with-network.dump").unwrap();
+        let dump = DumpReader::open(dump).unwrap();
+
+        // top level infos
+        insta::assert_snapshot!(dump.date().unwrap(), @"2025-01-29 15:45:32.738676 +00:00:00");
+        insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @"None");
+
+        // network
+
+        let network = dump.network().unwrap().unwrap();
+        insta::assert_snapshot!(network.local.as_ref().unwrap(), @"ms-0");
+        insta::assert_snapshot!(network.remotes.get("ms-0").as_ref().unwrap().url, @"http://localhost:7700");
+        insta::assert_snapshot!(network.remotes.get("ms-0").as_ref().unwrap().search_api_key.is_none(), @"true");
+        insta::assert_snapshot!(network.remotes.get("ms-1").as_ref().unwrap().url, @"http://localhost:7701");
+        insta::assert_snapshot!(network.remotes.get("ms-1").as_ref().unwrap().search_api_key.is_none(), @"true");
+        insta::assert_snapshot!(network.remotes.get("ms-2").as_ref().unwrap().url, @"http://ms-5679.example.meilisearch.io");
+        insta::assert_snapshot!(network.remotes.get("ms-2").as_ref().unwrap().search_api_key.as_ref().unwrap(), @"foo");
     }
 
     #[test]
@@ -249,13 +424,17 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
+        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+
+        // batches didn't exists at the time
+        let batches = dump.batches().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        meili_snap::snapshot!(meili_snap::json_string!(batches), @"[]");
 
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
         let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"41f91d3a94911b2735ec41b07540df5c");
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"4b03e23e740b27bfb9d2a1faffe512e2");
         assert_eq!(update_files.len(), 22);
         assert!(update_files[0].is_none()); // the dump creation
         assert!(update_files[1].is_some()); // the enqueued document addition
@@ -329,13 +508,17 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
+        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+
+        // batches didn't exists at the time
+        let batches = dump.batches().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        meili_snap::snapshot!(meili_snap::json_string!(batches), @"[]");
 
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
         let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"c2445ddd1785528b80f2ba534d3bd00c");
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"c1b06a5ca60d5805483c16c5b3ff61ef");
         assert_eq!(update_files.len(), 10);
         assert!(update_files[0].is_some()); // the enqueued document addition
         assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
@@ -406,13 +589,17 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
         assert_eq!(dump.instance_uid().unwrap(), None);
 
+        // batches didn't exists at the time
+        let batches = dump.batches().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        meili_snap::snapshot!(meili_snap::json_string!(batches), @"[]");
+
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
         let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"cd12efd308fe3ed226356a727ab42ed3");
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"0e203b6095f7c68dbdf788321dcc8215");
         assert_eq!(update_files.len(), 10);
         assert!(update_files[0].is_some()); // the enqueued document addition
         assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
@@ -499,13 +686,17 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
         assert_eq!(dump.instance_uid().unwrap(), None);
 
+        // batches didn't exists at the time
+        let batches = dump.batches().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        meili_snap::snapshot!(meili_snap::json_string!(batches), @"[]");
+
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
         let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"bc616290adfe7d09a624cf6065ca9069");
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"d216c7f90f538ffbb2a059531d7ac89a");
         assert_eq!(update_files.len(), 9);
         assert!(update_files[0].is_some()); // the enqueued document addition
         assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
@@ -592,13 +783,17 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2023-01-30 16:26:09.247261 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2023-01-30 16:26:09.247261 +00:00:00");
         assert_eq!(dump.instance_uid().unwrap(), None);
 
+        // batches didn't exists at the time
+        let batches = dump.batches().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        meili_snap::snapshot!(meili_snap::json_string!(batches), @"[]");
+
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
         let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"2db37756d8af1fb7623436b76e8956a6");
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"e27999f1112632222cb84f6cffff7c5f");
         assert_eq!(update_files.len(), 8);
         assert!(update_files[0..].iter().all(|u| u.is_none())); // everything already processed
 
@@ -671,10 +866,14 @@ pub(crate) mod test {
         assert_eq!(dump.date(), None);
         assert_eq!(dump.instance_uid().unwrap(), None);
 
+        // batches didn't exists at the time
+        let batches = dump.batches().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        meili_snap::snapshot!(meili_snap::json_string!(batches), @"[]");
+
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
         let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"8df6eab075a44b3c1af6b726f9fd9a43");
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"0155a664b0cf62aae23db5138b6b03d7");
         assert_eq!(update_files.len(), 9);
         assert!(update_files[..].iter().all(|u| u.is_none())); // no update file in dump v1
 

crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-5.snap

@@ -0,0 +1,56 @@
+---
+source: dump/src/reader/mod.rs
+expression: vector_index.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness"
+  ],
+  "stopWords": [],
+  "nonSeparatorTokens": [],
+  "separatorTokens": [],
+  "dictionary": [],
+  "synonyms": {},
+  "distinctAttribute": null,
+  "proximityPrecision": "byWord",
+  "typoTolerance": {
+    "enabled": true,
+    "minWordSizeForTypos": {
+      "oneTypo": 5,
+      "twoTypos": 9
+    },
+    "disableOnWords": [],
+    "disableOnAttributes": []
+  },
+  "faceting": {
+    "maxValuesPerFacet": 100,
+    "sortFacetValuesBy": {
+      "*": "alpha"
+    }
+  },
+  "pagination": {
+    "maxTotalHits": 1000
+  },
+  "embedders": {
+    "default": {
+      "source": "huggingFace",
+      "model": "BAAI/bge-base-en-v1.5",
+      "revision": "617ca489d9e86b49b8167676d8220688b99db36e",
+      "documentTemplate": "{% for field in fields %} {{ field.name }}: {{ field.value }}\n{% endfor %}"
+    }
+  },
+  "searchCutoffMs": null
+}

crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-6.snap

@@ -0,0 +1,783 @@
+---
+source: dump/src/reader/mod.rs
+expression: document
+---
+{
+  "id": "e3",
+  "desc": "overriden vector + map",
+  "_vectors": {
+    "default": [
+      0.2,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1
+    ],
+    "toto": [
+      0.1
+    ]
+  }
+}

crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-7.snap

@@ -0,0 +1,786 @@
+---
+source: dump/src/reader/mod.rs
+expression: document
+---
+{
+  "id": "e2",
+  "desc": "natural vector + map",
+  "_vectors": {
+    "toto": [],
+    "default": {
+      "embeddings": [
+        [
+          -0.05189208313822746,
+          -0.9273212552070618,
+          0.1443813145160675,
+          0.0932632014155388,
+          0.2665371894836426,
+          0.36266782879829407,
+          0.6402910947799683,
+          0.32014018297195435,
+          0.030915971845388412,
+          -0.9312191605567932,
+          -0.3718109726905823,
+          -0.2700554132461548,
+          -1.1014580726623535,
+          0.9154956936836244,
+          -0.3406888246536255,
+          1.0077725648880005,
+          0.6577560901641846,
+          -0.3955195546150207,
+          -0.4148270785808563,
+          0.1855088472366333,
+          0.5062315464019775,
+          -0.3632686734199524,
+          -0.2277890294790268,
+          0.2560805082321167,
+          -0.3853609561920166,
+          -0.1604762226343155,
+          -0.13947471976280212,
+          -0.20147813856601715,
+          -0.4466346800327301,
+          -0.3761846721172333,
+          0.1443382054567337,
+          0.18205296993255615,
+          0.49359792470932007,
+          -0.22538000345230105,
+          -0.4996317625045776,
+          -0.22734887897968292,
+          -0.6034309267997742,
+          -0.7857939600944519,
+          -0.34923747181892395,
+          -0.3466345965862274,
+          0.21176661550998688,
+          -0.5101462006568909,
+          -0.3403083384037018,
+          0.000315118464641273,
+          0.236465722322464,
+          -0.10246097296476364,
+          -1.3013339042663574,
+          0.3419138789176941,
+          -0.32963496446609497,
+          -0.0901619717478752,
+          -0.5426247119903564,
+          0.22656650841236117,
+          -0.44758284091949463,
+          0.14151698350906372,
+          -0.1089438870549202,
+          0.5500766634941101,
+          -0.670711100101471,
+          -0.6227269768714905,
+          0.3894464075565338,
+          -0.27609574794769287,
+          0.7028202414512634,
+          -0.19697771966457367,
+          0.328511506319046,
+          0.5063360929489136,
+          0.4065195322036743,
+          0.2614171802997589,
+          -0.30274391174316406,
+          1.0393824577331543,
+          -0.7742937207221985,
+          -0.7874112129211426,
+          -0.6749666929244995,
+          0.5190866589546204,
+          0.004123548045754433,
+          -0.28312963247299194,
+          -0.038731709122657776,
+          -1.0142987966537476,
+          -0.09519586712121964,
+          0.8755272626876831,
+          0.4876938760280609,
+          0.7811151742935181,
+          0.85174959897995,
+          0.11826585978269576,
+          0.5373436808586121,
+          0.3649002015590668,
+          0.19064077734947205,
+          -0.00287026260048151,
+          -0.7305403351783752,
+          -0.015206154435873032,
+          -0.7899249196052551,
+          0.19407285749912265,
+          0.08596625179052353,
+          -0.28976231813430786,
+          -0.1525907665491104,
+          0.3798313438892365,
+          0.050306469202041626,
+          -0.5697937607765198,
+          0.4219021201133728,
+          0.276252806186676,
+          0.1559903472661972,
+          0.10030482709407806,
+          -0.4043720066547394,
+          -0.1969818025827408,
+          0.5739826560020447,
+          0.2116064727306366,
+          -1.4620544910430908,
+          -0.7802462577819824,
+          -0.24739810824394223,
+          -0.09791352599859238,
+          -0.4413802027702331,
+          0.21549351513385773,
+          -0.9520436525344848,
+          -0.08762510865926743,
+          0.08154498040676117,
+          -0.6154940724372864,
+          -1.01079523563385,
+          0.885427713394165,
+          0.6967288851737976,
+          0.27186504006385803,
+          -0.43194177746772766,
+          -0.11248451471328735,
+          0.7576630711555481,
+          0.4998855590820313,
+          0.0264343973249197,
+          0.9872855544090272,
+          0.5634694695472717,
+          0.053698331117630005,
+          0.19410227239131927,
+          0.3570743501186371,
+          -0.23670297861099243,
+          -0.9114483594894408,
+          0.07884842902421951,
+          0.7318344116210938,
+          0.44630110263824463,
+          0.08745364099740982,
+          -0.347101628780365,
+          -0.4314247667789459,
+          -0.5060274004936218,
+          0.003706763498485088,
+          0.44320008158683777,
+          -0.00788921769708395,
+          -0.1368623524904251,
+          -0.17391923069953918,
+          0.14473655819892883,
+          0.10927865654230118,
+          0.6974599361419678,
+          0.005052129738032818,
+          -0.016953065991401672,
+          -0.1256176233291626,
+          -0.036742497235536575,
+          0.5591985583305359,
+          -0.37619709968566895,
+          0.22429119050502777,
+          0.5403043031692505,
+          -0.8603790998458862,
+          -0.3456307053565979,
+          0.9292937517166138,
+          0.5074859261512756,
+          0.6310645937919617,
+          -0.3091641068458557,
+          0.46902573108673096,
+          0.7891915440559387,
+          0.4499550759792328,
+          0.2744995653629303,
+          0.2712305784225464,
+          -0.04349074140191078,
+          -0.3638863265514374,
+          0.7839881777763367,
+          0.7352104783058167,
+          -0.19457511603832245,
+          -0.5957832932472229,
+          -0.43704694509506226,
+          -1.084769368171692,
+          0.4904985725879669,
+          0.5385226011276245,
+          0.1891629993915558,
+          0.12338479608297348,
+          0.8315675258636475,
+          -0.07830192148685455,
+          1.0916285514831543,
+          -0.28066861629486084,
+          -1.3585069179534912,
+          0.5203898549079895,
+          0.08678033947944641,
+          -0.2566044330596924,
+          0.09484415501356123,
+          -0.0180208683013916,
+          1.0264745950698853,
+          -0.023572135716676712,
+          0.5864979028701782,
+          0.7625196576118469,
+          -0.2543414533138275,
+          -0.8877770900726318,
+          0.7611982822418213,
+          -0.06220436468720436,
+          0.937336564064026,
+          0.2704363465309143,
+          -0.37733694911003113,
+          0.5076137781143188,
+          -0.30641937255859375,
+          0.6252772808074951,
+          -0.0823579877614975,
+          -0.03736555948853493,
+          0.4131673276424408,
+          -0.6514252424240112,
+          0.12918265163898468,
+          -0.4483584463596344,
+          0.6750786304473877,
+          -0.37008383870124817,
+          -0.02324833907186985,
+          0.38027650117874146,
+          -0.26374951004981995,
+          0.4346931278705597,
+          0.42882832884788513,
+          -0.48798441886901855,
+          1.1882442235946655,
+          0.5132288336753845,
+          0.5284568667411804,
+          -0.03538886830210686,
+          0.29620853066444397,
+          -1.0683696269989014,
+          0.25936177372932434,
+          0.10404160618782043,
+          -0.25796034932136536,
+          0.027896970510482788,
+          -0.09225251525640488,
+          1.4811025857925415,
+          0.641173779964447,
+          -0.13838383555412292,
+          -0.3437179923057556,
+          0.5667019486427307,
+          -0.5400741696357727,
+          0.31090837717056274,
+          0.6470608115196228,
+          -0.3747067153453827,
+          -0.7364534735679626,
+          -0.07431528717279434,
+          0.5173454880714417,
+          -0.6578747034072876,
+          0.7107478976249695,
+          -0.7918999791145325,
+          -0.0648345872759819,
+          0.609937846660614,
+          -0.7329513430595398,
+          0.9741371870040894,
+          0.17912346124649048,
+          -0.02658769302070141,
+          0.5162150859832764,
+          -0.3978803157806397,
+          -0.7833885550498962,
+          -0.6497276425361633,
+          -0.3898126780986786,
+          -0.0952848568558693,
+          0.2663288116455078,
+          -0.1604052186012268,
+          0.373076468706131,
+          -0.8357769250869751,
+          -0.05217683315277099,
+          -0.2680160701274872,
+          0.8389158248901367,
+          0.6833611130714417,
+          -0.6712407469749451,
+          0.7406917214393616,
+          -0.44522786140441895,
+          -0.34645363688468933,
+          -0.27384576201438904,
+          -0.9878405928611756,
+          -0.8166060447692871,
+          0.06268279999494553,
+          0.38567957282066345,
+          -0.3274703919887543,
+          0.5296315550804138,
+          -0.11810623109340668,
+          0.23029841482639313,
+          0.08616159111261368,
+          -0.2195747196674347,
+          0.09430307894945145,
+          0.4057176411151886,
+          0.4892159104347229,
+          -0.1636916548013687,
+          -0.6071445345878601,
+          0.41256585717201233,
+          0.622254490852356,
+          -0.41223976016044617,
+          -0.6686707139015198,
+          -0.7474371790885925,
+          -0.8509522080421448,
+          -0.16754287481307983,
+          -0.9078601002693176,
+          -0.29653599858283997,
+          -0.5020652413368225,
+          0.4692700505256653,
+          0.01281109917908907,
+          -0.16071580350399017,
+          0.03388889133930206,
+          -0.020511148497462273,
+          0.5027827024459839,
+          -0.20729811489582065,
+          0.48107290267944336,
+          0.33669769763946533,
+          -0.5275911688804626,
+          0.48271527886390686,
+          0.2738940715789795,
+          -0.033152539283037186,
+          -0.13629786670207977,
+          -0.05965912342071533,
+          -0.26200807094573975,
+          0.04002794995903969,
+          -0.34095603227615356,
+          -3.986898899078369,
+          -0.46819332242012024,
+          -0.422744482755661,
+          -0.169097900390625,
+          0.6008929014205933,
+          0.058016058057546616,
+          -0.11401277780532836,
+          -0.3077819049358368,
+          -0.09595538675785063,
+          0.6723822355270386,
+          0.19367831945419312,
+          0.28304359316825867,
+          0.1609862744808197,
+          0.7567598819732666,
+          0.6889985799789429,
+          0.06907720118761063,
+          -0.04188092052936554,
+          -0.7434936165809631,
+          0.13321782648563385,
+          0.8456063270568848,
+          -0.10364038497209548,
+          -0.45084846019744873,
+          -0.4758241474628449,
+          0.43882066011428833,
+          -0.6432598829269409,
+          0.7217311859130859,
+          -0.24189773201942444,
+          0.12737572193145752,
+          -1.1008601188659668,
+          -0.3305315673351288,
+          0.14614742994308472,
+          -0.7819333076477051,
+          0.5287120342254639,
+          -0.055538054555654526,
+          0.1877404749393463,
+          -0.6907662153244019,
+          0.5616975426673889,
+          -0.4611121714115143,
+          -0.26109233498573303,
+          -0.12898315489292145,
+          -0.3724522292613983,
+          -0.7191406488418579,
+          -0.4425233602523804,
+          -0.644108235836029,
+          0.8424481153488159,
+          0.17532426118850708,
+          -0.5121750235557556,
+          -0.6467239260673523,
+          -0.0008507720194756985,
+          0.7866212129592896,
+          -0.02644744887948036,
+          -0.005045140627771616,
+          0.015782782807946205,
+          0.16334445774555206,
+          -0.1913367658853531,
+          -0.13697923719882965,
+          -0.6684983372688293,
+          0.18346354365348816,
+          -0.341105580329895,
+          0.5427411198616028,
+          0.3779832422733307,
+          -0.6778115034103394,
+          -0.2931850254535675,
+          -0.8805161714553833,
+          -0.4212774932384491,
+          -0.5368952751159668,
+          -1.3937891721725464,
+          -1.225494146347046,
+          0.4276703894138336,
+          1.1205668449401855,
+          -0.6005299687385559,
+          0.15732505917549133,
+          -0.3914784789085388,
+          -1.357046604156494,
+          -0.4707142114639282,
+          -0.1497287154197693,
+          -0.25035548210144043,
+          -0.34328439831733704,
+          0.39083412289619446,
+          0.1623048633337021,
+          -0.9275814294815063,
+          -0.6430015563964844,
+          0.2973862886428833,
+          0.5580436587333679,
+          -0.6232585310935974,
+          -0.6611042022705078,
+          0.4015969038009643,
+          -1.0232892036437988,
+          -0.2585645020008087,
+          -0.5431421399116516,
+          0.5021264553070068,
+          -0.48601630330085754,
+          -0.010242084041237833,
+          0.5862035155296326,
+          0.7316920161247253,
+          0.4036808013916016,
+          0.4269520044326782,
+          -0.705938458442688,
+          0.7747307419776917,
+          0.10164368897676468,
+          0.7887958884239197,
+          -0.9612497091293336,
+          0.12755516171455383,
+          0.06812842190265656,
+          -0.022603651508688927,
+          0.14722754061222076,
+          -0.5588505268096924,
+          -0.20689940452575684,
+          0.3557641804218292,
+          -0.6812759637832642,
+          0.2860803008079529,
+          -0.38954633474349976,
+          0.1759403496980667,
+          -0.5678874850273132,
+          -0.1692986786365509,
+          -0.14578519761562347,
+          0.5711379051208496,
+          1.0208125114440918,
+          0.7759483456611633,
+          -0.372348427772522,
+          -0.5460885763168335,
+          0.7190321683883667,
+          -0.6914990544319153,
+          0.13365162909030914,
+          -0.4854792356491089,
+          0.4054908752441406,
+          0.4502798914909363,
+          -0.3041122555732727,
+          -0.06726965308189392,
+          -0.05570871382951737,
+          -0.0455719493329525,
+          0.4785125255584717,
+          0.8867972493171692,
+          0.4107886850833893,
+          0.6121342182159424,
+          -0.20477132499217987,
+          -0.5598517656326294,
+          -0.6443566679954529,
+          -0.5905212759971619,
+          -0.5571200251579285,
+          0.17573799192905426,
+          -0.28621870279312134,
+          0.1685224026441574,
+          0.09719007462263109,
+          -0.04223639518022537,
+          -0.28623101115226746,
+          -0.1449810117483139,
+          -0.3789580464363098,
+          -0.5227636098861694,
+          -0.049728814512491226,
+          0.7849089503288269,
+          0.16792525351047516,
+          0.9849340915679932,
+          -0.6559549570083618,
+          0.35723909735679626,
+          -0.6822739243507385,
+          1.2873116731643677,
+          0.19993330538272855,
+          0.03512010723352432,
+          -0.6972134113311768,
+          0.18453484773635864,
+          -0.2437680810689926,
+          0.2156416028738022,
+          0.5230382680892944,
+          0.22020135819911957,
+          0.8314080238342285,
+          0.15627102553844452,
+          -0.7330264449119568,
+          0.3888184726238251,
+          -0.22034703195095065,
+          0.5457669496536255,
+          -0.48084837198257446,
+          -0.45576658844947815,
+          -0.09287727624177931,
+          -0.06968110054731369,
+          0.35125672817230225,
+          -0.4278119504451752,
+          0.2038476765155792,
+          0.11392722278833388,
+          0.9433983564376832,
+          -0.4097744226455689,
+          0.035297419875860214,
+          -0.4274404048919678,
+          -0.25100165605545044,
+          1.0943366289138794,
+          -0.07634022831916809,
+          -0.2925529479980469,
+          -0.7512530088424683,
+          0.2649727463722229,
+          -0.4078235328197479,
+          -0.3372223973274231,
+          0.05190162733197212,
+          0.005654910113662481,
+          -0.0001571219472680241,
+          -0.35445958375930786,
+          -0.7837416529655457,
+          0.1500556766986847,
+          0.4383024573326111,
+          0.6099548935890198,
+          0.05951934307813645,
+          -0.21325334906578064,
+          0.0199207104742527,
+          -0.22704418003559113,
+          -0.6481077671051025,
+          0.37442275881767273,
+          -1.015955924987793,
+          0.38637226819992065,
+          -0.06489371508359909,
+          -0.494120329618454,
+          0.3469836115837097,
+          0.15402406454086304,
+          -0.7660972476005554,
+          -0.7053225040435791,
+          -0.25964751839637756,
+          0.014004424214363098,
+          -0.2860170006752014,
+          -0.17565494775772095,
+          -0.45117494463920593,
+          -0.0031954257283359766,
+          0.09676837921142578,
+          -0.514464259147644,
+          0.41698193550109863,
+          -0.21642713248729703,
+          -0.5398141145706177,
+          -0.3647628426551819,
+          0.37005379796028137,
+          0.239425927400589,
+          -0.08833975344896317,
+          0.934946596622467,
+          -0.48340797424316406,
+          0.6241437792778015,
+          -0.7253676652908325,
+          -0.04303571209311485,
+          1.1125205755233765,
+          -0.15692919492721558,
+          -0.2914651036262512,
+          -0.5117168426513672,
+          0.21365483105182648,
+          0.4924402534961701,
+          0.5269662141799927,
+          0.0352792888879776,
+          -0.149167999625206,
+          -0.6019760370254517,
+          0.08245442807674408,
+          0.4900692105293274,
+          0.518824577331543,
+          -0.00005570516441366635,
+          -0.553304135799408,
+          0.22217543423175812,
+          0.5047767758369446,
+          0.135724738240242,
+          1.1511540412902832,
+          -0.3541218340396881,
+          -0.9712511897087096,
+          0.8353699445724487,
+          -0.39227569103240967,
+          -0.9117669463157654,
+          -0.26349931955337524,
+          0.05597023293375969,
+          0.20695461332798004,
+          0.3178807199001312,
+          1.0663238763809204,
+          0.5062212347984314,
+          0.7288597822189331,
+          0.09899299591779707,
+          0.553720235824585,
+          0.675009548664093,
+          -0.20067055523395536,
+          0.3138423264026642,
+          -0.6886593103408813,
+          -0.2910398542881012,
+          -1.3186300992965698,
+          -0.4684459865093231,
+          -0.095743365585804,
+          -0.1257995069026947,
+          -0.4858281314373016,
+          -0.4935407340526581,
+          -0.3266896903514862,
+          -0.3928797245025635,
+          -0.40803104639053345,
+          -0.9975396394729614,
+          0.4229583740234375,
+          0.37309643626213074,
+          0.4431034922599793,
+          0.30364808440208435,
+          -0.3765178918838501,
+          0.5616499185562134,
+          0.16904796659946442,
+          -0.7343707084655762,
+          0.2560209631919861,
+          0.6166825294494629,
+          0.3200829327106476,
+          -0.4483652710914612,
+          0.16224201023578644,
+          -0.31495288014411926,
+          -0.42713335156440735,
+          0.7270734906196594,
+          0.7049484848976135,
+          -0.0571461021900177,
+          0.04477125033736229,
+          -0.6647796034812927,
+          1.183672308921814,
+          0.36199676990509033,
+          0.046881116926670074,
+          0.4515796303749085,
+          0.9278061985969543,
+          0.31471705436706543,
+          -0.7073333859443665,
+          -0.3443860113620758,
+          0.5440067052841187,
+          -0.15020819008350372,
+          -0.541202962398529,
+          0.5203295946121216,
+          1.2192286252975464,
+          -0.9983593225479126,
+          -0.18758884072303772,
+          0.2758221924304962,
+          -0.6511523723602295,
+          -0.1584404855966568,
+          -0.236241415143013,
+          0.2692437767982483,
+          -0.4941152036190033,
+          0.4987454116344452,
+          -0.3331359028816223,
+          0.3163745701313019,
+          0.745529294013977,
+          -0.2905873656272888,
+          0.13602906465530396,
+          0.4679684340953827,
+          1.0555986166000366,
+          1.075700044631958,
+          0.5368486046791077,
+          -0.5118206739425659,
+          0.8668332099914551,
+          -0.5726966857910156,
+          -0.7811751961708069,
+          0.1938626915216446,
+          -0.1929349899291992,
+          0.1757766306400299,
+          0.6384295225143433,
+          0.26462844014167786,
+          0.9542630314826964,
+          0.19313029944896695,
+          1.264248013496399,
+          -0.6304428577423096,
+          0.0487106591463089,
+          -0.16211535036563873,
+          -0.7894763350486755,
+          0.3582514822483063,
+          -0.04153040423989296,
+          0.635784387588501,
+          0.6554391980171204,
+          -0.47010496258735657,
+          -0.8302040696144104,
+          -0.1350124627351761,
+          0.2568812072277069,
+          0.13614831864833832,
+          -0.2563649117946625,
+          -1.0434694290161133,
+          0.3232482671737671,
+          0.47882452607154846,
+          0.4298652410507202,
+          1.0563770532608032,
+          -0.28917592763900757,
+          -0.8533256649971008,
+          0.10648339986801147,
+          0.6376127004623413,
+          -0.20832888782024384,
+          0.2370245456695557,
+          0.0018312990432605147,
+          -0.2034837007522583,
+          0.01051164511591196,
+          -1.105310082435608,
+          0.29724350571632385,
+          0.15604574978351593,
+          0.1973688006401062,
+          0.44394731521606445,
+          0.3974513411521912,
+          -0.13625948131084442,
+          0.9571986198425292,
+          0.2257384955883026,
+          0.2323588728904724,
+          -0.5583669543266296,
+          -0.7854922413825989,
+          0.1647188365459442,
+          -1.6098142862319946,
+          0.318587988615036,
+          -0.13399995863437653,
+          -0.2172701060771942,
+          -0.767514705657959,
+          -0.5813586711883545,
+          -0.3195130527019501,
+          -0.04894036799669266,
+          0.2929930090904236,
+          -0.8213384747505188,
+          0.07181350141763687,
+          0.7469993829727173,
+          0.6407455801963806,
+          0.16365697979927063,
+          0.7870153188705444,
+          0.6524736881256104,
+          0.6399973630905151,
+          -0.04992736503481865,
+          -0.03959266096353531,
+          -0.2512352466583252,
+          0.8448855876922607,
+          -0.1422702670097351,
+          0.1216789186000824,
+          -1.2647287845611572,
+          0.5931149125099182,
+          0.7186052203178406,
+          -0.06118432432413101,
+          -1.1942816972732544,
+          -0.17677085101604462,
+          0.31543800234794617,
+          -0.32252824306488037,
+          0.8255583047866821,
+          -0.14529970288276672,
+          -0.2695446312427521,
+          -0.33378756046295166,
+          -0.1653425395488739,
+          0.1454019844532013,
+          -0.3920115828514099,
+          0.912214994430542,
+          -0.7279734015464783,
+          0.7374742031097412,
+          0.933980405330658,
+          0.13429680466651917,
+          -0.514870285987854,
+          0.3989711999893189,
+          -0.11613689363002776,
+          0.4022413492202759,
+          -0.9990655779838562,
+          -0.33749932050704956,
+          -0.4334589838981629,
+          -1.376373291015625,
+          -0.2993924915790558,
+          -0.09454808384180068,
+          -0.01314175222069025,
+          -0.001090060803107917,
+          0.2137461006641388,
+          0.2938512861728668,
+          0.17508235573768616,
+          0.8260607123374939,
+          -0.7218498587608337,
+          0.2414487451314926,
+          -0.47296759486198425,
+          -0.3002610504627228,
+          -1.238540768623352,
+          0.08663805574178696,
+          0.6805586218833923,
+          0.5909030437469482,
+          -0.42807504534721375,
+          -0.22887496650218964,
+          0.47537800669670105,
+          -1.0474627017974854,
+          0.6338009238243103,
+          0.06548397243022919,
+          0.4971011281013489,
+          1.3484878540039063
+        ]
+      ],
+      "regenerate": true
+    }
+  }
+}

crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-8.snap

@@ -0,0 +1,785 @@
+---
+source: dump/src/reader/mod.rs
+expression: document
+---
+{
+  "id": "e1",
+  "desc": "natural vector",
+  "_vectors": {
+    "default": {
+      "embeddings": [
+        [
+          -0.2979458272457123,
+          -0.5288640856742859,
+          -0.019957859069108963,
+          -0.18495318293571472,
+          0.7429973483085632,
+          0.5238497257232666,
+          0.432366281747818,
+          0.32744166254997253,
+          0.0020762972999364138,
+          -0.9507834911346436,
+          -0.35097137093544006,
+          0.08469701558351517,
+          -1.4176613092422483,
+          0.4647577106952667,
+          -0.69340580701828,
+          1.0372896194458008,
+          0.3716741800308227,
+          0.06031008064746857,
+          -0.6152024269104004,
+          0.007914665155112743,
+          0.7954924702644348,
+          -0.20773003995418549,
+          0.09376765787601472,
+          0.04508133605122566,
+          -0.2084471583366394,
+          -0.1518009901046753,
+          0.018195509910583496,
+          -0.07044368237257004,
+          -0.18119366466999057,
+          -0.4480230510234833,
+          0.3822529911994934,
+          0.1911812424659729,
+          0.4674372375011444,
+          0.06963984668254852,
+          -0.09341949224472046,
+          0.005675444379448891,
+          -0.6774799227714539,
+          -0.7066726684570313,
+          -0.39256376028060913,
+          0.04005039855837822,
+          0.2084812968969345,
+          -0.7872875928878784,
+          -0.8205880522727966,
+          0.2919981777667999,
+          -0.06004738807678223,
+          -0.4907574355602264,
+          -1.5937862396240234,
+          0.24249385297298431,
+          -0.14709846675395966,
+          -0.11860740929841997,
+          -0.8299489617347717,
+          0.472964346408844,
+          -0.497518390417099,
+          -0.22205302119255063,
+          -0.4196169078350067,
+          0.32697558403015137,
+          -0.360930860042572,
+          -0.9789686799049376,
+          0.1887447088956833,
+          -0.403737336397171,
+          0.18524253368377688,
+          0.3768732249736786,
+          0.3666233420372009,
+          0.3511938452720642,
+          0.6985810995101929,
+          0.41721710562705994,
+          0.09754953533411026,
+          0.6204307079315186,
+          -1.0762996673583984,
+          -0.06263761967420578,
+          -0.7376511693000793,
+          0.6849768161773682,
+          -0.1745152473449707,
+          -0.40449759364128113,
+          0.20757411420345304,
+          -0.8424443006515503,
+          0.330015629529953,
+          0.3489064872264862,
+          1.0954371690750122,
+          0.8487558960914612,
+          1.1076823472976685,
+          0.61430823802948,
+          0.4155903458595276,
+          0.4111340939998626,
+          0.05753209814429283,
+          -0.06429877132177353,
+          -0.765606164932251,
+          -0.41703930497169495,
+          -0.508820652961731,
+          0.19859947264194489,
+          -0.16607828438282013,
+          -0.28112146258354187,
+          0.11032675206661224,
+          0.38809511065483093,
+          -0.36498191952705383,
+          -0.48671194911003113,
+          0.6755134463310242,
+          0.03958442434668541,
+          0.4478721618652344,
+          -0.10335399955511092,
+          -0.9546685814857484,
+          -0.6087718605995178,
+          0.17498846352100372,
+          0.08320838958024979,
+          -1.4478336572647097,
+          -0.605027437210083,
+          -0.5867993235588074,
+          -0.14711688458919525,
+          -0.5447602272033691,
+          -0.026259321719408035,
+          -0.6997418403625488,
+          -0.07349082082509995,
+          0.10638900846242905,
+          -0.7133527398109436,
+          -0.9396815299987792,
+          1.087092399597168,
+          1.1885089874267578,
+          0.4011896848678589,
+          -0.4089202582836151,
+          -0.10938972979784012,
+          0.6726722121238708,
+          0.24576938152313232,
+          -0.24247920513153076,
+          1.1499971151351929,
+          0.47813335061073303,
+          -0.05331678315997124,
+          0.32338133454322815,
+          0.4870913326740265,
+          -0.23144258558750153,
+          -1.2023426294326782,
+          0.2349330335855484,
+          1.080536961555481,
+          0.29334118962287903,
+          0.391574501991272,
+          -0.15818795561790466,
+          -0.2948290705680847,
+          -0.024689948186278343,
+          0.06602869182825089,
+          0.5937030911445618,
+          -0.047901444137096405,
+          -0.512734591960907,
+          -0.35780075192451477,
+          0.28751692175865173,
+          0.4298716187477112,
+          0.9242428541183472,
+          -0.17208744585514069,
+          0.11515070497989656,
+          -0.0335976779460907,
+          -0.3422986567020416,
+          0.5344581604003906,
+          0.19895796477794647,
+          0.33001241087913513,
+          0.6390730142593384,
+          -0.6074934005737305,
+          -0.2553696632385254,
+          0.9644920229911804,
+          0.2699219584465027,
+          0.6403993368148804,
+          -0.6380003690719604,
+          -0.027310986071825027,
+          0.638815701007843,
+          0.27719101309776306,
+          -0.13553589582443237,
+          0.750195324420929,
+          0.1224869191646576,
+          -0.20613941550254825,
+          0.8444448709487915,
+          0.16200250387191772,
+          -0.24750925600528717,
+          -0.739950954914093,
+          -0.28443849086761475,
+          -1.176282525062561,
+          0.516107976436615,
+          0.3774825632572174,
+          0.10906043648719788,
+          0.07962015271186829,
+          0.7384604215621948,
+          -0.051241904497146606,
+          1.1730090379714966,
+          -0.4828610122203827,
+          -1.404372215270996,
+          0.8811132311820984,
+          -0.3839482367038727,
+          0.022516896948218346,
+          -0.0491158664226532,
+          -0.43027013540267944,
+          1.2049334049224854,
+          -0.27309560775756836,
+          0.6883630752563477,
+          0.8264574408531189,
+          -0.5020735263824463,
+          -0.4874092042446137,
+          0.6007202863693237,
+          -0.4965405762195587,
+          1.1302915811538696,
+          0.032572727650403976,
+          -0.3731859028339386,
+          0.658271849155426,
+          -0.9023059010505676,
+          0.7400162220001221,
+          0.014550759457051754,
+          -0.19699542224407196,
+          0.2319706380367279,
+          -0.789058268070221,
+          -0.14905710518360138,
+          -0.5826214551925659,
+          0.207652747631073,
+          -0.4507439732551574,
+          -0.3163885474205017,
+          0.3604124188423157,
+          -0.45119962096214294,
+          0.3428427278995514,
+          0.3005594313144684,
+          -0.36026081442832947,
+          1.1014249324798584,
+          0.40884315967559814,
+          0.34991952776908875,
+          -0.1806638240814209,
+          0.27440476417541504,
+          -0.7118373513221741,
+          0.4645499587059021,
+          0.214790478348732,
+          -0.2343102991580963,
+          0.10500429570674896,
+          -0.28034430742263794,
+          1.2267805337905884,
+          1.0561333894729614,
+          -0.497364342212677,
+          -0.6143305897712708,
+          0.24963727593421936,
+          -0.33136463165283203,
+          -0.01473914459347725,
+          0.495918869972229,
+          -0.6985538005828857,
+          -1.0033197402954102,
+          0.35937801003456116,
+          0.6325868368148804,
+          -0.6808838844299316,
+          1.0354058742523191,
+          -0.7214401960372925,
+          -0.33318862318992615,
+          0.874398410320282,
+          -0.6594992280006409,
+          0.6830640435218811,
+          -0.18534131348133087,
+          0.024834271520376205,
+          0.19901277124881744,
+          -0.5992477536201477,
+          -1.2126628160476685,
+          -0.9245557188987732,
+          -0.3898217976093292,
+          -0.1286519467830658,
+          0.4217943847179413,
+          -0.1143646091222763,
+          0.5630772709846497,
+          -0.5240639448165894,
+          0.21152715384960177,
+          -0.3792001008987427,
+          0.8266305327415466,
+          1.170984387397766,
+          -0.8072142004966736,
+          0.11382893472909927,
+          -0.17953898012638092,
+          -0.1789460331201553,
+          -0.15078622102737427,
+          -1.2082908153533936,
+          -0.7812382578849792,
+          -0.10903695970773696,
+          0.7303897142410278,
+          -0.39054441452026367,
+          0.19511254131793976,
+          -0.09121843427419662,
+          0.22400228679180145,
+          0.30143046379089355,
+          0.1141919493675232,
+          0.48112115263938904,
+          0.7307931780815125,
+          0.09701362252235413,
+          -0.2795647978782654,
+          -0.3997688889503479,
+          0.5540812611579895,
+          0.564578115940094,
+          -0.40065160393714905,
+          -0.3629159033298493,
+          -0.3789091110229492,
+          -0.7298538088798523,
+          -0.6996853351593018,
+          -0.4477842152118683,
+          -0.289089560508728,
+          -0.6430277824401855,
+          0.2344944179058075,
+          0.3742927014827728,
+          -0.5079357028007507,
+          0.28841453790664673,
+          0.06515737622976303,
+          0.707315981388092,
+          0.09498685598373412,
+          0.8365515470504761,
+          0.10002726316452026,
+          -0.7695478200912476,
+          0.6264724135398865,
+          0.7562043070793152,
+          -0.23112858831882477,
+          -0.2871039807796478,
+          -0.25010058283805847,
+          0.2783474028110504,
+          -0.03224996477365494,
+          -0.9119359850883484,
+          -3.6940200328826904,
+          -0.5099936127662659,
+          -0.1604711413383484,
+          0.17453284561634064,
+          0.41759559512138367,
+          0.1419190913438797,
+          -0.11362407356500626,
+          -0.33312007784843445,
+          0.11511333286762238,
+          0.4667884409427643,
+          -0.0031647447030991316,
+          0.15879854559898376,
+          0.3042248487472534,
+          0.5404849052429199,
+          0.8515422344207764,
+          0.06286454200744629,
+          0.43790125846862793,
+          -0.8682025074958801,
+          -0.06363756954669952,
+          0.5547921657562256,
+          -0.01483887154608965,
+          -0.07361344993114471,
+          -0.929947018623352,
+          0.3502565622329712,
+          -0.5080993175506592,
+          1.0380364656448364,
+          -0.2017953395843506,
+          0.21319580078125,
+          -1.0763001441955566,
+          -0.556368887424469,
+          0.1949922740459442,
+          -0.6445739269256592,
+          0.6791343688964844,
+          0.21188358962535855,
+          0.3736183941364288,
+          -0.21800459921360016,
+          0.7597446441650391,
+          -0.3732394874095917,
+          -0.4710160195827484,
+          0.025146087631583217,
+          0.05341297015547752,
+          -0.9522109627723694,
+          -0.6000866889953613,
+          -0.08469046652317047,
+          0.5966026186943054,
+          0.3444081246852875,
+          -0.461188405752182,
+          -0.5279349088668823,
+          0.10296865552663804,
+          0.5175143480300903,
+          -0.20671147108078003,
+          0.13392412662506104,
+          0.4812754988670349,
+          0.2993808686733246,
+          -0.3005635440349579,
+          0.5141698122024536,
+          -0.6239235401153564,
+          0.2877119481563568,
+          -0.4452739953994751,
+          0.5621107816696167,
+          0.5047508478164673,
+          -0.4226335883140564,
+          -0.18578553199768064,
+          -1.1967322826385498,
+          0.28178197145462036,
+          -0.8692031502723694,
+          -1.1812998056411743,
+          -1.4526212215423584,
+          0.4645712077617645,
+          0.9327932000160216,
+          -0.6560136675834656,
+          0.461549699306488,
+          -0.5621527433395386,
+          -1.328449010848999,
+          -0.08676894754171371,
+          0.00021918353741057217,
+          -0.18864136934280396,
+          0.1259666532278061,
+          0.18240638077259064,
+          -0.14919660985469818,
+          -0.8965857625007629,
+          -0.7539900541305542,
+          0.013973715715110302,
+          0.504276692867279,
+          -0.704748272895813,
+          -0.6428424119949341,
+          0.6303996443748474,
+          -0.5404738187789917,
+          -0.31176653504371643,
+          -0.21262824535369873,
+          0.18736739456653595,
+          -0.7998970746994019,
+          0.039946746081113815,
+          0.7390344738960266,
+          0.4283199906349182,
+          0.3795057237148285,
+          0.07204607129096985,
+          -0.9230587482452391,
+          0.9440426230430604,
+          0.26272690296173096,
+          0.5598306655883789,
+          -1.0520871877670288,
+          -0.2677186131477356,
+          -0.1888762265443802,
+          0.30426350235939026,
+          0.4746131896972656,
+          -0.5746733546257019,
+          -0.4197768568992615,
+          0.8565112948417664,
+          -0.6767723560333252,
+          0.23448683321475983,
+          -0.2010004222393036,
+          0.4112907350063324,
+          -0.6497949957847595,
+          -0.418667733669281,
+          -0.4950824975967407,
+          0.44438859820365906,
+          1.026281714439392,
+          0.482397586107254,
+          -0.26220494508743286,
+          -0.3640787005424499,
+          0.5907743573188782,
+          -0.8771642446517944,
+          0.09708411991596222,
+          -0.3671700060367584,
+          0.4331349730491638,
+          0.619417667388916,
+          -0.2684665620326996,
+          -0.5123821496963501,
+          -0.1502324342727661,
+          -0.012190685607492924,
+          0.3580845892429352,
+          0.8617186546325684,
+          0.3493645489215851,
+          1.0270192623138428,
+          0.18297909200191495,
+          -0.5881339311599731,
+          -0.1733516901731491,
+          -0.5040576457977295,
+          -0.340370237827301,
+          -0.26767754554748535,
+          -0.28570041060447693,
+          -0.032928116619586945,
+          0.6029254794120789,
+          0.17397655546665192,
+          0.09346921741962431,
+          0.27815181016921997,
+          -0.46699589490890503,
+          -0.8148876428604126,
+          -0.3964351713657379,
+          0.3812595009803772,
+          0.13547226786613464,
+          0.7126688361167908,
+          -0.3473474085330963,
+          -0.06573959439992905,
+          -0.6483767032623291,
+          1.4808889627456665,
+          0.30924928188323975,
+          -0.5085946917533875,
+          -0.8613000512123108,
+          0.3048902451992035,
+          -0.4241599142551422,
+          0.15909206867218018,
+          0.5764641761779785,
+          -0.07879110425710678,
+          1.015336513519287,
+          0.07599356025457382,
+          -0.7025855779647827,
+          0.30047643184661865,
+          -0.35094937682151794,
+          0.2522146999835968,
+          -0.2338722199201584,
+          -0.8326804637908936,
+          -0.13695412874221802,
+          -0.03452421352267265,
+          0.47974953055381775,
+          -0.18385636806488037,
+          0.32438594102859497,
+          0.1797013282775879,
+          0.787494957447052,
+          -0.12579888105392456,
+          -0.07507286965847015,
+          -0.4389670491218567,
+          0.2720070779323578,
+          0.8138866424560547,
+          0.01974171027541161,
+          -0.3057698905467987,
+          -0.6709924936294556,
+          0.0885881632566452,
+          -0.2862754464149475,
+          0.03475658595561981,
+          -0.1285519152879715,
+          0.3838353455066681,
+          -0.2944154739379883,
+          -0.4204859137535095,
+          -0.4416137933731079,
+          0.13426260650157928,
+          0.36733248829841614,
+          0.573428750038147,
+          -0.14928072690963745,
+          -0.026076916605234143,
+          0.33286052942276,
+          -0.5340145826339722,
+          -0.17279052734375,
+          -0.01154550164937973,
+          -0.6620771884918213,
+          0.18390542268753052,
+          -0.08265615254640579,
+          -0.2489682286977768,
+          0.2429984211921692,
+          -0.044153645634651184,
+          -0.986578404903412,
+          -0.33574509620666504,
+          -0.5387663841247559,
+          0.19767941534519196,
+          0.12540718913078308,
+          -0.3403128981590271,
+          -0.4154576361179352,
+          0.17275673151016235,
+          0.09407442808151244,
+          -0.5414086580276489,
+          0.4393929839134216,
+          0.1725579798221588,
+          -0.4998118281364441,
+          -0.6926208138465881,
+          0.16552448272705078,
+          0.6659538149833679,
+          -0.10949844866991044,
+          0.986426830291748,
+          0.01748848147690296,
+          0.4003709554672241,
+          -0.5430638194084167,
+          0.35347291827201843,
+          0.6887399554252625,
+          0.08274628221988678,
+          0.13407137989997864,
+          -0.591465950012207,
+          0.3446292281150818,
+          0.6069018244743347,
+          0.1935492902994156,
+          -0.0989871397614479,
+          0.07008486241102219,
+          -0.8503749370574951,
+          -0.09507356584072112,
+          0.6259510517120361,
+          0.13934025168418884,
+          0.06392545253038406,
+          -0.4112265408039093,
+          -0.08475656062364578,
+          0.4974113404750824,
+          -0.30606114864349365,
+          1.111435890197754,
+          -0.018766529858112335,
+          -0.8422622680664063,
+          0.4325508773326874,
+          -0.2832120656967163,
+          -0.4859798848628998,
+          -0.41498348116874695,
+          0.015977520495653152,
+          0.5292825698852539,
+          0.4538311660289765,
+          1.1328668594360352,
+          0.22632671892642975,
+          0.7918671369552612,
+          0.33401933312416077,
+          0.7306135296821594,
+          0.3548600673675537,
+          0.12506209313869476,
+          0.8573207855224609,
+          -0.5818327069282532,
+          -0.6953738927841187,
+          -1.6171947717666626,
+          -0.1699674427509308,
+          0.6318262815475464,
+          -0.05671752244234085,
+          -0.28145185112953186,
+          -0.3976689279079437,
+          -0.2041076272726059,
+          -0.5495951175689697,
+          -0.5152917504310608,
+          -0.9309796094894408,
+          0.101932130753994,
+          0.1367802917957306,
+          0.1490798443555832,
+          0.5304336547851563,
+          -0.5082434415817261,
+          0.06688683480024338,
+          0.14657628536224365,
+          -0.782435953617096,
+          0.2962816655635834,
+          0.6965363621711731,
+          0.8496337532997131,
+          -0.3042965829372406,
+          0.04343798756599426,
+          0.0330701619386673,
+          -0.5662598013877869,
+          1.1086925268173218,
+          0.756072998046875,
+          -0.204134538769722,
+          0.2404300570487976,
+          -0.47848284244537354,
+          1.3659011125564575,
+          0.5645433068275452,
+          -0.15836156904697418,
+          0.43395575881004333,
+          0.5944653749465942,
+          1.0043466091156006,
+          -0.49446743726730347,
+          -0.5954391360282898,
+          0.5341240763664246,
+          0.020598189905285835,
+          -0.4036853015422821,
+          0.4473709762096405,
+          1.1998231410980225,
+          -0.9317775368690492,
+          -0.23321466147899628,
+          0.2052552700042725,
+          -0.7423108816146851,
+          -0.19917210936546328,
+          -0.1722569614648819,
+          -0.034072667360305786,
+          -0.00671181408688426,
+          0.46396249532699585,
+          -0.1372445821762085,
+          0.053376372903585434,
+          0.7392690777778625,
+          -0.38447609543800354,
+          0.07497968524694443,
+          0.5197252631187439,
+          1.3746477365493774,
+          0.9060075879096984,
+          0.20000585913658145,
+          -0.4053704142570496,
+          0.7497360110282898,
+          -0.34087055921554565,
+          -1.101803183555603,
+          0.273650586605072,
+          -0.5125769376754761,
+          0.22472351789474487,
+          0.480757474899292,
+          -0.19845178723335263,
+          0.8857700824737549,
+          0.30752456188201904,
+          1.1109285354614258,
+          -0.6768012642860413,
+          0.524367094039917,
+          -0.22495046257972717,
+          -0.4224412739276886,
+          0.40753406286239624,
+          -0.23133376240730288,
+          0.3297771215438843,
+          0.4905449151992798,
+          -0.6813114285469055,
+          -0.7543983459472656,
+          -0.5599071383476257,
+          0.14351597428321838,
+          -0.029278717935085297,
+          -0.3970443606376648,
+          -0.303079217672348,
+          0.24161772429943085,
+          0.008353390730917454,
+          -0.0062365154735744,
+          1.0824860334396362,
+          -0.3704061508178711,
+          -1.0337258577346802,
+          0.04638749733567238,
+          1.163011074066162,
+          -0.31737643480300903,
+          0.013986887410283089,
+          0.19223114848136905,
+          -0.2260770797729492,
+          -0.210910826921463,
+          -1.0191949605941772,
+          0.22356095910072327,
+          0.09353553503751756,
+          0.18096882104873657,
+          0.14867214858531952,
+          0.43408671021461487,
+          -0.33312076330184937,
+          0.8173948526382446,
+          0.6428242921829224,
+          0.20215003192424777,
+          -0.6634518504142761,
+          -0.4132290482521057,
+          0.29815030097961426,
+          -1.579406976699829,
+          -0.0981958732008934,
+          -0.03941014781594277,
+          0.1709178239107132,
+          -0.5481140613555908,
+          -0.5338194966316223,
+          -0.3528362512588501,
+          -0.11561278253793716,
+          -0.21793591976165771,
+          -1.1570470333099363,
+          0.2157980799674988,
+          0.42083489894866943,
+          0.9639263153076172,
+          0.09747201204299928,
+          0.15671424567699432,
+          0.4034591615200043,
+          0.6728067994117737,
+          -0.5216875672340393,
+          0.09657668322324751,
+          -0.2416689097881317,
+          0.747975766658783,
+          0.1021689772605896,
+          0.11652665585279463,
+          -1.0484966039657593,
+          0.8489304780960083,
+          0.7169828414916992,
+          -0.09012343734502792,
+          -1.3173753023147583,
+          0.057890523225069046,
+          -0.006231260951608419,
+          -0.1018214002251625,
+          0.936040461063385,
+          -0.0502331368625164,
+          -0.4284322261810303,
+          -0.38209280371665955,
+          -0.22668412327766416,
+          0.0782942995429039,
+          -0.4881664514541626,
+          0.9268959760665894,
+          0.001867273123934865,
+          0.42261114716529846,
+          0.8283362984657288,
+          0.4256294071674347,
+          -0.7965338826179504,
+          0.4840078353881836,
+          -0.19861412048339844,
+          0.33977967500686646,
+          -0.4604192078113556,
+          -0.3107339143753052,
+          -0.2839638590812683,
+          -1.5734281539916992,
+          0.005220232997089624,
+          0.09239906817674635,
+          -0.7828494906425476,
+          -0.1397123783826828,
+          0.2576255202293396,
+          0.21372435986995697,
+          -0.23169949650764465,
+          0.4016408920288086,
+          -0.462497353553772,
+          -0.2186472862958908,
+          -0.5617868900299072,
+          -0.3649831712245941,
+          -1.1585862636566162,
+          -0.08222806453704834,
+          0.931126832962036,
+          0.4327389597892761,
+          -0.46451422572135925,
+          -0.5430706143379211,
+          -0.27434298396110535,
+          -0.9479129314422609,
+          0.1845661848783493,
+          0.3972720205783844,
+          0.4883299469947815,
+          1.04031240940094
+        ]
+      ],
+      "regenerate": true
+    }
+  }
+}

crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-9.snap

@@ -0,0 +1,780 @@
+---
+source: dump/src/reader/mod.rs
+expression: document
+---
+{
+  "id": "e0",
+  "desc": "overriden vector",
+  "_vectors": {
+    "default": [
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1
+    ]
+  }
+}