Make max retry duration configurable with MEILI_EXPERIMENTAL_REST_EMBEDDER_MAX_RETRY_DURATION_SECONDS

Cargo.lock

@@ -589,7 +589,7 @@ source = "git+https://github.com/meilisearch/bbqueue#cbb87cc707b5af415ef203bdaf2
 
 [[package]]
 name = "benchmarks"
-version = "1.25.0"
+version = "1.24.0"
 dependencies = [
  "anyhow",
  "bumpalo",
@@ -799,7 +799,7 @@ dependencies = [
 
 [[package]]
 name = "build-info"
-version = "1.25.0"
+version = "1.24.0"
 dependencies = [
  "anyhow",
  "time",
@@ -1128,9 +1128,9 @@ dependencies = [
 
 [[package]]
 name = "charabia"
-version = "0.9.8"
+version = "0.9.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbdc8cd8f999e8b8b13ed71d30962bbf98cf39e2f2a9f1ae1ba354199239d66e"
+checksum = "7c2f456825b7f15eac01a1cae40c12c3f55e931d4327e6e4fa59508d664e9568"
 dependencies = [
  "aho-corasick",
  "csv",
@@ -1139,6 +1139,7 @@ dependencies = [
  "irg-kvariants",
  "jieba-rs",
  "lindera",
+ "once_cell",
  "pinyin",
  "serde",
  "slice-group-by",
@@ -1823,12 +1824,12 @@ version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9"
 dependencies = [
- "phf 0.11.3",
+ "phf",
 ]
 
 [[package]]
 name = "dump"
-version = "1.25.0"
+version = "1.24.0"
 dependencies = [
  "anyhow",
  "big_s",
@@ -2071,7 +2072,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 
 [[package]]
 name = "file-store"
-version = "1.25.0"
+version = "1.24.0"
 dependencies = [
  "tempfile",
  "thiserror 2.0.16",
@@ -2093,7 +2094,7 @@ dependencies = [
 
 [[package]]
 name = "filter-parser"
-version = "1.25.0"
+version = "1.24.0"
 dependencies = [
  "insta",
  "levenshtein_automata",
@@ -2121,7 +2122,7 @@ dependencies = [
 
 [[package]]
 name = "flatten-serde-json"
-version = "1.25.0"
+version = "1.24.0"
 dependencies = [
  "criterion",
  "serde_json",
@@ -2278,7 +2279,7 @@ dependencies = [
 
 [[package]]
 name = "fuzzers"
-version = "1.25.0"
+version = "1.24.0"
 dependencies = [
  "arbitrary",
  "bumpalo",
@@ -2837,9 +2838,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
 [[package]]
 name = "heed"
-version = "0.22.1-nested-rtxns-6"
+version = "0.22.1-nested-rtxns"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c69e07cd539834bedcfa938f3d7d8520cce1ad2b0776c122b5ccdf8fd5bafe12"
+checksum = "0ff115ba5712b1f1fc7617b195f5c2f139e29c397ff79da040cd19db75ccc240"
 dependencies = [
  "bitflags 2.9.4",
  "byteorder",
@@ -3232,7 +3233,7 @@ dependencies = [
 
 [[package]]
 name = "index-scheduler"
-version = "1.25.0"
+version = "1.24.0"
 dependencies = [
  "anyhow",
  "backoff",
@@ -3241,7 +3242,6 @@ dependencies = [
  "bumpalo",
  "bumparaw-collections",
  "byte-unit",
- "bytes",
  "convert_case 0.8.0",
  "crossbeam-channel",
  "csv",
@@ -3259,17 +3259,13 @@ dependencies = [
  "memmap2",
  "page_size",
  "rayon",
- "reqwest",
  "roaring 0.10.12",
- "rusty-s3",
  "serde",
  "serde_json",
  "synchronoise",
- "tar",
  "tempfile",
  "thiserror 2.0.16",
  "time",
- "tokio",
  "tracing",
  "ureq",
  "uuid",
@@ -3447,49 +3443,26 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
 
 [[package]]
 name = "jieba-macros"
-version = "0.8.1"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "348294e44ee7e3c42685da656490f8febc7359632544019621588902216da95c"
+checksum = "7c676b32a471d3cfae8dac2ad2f8334cd52e53377733cca8c1fb0a5062fec192"
 dependencies = [
  "phf_codegen",
 ]
 
 [[package]]
 name = "jieba-rs"
-version = "0.8.1"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "766bd7012aa5ba49411ebdf4e93bddd59b182d2918e085d58dec5bb9b54b7105"
+checksum = "f5dd552bbb95d578520ee68403bf8aaf0dbbb2ce55b0854d019f9350ad61040a"
 dependencies = [
  "cedarwood",
+ "fxhash",
  "include-flate",
  "jieba-macros",
- "phf 0.13.1",
+ "lazy_static",
+ "phf",
  "regex",
- "rustc-hash 2.1.1",
-]
-
-[[package]]
-name = "jiff"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49"
-dependencies = [
- "jiff-static",
- "log",
- "portable-atomic",
- "portable-atomic-util",
- "serde",
-]
-
-[[package]]
-name = "jiff-static"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.106",
 ]
 
 [[package]]
@@ -3514,7 +3487,7 @@ dependencies = [
 
 [[package]]
 name = "json-depth-checker"
-version = "1.25.0"
+version = "1.24.0"
 dependencies = [
  "criterion",
  "serde_json",
@@ -3915,9 +3888,9 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956"
 
 [[package]]
 name = "lmdb-master-sys"
-version = "0.2.6-nested-rtxns-6"
+version = "0.2.6-nested-rtxns"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e113d9bf240f974fbe7fd516cbfd8c422e925c0655495501c7237548425493d0"
+checksum = "f4ff85130e3c994b36877045fbbb138d521dea7197bfc19dc3d5d95101a8e20a"
 dependencies = [
  "cc",
  "doxygen-rs",
@@ -4015,16 +3988,6 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
 
-[[package]]
-name = "md-5"
-version = "0.10.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
-dependencies = [
- "cfg-if",
- "digest",
-]
-
 [[package]]
 name = "md5"
 version = "0.7.0"
@@ -4033,7 +3996,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
 
 [[package]]
 name = "meili-snap"
-version = "1.25.0"
+version = "1.24.0"
 dependencies = [
  "insta",
  "md5",
@@ -4044,7 +4007,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch"
-version = "1.25.0"
+version = "1.24.0"
 dependencies = [
  "actix-cors",
  "actix-http",
@@ -4141,7 +4104,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-auth"
-version = "1.25.0"
+version = "1.24.0"
 dependencies = [
  "base64 0.22.1",
  "enum-iterator",
@@ -4160,7 +4123,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-types"
-version = "1.25.0"
+version = "1.24.0"
 dependencies = [
  "actix-web",
  "anyhow",
@@ -4195,7 +4158,7 @@ dependencies = [
 
 [[package]]
 name = "meilitool"
-version = "1.25.0"
+version = "1.24.0"
 dependencies = [
  "anyhow",
  "clap",
@@ -4229,7 +4192,7 @@ dependencies = [
 
 [[package]]
 name = "milli"
-version = "1.25.0"
+version = "1.24.0"
 dependencies = [
  "allocator-api2 0.3.1",
  "arroy",
@@ -4810,7 +4773,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
 
 [[package]]
 name = "permissive-json-pointer"
-version = "1.25.0"
+version = "1.24.0"
 dependencies = [
  "big_s",
  "serde_json",
@@ -4867,27 +4830,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
 dependencies = [
  "phf_macros",
- "phf_shared 0.11.3",
-]
-
-[[package]]
-name = "phf"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
-dependencies = [
- "phf_shared 0.13.1",
- "serde",
+ "phf_shared",
 ]
 
 [[package]]
 name = "phf_codegen"
-version = "0.13.1"
+version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1"
+checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
 dependencies = [
- "phf_generator 0.13.1",
- "phf_shared 0.13.1",
+ "phf_generator",
+ "phf_shared",
 ]
 
 [[package]]
@@ -4896,28 +4849,18 @@ version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
 dependencies = [
- "phf_shared 0.11.3",
+ "phf_shared",
  "rand 0.8.5",
 ]
 
-[[package]]
-name = "phf_generator"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"
-dependencies = [
- "fastrand",
- "phf_shared 0.13.1",
-]
-
 [[package]]
 name = "phf_macros"
 version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216"
 dependencies = [
- "phf_generator 0.11.3",
- "phf_shared 0.11.3",
+ "phf_generator",
+ "phf_shared",
  "proc-macro2",
  "quote",
  "syn 2.0.106",
@@ -4932,15 +4875,6 @@ dependencies = [
  "siphasher",
 ]
 
-[[package]]
-name = "phf_shared"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266"
-dependencies = [
- "siphasher",
-]
-
 [[package]]
 name = "pin-project"
 version = "1.1.10"
@@ -5028,15 +4962,6 @@ version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
 
-[[package]]
-name = "portable-atomic-util"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507"
-dependencies = [
- "portable-atomic",
-]
-
 [[package]]
 name = "potential_utf"
 version = "0.1.3"
@@ -5214,16 +5139,6 @@ dependencies = [
  "version_check",
 ]
 
-[[package]]
-name = "quick-xml"
-version = "0.38.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42a232e7487fc2ef313d96dde7948e7a3c05101870d8985e4fd8d26aedd27b89"
-dependencies = [
- "memchr",
- "serde",
-]
-
 [[package]]
 name = "quinn"
 version = "0.11.9"
@@ -5499,7 +5414,6 @@ dependencies = [
  "futures-channel",
  "futures-core",
  "futures-util",
- "h2 0.4.12",
  "http 1.3.1",
  "http-body",
  "http-body-util",
@@ -5790,25 +5704,6 @@ version = "1.0.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
 
-[[package]]
-name = "rusty-s3"
-version = "0.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fac2edd2f0b56bd79a7343f49afc01c2d41010df480538a510e0abc56044f66c"
-dependencies = [
- "base64 0.22.1",
- "hmac",
- "jiff",
- "md-5",
- "percent-encoding",
- "quick-xml",
- "serde",
- "serde_json",
- "sha2",
- "url",
- "zeroize",
-]
-
 [[package]]
 name = "ryu"
 version = "1.0.20"
@@ -7925,7 +7820,7 @@ dependencies = [
 
 [[package]]
 name = "xtask"
-version = "1.25.0"
+version = "1.24.0"
 dependencies = [
  "anyhow",
  "build-info",

Cargo.toml

@@ -23,7 +23,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.25.0"
+version = "1.24.0"
 authors = [
     "Quentin de Quelen <quentin@dequelen.me>",
     "Clément Renault <clement@meilisearch.com>",

README.md

@@ -39,7 +39,6 @@
 ## 🖥 Examples
 
 - [**Movies**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=organization) — An application to help you find streaming platforms to watch movies using [hybrid search](https://www.meilisearch.com/solutions/hybrid-search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos).
-- [**Flickr**](https://flickr.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=organization) — Search and explore one hundred million Flickr images with semantic search.
 - [**Ecommerce**](https://ecommerce.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Ecommerce website using disjunctive [facets](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos), range and rating filtering, and pagination.
 - [**Songs**](https://music.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search through 47 million of songs.
 - [**SaaS**](https://saas.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search for contacts, deals, and companies in this [multi-tenant](https://www.meilisearch.com/docs/learn/security/multitenancy_tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) CRM application.

crates/file-store/src/lib.rs

@@ -60,7 +60,7 @@ impl FileStore {
 
     /// Returns the file corresponding to the requested uuid.
     pub fn get_update(&self, uuid: Uuid) -> Result<StdFile> {
-        let path = self.update_path(uuid);
+        let path = self.get_update_path(uuid);
         let file = match StdFile::open(path) {
             Ok(file) => file,
             Err(e) => {
@@ -72,7 +72,7 @@ impl FileStore {
     }
 
     /// Returns the path that correspond to this uuid, the path could not exists.
-    pub fn update_path(&self, uuid: Uuid) -> PathBuf {
+    pub fn get_update_path(&self, uuid: Uuid) -> PathBuf {
         self.path.join(uuid.to_string())
     }
 

crates/index-scheduler/Cargo.toml

@@ -14,7 +14,6 @@ license.workspace = true
 anyhow = "1.0.98"
 bincode = "1.3.3"
 byte-unit = "5.1.6"
-bytes = "1.10.1"
 bumpalo = "3.18.1"
 bumparaw-collections = "0.1.4"
 convert_case = "0.8.0"
@@ -33,7 +32,6 @@ rayon = "1.10.0"
 roaring = { version = "0.10.12", features = ["serde"] }
 serde = { version = "1.0.219", features = ["derive"] }
 serde_json = { version = "1.0.140", features = ["preserve_order"] }
-tar = "0.4.44"
 synchronoise = "1.0.1"
 tempfile = "3.20.0"
 thiserror = "2.0.12"
@@ -47,9 +45,6 @@ tracing = "0.1.41"
 ureq = "2.12.1"
 uuid = { version = "1.17.0", features = ["serde", "v4"] }
 backoff = "0.4.0"
-reqwest = { version = "0.12.23", features = ["rustls-tls", "http2"], default-features = false }
-rusty-s3 = "0.8.1"
-tokio = { version = "1.47.1", features = ["full"] }
 
 [dev-dependencies]
 big_s = "1.0.2"

crates/index-scheduler/src/error.rs

@@ -5,7 +5,6 @@ use meilisearch_types::error::{Code, ErrorCode};
 use meilisearch_types::milli::index::RollbackOutcome;
 use meilisearch_types::tasks::{Kind, Status};
 use meilisearch_types::{heed, milli};
-use reqwest::StatusCode;
 use thiserror::Error;
 
 use crate::TaskId;
@@ -128,14 +127,6 @@ pub enum Error {
     #[error("Aborted task")]
     AbortedTask,
 
-    #[error("S3 error: status: {status}, body: {body}")]
-    S3Error { status: StatusCode, body: String },
-    #[error("S3 HTTP error: {0}")]
-    S3HttpError(reqwest::Error),
-    #[error("S3 XML error: {0}")]
-    S3XmlError(Box<dyn std::error::Error + Send + Sync>),
-    #[error("S3 bucket error: {0}")]
-    S3BucketError(rusty_s3::BucketError),
     #[error(transparent)]
     Dump(#[from] dump::Error),
     #[error(transparent)]
@@ -235,10 +226,6 @@ impl Error {
             | Error::TaskCancelationWithEmptyQuery
             | Error::FromRemoteWhenExporting { .. }
             | Error::AbortedTask
-            | Error::S3Error { .. }
-            | Error::S3HttpError(_)
-            | Error::S3XmlError(_)
-            | Error::S3BucketError(_)
             | Error::Dump(_)
             | Error::Heed(_)
             | Error::Milli { .. }
@@ -306,14 +293,8 @@ impl ErrorCode for Error {
             Error::BatchNotFound(_) => Code::BatchNotFound,
             Error::TaskDeletionWithEmptyQuery => Code::MissingTaskFilters,
             Error::TaskCancelationWithEmptyQuery => Code::MissingTaskFilters,
+            // TODO: not sure of the Code to use
             Error::NoSpaceLeftInTaskQueue => Code::NoSpaceLeftOnDevice,
-            Error::S3Error { status, .. } if status.is_client_error() => {
-                Code::InvalidS3SnapshotRequest
-            }
-            Error::S3Error { .. } => Code::S3SnapshotServerError,
-            Error::S3HttpError(_) => Code::S3SnapshotServerError,
-            Error::S3XmlError(_) => Code::S3SnapshotServerError,
-            Error::S3BucketError(_) => Code::InvalidS3SnapshotParameters,
             Error::Dump(e) => e.error_code(),
             Error::Milli { error, .. } => error.error_code(),
             Error::ProcessBatchPanicked(_) => Code::Internal,

crates/index-scheduler/src/insta_snapshot.rs

@@ -36,7 +36,6 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
         run_loop_iteration: _,
         embedders: _,
         chat_settings: _,
-        runtime: _,
     } = scheduler;
 
     let rtxn = env.read_txn().unwrap();

crates/index-scheduler/src/lib.rs

@@ -216,9 +216,6 @@ pub struct IndexScheduler {
     /// A counter that is incremented before every call to [`tick`](IndexScheduler::tick)
     #[cfg(test)]
     run_loop_iteration: Arc<RwLock<usize>>,
-
-    /// The tokio runtime used for asynchronous tasks.
-    runtime: Option<tokio::runtime::Handle>,
 }
 
 impl IndexScheduler {
@@ -245,7 +242,6 @@ impl IndexScheduler {
             run_loop_iteration: self.run_loop_iteration.clone(),
             features: self.features.clone(),
             chat_settings: self.chat_settings,
-            runtime: self.runtime.clone(),
         }
     }
 
@@ -259,23 +255,13 @@ impl IndexScheduler {
     }
 
     /// Create an index scheduler and start its run loop.
+    #[allow(private_interfaces)] // because test_utils is private
     pub fn new(
         options: IndexSchedulerOptions,
         auth_env: Env<WithoutTls>,
         from_db_version: (u32, u32, u32),
-        runtime: Option<tokio::runtime::Handle>,
-    ) -> Result<Self> {
-        let this = Self::new_without_run(options, auth_env, from_db_version, runtime)?;
-
-        this.run();
-        Ok(this)
-    }
-
-    fn new_without_run(
-        options: IndexSchedulerOptions,
-        auth_env: Env<WithoutTls>,
-        from_db_version: (u32, u32, u32),
-        runtime: Option<tokio::runtime::Handle>,
+        #[cfg(test)] test_breakpoint_sdr: crossbeam_channel::Sender<(test_utils::Breakpoint, bool)>,
+        #[cfg(test)] planned_failures: Vec<(usize, test_utils::FailureLocation)>,
     ) -> Result<Self> {
         std::fs::create_dir_all(&options.tasks_path)?;
         std::fs::create_dir_all(&options.update_file_path)?;
@@ -330,7 +316,8 @@ impl IndexScheduler {
 
         wtxn.commit()?;
 
-        Ok(Self {
+        // allow unreachable_code to get rids of the warning in the case of a test build.
+        let this = Self {
             processing_tasks: Arc::new(RwLock::new(ProcessingTasks::new())),
             version,
             queue,
@@ -346,32 +333,15 @@ impl IndexScheduler {
             webhooks: Arc::new(webhooks),
             embedders: Default::default(),
 
-            #[cfg(test)] // Will be replaced in `new_tests` in test environments
-            test_breakpoint_sdr: crossbeam_channel::bounded(0).0,
-            #[cfg(test)] // Will be replaced in `new_tests` in test environments
-            planned_failures: Default::default(),
+            #[cfg(test)]
+            test_breakpoint_sdr,
+            #[cfg(test)]
+            planned_failures,
             #[cfg(test)]
             run_loop_iteration: Arc::new(RwLock::new(0)),
             features,
             chat_settings,
-            runtime,
-        })
-    }
-
-    /// Create an index scheduler and start its run loop.
-    #[cfg(test)]
-    fn new_test(
-        options: IndexSchedulerOptions,
-        auth_env: Env<WithoutTls>,
-        from_db_version: (u32, u32, u32),
-        runtime: Option<tokio::runtime::Handle>,
-        test_breakpoint_sdr: crossbeam_channel::Sender<(test_utils::Breakpoint, bool)>,
-        planned_failures: Vec<(usize, test_utils::FailureLocation)>,
-    ) -> Result<Self> {
-        let mut this = Self::new_without_run(options, auth_env, from_db_version, runtime)?;
-
-        this.test_breakpoint_sdr = test_breakpoint_sdr;
-        this.planned_failures = planned_failures;
+        };
 
         this.run();
         Ok(this)

crates/index-scheduler/src/scheduler/mod.rs

@@ -25,7 +25,6 @@ use convert_case::{Case, Casing as _};
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::heed::{Env, WithoutTls};
 use meilisearch_types::milli;
-use meilisearch_types::milli::update::S3SnapshotOptions;
 use meilisearch_types::tasks::Status;
 use process_batch::ProcessBatchInfo;
 use rayon::current_num_threads;
@@ -88,14 +87,11 @@ pub struct Scheduler {
 
     /// Snapshot compaction status.
     pub(crate) experimental_no_snapshot_compaction: bool,
-
-    /// S3 Snapshot options.
-    pub(crate) s3_snapshot_options: Option<S3SnapshotOptions>,
 }
 
 impl Scheduler {
-    pub(crate) fn private_clone(&self) -> Self {
-        Self {
+    pub(crate) fn private_clone(&self) -> Scheduler {
+        Scheduler {
             must_stop_processing: self.must_stop_processing.clone(),
             wake_up: self.wake_up.clone(),
             autobatching_enabled: self.autobatching_enabled,
@@ -107,52 +103,23 @@ impl Scheduler {
             version_file_path: self.version_file_path.clone(),
             embedding_cache_cap: self.embedding_cache_cap,
             experimental_no_snapshot_compaction: self.experimental_no_snapshot_compaction,
-            s3_snapshot_options: self.s3_snapshot_options.clone(),
         }
     }
 
     pub fn new(options: &IndexSchedulerOptions, auth_env: Env<WithoutTls>) -> Scheduler {
-        let IndexSchedulerOptions {
-            version_file_path,
-            auth_path: _,
-            tasks_path: _,
-            update_file_path: _,
-            indexes_path: _,
-            snapshots_path,
-            dumps_path,
-            cli_webhook_url: _,
-            cli_webhook_authorization: _,
-            task_db_size: _,
-            index_base_map_size: _,
-            enable_mdb_writemap: _,
-            index_growth_amount: _,
-            index_count: _,
-            indexer_config,
-            autobatching_enabled,
-            cleanup_enabled: _,
-            max_number_of_tasks: _,
-            max_number_of_batched_tasks,
-            batched_tasks_size_limit,
-            instance_features: _,
-            auto_upgrade: _,
-            embedding_cache_cap,
-            experimental_no_snapshot_compaction,
-        } = options;
-
         Scheduler {
             must_stop_processing: MustStopProcessing::default(),
             // we want to start the loop right away in case meilisearch was ctrl+Ced while processing things
             wake_up: Arc::new(SignalEvent::auto(true)),
-            autobatching_enabled: *autobatching_enabled,
-            max_number_of_batched_tasks: *max_number_of_batched_tasks,
-            batched_tasks_size_limit: *batched_tasks_size_limit,
-            dumps_path: dumps_path.clone(),
-            snapshots_path: snapshots_path.clone(),
+            autobatching_enabled: options.autobatching_enabled,
+            max_number_of_batched_tasks: options.max_number_of_batched_tasks,
+            batched_tasks_size_limit: options.batched_tasks_size_limit,
+            dumps_path: options.dumps_path.clone(),
+            snapshots_path: options.snapshots_path.clone(),
             auth_env,
-            version_file_path: version_file_path.clone(),
-            embedding_cache_cap: *embedding_cache_cap,
-            experimental_no_snapshot_compaction: *experimental_no_snapshot_compaction,
-            s3_snapshot_options: indexer_config.s3_snapshot_options.clone(),
+            version_file_path: options.version_file_path.clone(),
+            embedding_cache_cap: options.embedding_cache_cap,
+            experimental_no_snapshot_compaction: options.experimental_no_snapshot_compaction,
         }
     }
 }

crates/index-scheduler/src/scheduler/process_snapshot_creation.rs

@@ -12,8 +12,6 @@ use crate::processing::{AtomicUpdateFileStep, SnapshotCreationProgress};
 use crate::queue::TaskQueue;
 use crate::{Error, IndexScheduler, Result};
 
-const UPDATE_FILES_DIR_NAME: &str = "update_files";
-
 /// # Safety
 ///
 /// See [`EnvOpenOptions::open`].
@@ -80,32 +78,10 @@ impl IndexScheduler {
     pub(super) fn process_snapshot(
         &self,
         progress: Progress,
-        tasks: Vec<Task>,
+        mut tasks: Vec<Task>,
     ) -> Result<Vec<Task>> {
         progress.update_progress(SnapshotCreationProgress::StartTheSnapshotCreation);
 
-        match self.scheduler.s3_snapshot_options.clone() {
-            Some(options) => {
-                #[cfg(not(unix))]
-                {
-                    let _ = options;
-                    panic!("Non-unix platform does not support S3 snapshotting");
-                }
-                #[cfg(unix)]
-                self.runtime
-                    .as_ref()
-                    .expect("Runtime not initialized")
-                    .block_on(self.process_snapshot_to_s3(progress, options, tasks))
-            }
-            None => self.process_snapshots_to_disk(progress, tasks),
-        }
-    }
-
-    fn process_snapshots_to_disk(
-        &self,
-        progress: Progress,
-        mut tasks: Vec<Task>,
-    ) -> Result<Vec<Task>, Error> {
         fs::create_dir_all(&self.scheduler.snapshots_path)?;
         let temp_snapshot_dir = tempfile::tempdir()?;
 
@@ -152,7 +128,7 @@ impl IndexScheduler {
         let rtxn = self.env.read_txn()?;
 
         // 2.4 Create the update files directory
-        let update_files_dir = temp_snapshot_dir.path().join(UPDATE_FILES_DIR_NAME);
+        let update_files_dir = temp_snapshot_dir.path().join("update_files");
         fs::create_dir_all(&update_files_dir)?;
 
         // 2.5 Only copy the update files of the enqueued tasks
@@ -164,7 +140,7 @@ impl IndexScheduler {
             let task =
                 self.queue.tasks.get_task(&rtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?;
             if let Some(content_uuid) = task.content_uuid() {
-                let src = self.queue.file_store.update_path(content_uuid);
+                let src = self.queue.file_store.get_update_path(content_uuid);
                 let dst = update_files_dir.join(content_uuid.to_string());
                 fs::copy(src, dst)?;
             }
@@ -230,403 +206,4 @@ impl IndexScheduler {
 
         Ok(tasks)
     }
-
-    #[cfg(unix)]
-    pub(super) async fn process_snapshot_to_s3(
-        &self,
-        progress: Progress,
-        opts: meilisearch_types::milli::update::S3SnapshotOptions,
-        mut tasks: Vec<Task>,
-    ) -> Result<Vec<Task>> {
-        use meilisearch_types::milli::update::S3SnapshotOptions;
-
-        let S3SnapshotOptions {
-            s3_bucket_url,
-            s3_bucket_region,
-            s3_bucket_name,
-            s3_snapshot_prefix,
-            s3_access_key,
-            s3_secret_key,
-            s3_max_in_flight_parts,
-            s3_compression_level: level,
-            s3_signature_duration,
-            s3_multipart_part_size,
-        } = opts;
-
-        let must_stop_processing = self.scheduler.must_stop_processing.clone();
-        let retry_backoff = backoff::ExponentialBackoff::default();
-        let db_name = {
-            let mut base_path = self.env.path().to_owned();
-            base_path.pop();
-            base_path.file_name().and_then(OsStr::to_str).unwrap_or("data.ms").to_string()
-        };
-
-        let (reader, writer) = std::io::pipe()?;
-        let uploader_task = tokio::spawn(multipart_stream_to_s3(
-            s3_bucket_url,
-            s3_bucket_region,
-            s3_bucket_name,
-            s3_snapshot_prefix,
-            s3_access_key,
-            s3_secret_key,
-            s3_max_in_flight_parts,
-            s3_signature_duration,
-            s3_multipart_part_size,
-            must_stop_processing,
-            retry_backoff,
-            db_name,
-            reader,
-        ));
-
-        let index_scheduler = IndexScheduler::private_clone(self);
-        let builder_task = tokio::task::spawn_blocking(move || {
-            stream_tarball_into_pipe(progress, level, writer, index_scheduler)
-        });
-
-        let (uploader_result, builder_result) = tokio::join!(uploader_task, builder_task);
-
-        // Check uploader result first to early return on task abortion.
-        // safety: JoinHandle can return an error if the task was aborted, cancelled, or panicked.
-        uploader_result.unwrap()?;
-        builder_result.unwrap()?;
-
-        for task in &mut tasks {
-            task.status = Status::Succeeded;
-        }
-
-        Ok(tasks)
-    }
-}
-
-/// Streams a tarball of the database content into a pipe.
-#[cfg(unix)]
-fn stream_tarball_into_pipe(
-    progress: Progress,
-    level: u32,
-    writer: std::io::PipeWriter,
-    index_scheduler: IndexScheduler,
-) -> std::result::Result<(), Error> {
-    use std::io::Write as _;
-    use std::path::Path;
-
-    let writer = flate2::write::GzEncoder::new(writer, flate2::Compression::new(level));
-    let mut tarball = tar::Builder::new(writer);
-
-    // 1. Snapshot the version file
-    tarball
-        .append_path_with_name(&index_scheduler.scheduler.version_file_path, VERSION_FILE_NAME)?;
-
-    // 2. Snapshot the index scheduler LMDB env
-    progress.update_progress(SnapshotCreationProgress::SnapshotTheIndexScheduler);
-    let tasks_env_file = index_scheduler.env.try_clone_inner_file()?;
-    let path = Path::new("tasks").join("data.mdb");
-    append_file_to_tarball(&mut tarball, path, tasks_env_file)?;
-
-    // 2.3 Create a read transaction on the index-scheduler
-    let rtxn = index_scheduler.env.read_txn()?;
-
-    // 2.4 Create the update files directory
-    //     And only copy the update files of the enqueued tasks
-    progress.update_progress(SnapshotCreationProgress::SnapshotTheUpdateFiles);
-    let enqueued = index_scheduler.queue.tasks.get_status(&rtxn, Status::Enqueued)?;
-    let (atomic, update_file_progress) = AtomicUpdateFileStep::new(enqueued.len() as u32);
-    progress.update_progress(update_file_progress);
-
-    // We create the update_files directory so that it
-    // always exists even if there are no update files
-    let update_files_dir = Path::new(UPDATE_FILES_DIR_NAME);
-    let src_update_files_dir = {
-        let mut path = index_scheduler.env.path().to_path_buf();
-        path.pop();
-        path.join(UPDATE_FILES_DIR_NAME)
-    };
-    tarball.append_dir(update_files_dir, src_update_files_dir)?;
-
-    for task_id in enqueued {
-        let task = index_scheduler
-            .queue
-            .tasks
-            .get_task(&rtxn, task_id)?
-            .ok_or(Error::CorruptedTaskQueue)?;
-        if let Some(content_uuid) = task.content_uuid() {
-            use std::fs::File;
-
-            let src = index_scheduler.queue.file_store.update_path(content_uuid);
-            let mut update_file = File::open(src)?;
-            let path = update_files_dir.join(content_uuid.to_string());
-            tarball.append_file(path, &mut update_file)?;
-        }
-        atomic.fetch_add(1, Ordering::Relaxed);
-    }
-
-    // 3. Snapshot every indexes
-    progress.update_progress(SnapshotCreationProgress::SnapshotTheIndexes);
-    let index_mapping = index_scheduler.index_mapper.index_mapping;
-    let nb_indexes = index_mapping.len(&rtxn)? as u32;
-    let indexes_dir = Path::new("indexes");
-    let indexes_references: Vec<_> = index_scheduler
-        .index_mapper
-        .index_mapping
-        .iter(&rtxn)?
-        .map(|res| res.map_err(Error::from).map(|(name, uuid)| (name.to_string(), uuid)))
-        .collect::<Result<_, Error>>()?;
-
-    // It's prettier to use a for loop instead of the IndexMapper::try_for_each_index
-    // method, especially when we need to access the UUID, local path and index number.
-    for (i, (name, uuid)) in indexes_references.into_iter().enumerate() {
-        progress.update_progress(VariableNameStep::<SnapshotCreationProgress>::new(
-            &name, i as u32, nb_indexes,
-        ));
-        let path = indexes_dir.join(uuid.to_string()).join("data.mdb");
-        let index = index_scheduler.index_mapper.index(&rtxn, &name)?;
-        let index_file = index.try_clone_inner_file()?;
-        tracing::trace!("Appending index file for {name} in {}", path.display());
-        append_file_to_tarball(&mut tarball, path, index_file)?;
-    }
-
-    drop(rtxn);
-
-    // 4. Snapshot the auth LMDB env
-    progress.update_progress(SnapshotCreationProgress::SnapshotTheApiKeys);
-    let auth_env_file = index_scheduler.scheduler.auth_env.try_clone_inner_file()?;
-    let path = Path::new("auth").join("data.mdb");
-    append_file_to_tarball(&mut tarball, path, auth_env_file)?;
-
-    let mut gzencoder = tarball.into_inner()?;
-    gzencoder.flush()?;
-    gzencoder.try_finish()?;
-    let mut writer = gzencoder.finish()?;
-    writer.flush()?;
-
-    Result::<_, Error>::Ok(())
-}
-
-#[cfg(unix)]
-fn append_file_to_tarball<W, P>(
-    tarball: &mut tar::Builder<W>,
-    path: P,
-    mut auth_env_file: fs::File,
-) -> Result<(), Error>
-where
-    W: std::io::Write,
-    P: AsRef<std::path::Path>,
-{
-    use std::io::{Seek as _, SeekFrom};
-
-    // Note: A previous snapshot operation may have left the cursor
-    //       at the end of the file so we need to seek to the start.
-    auth_env_file.seek(SeekFrom::Start(0))?;
-    tarball.append_file(path, &mut auth_env_file)?;
-    Ok(())
-}
-
-/// Streams the content read from the given reader to S3.
-#[cfg(unix)]
-#[allow(clippy::too_many_arguments)]
-async fn multipart_stream_to_s3(
-    s3_bucket_url: String,
-    s3_bucket_region: String,
-    s3_bucket_name: String,
-    s3_snapshot_prefix: String,
-    s3_access_key: String,
-    s3_secret_key: String,
-    s3_max_in_flight_parts: std::num::NonZero<usize>,
-    s3_signature_duration: std::time::Duration,
-    s3_multipart_part_size: u64,
-    must_stop_processing: super::MustStopProcessing,
-    retry_backoff: backoff::exponential::ExponentialBackoff<backoff::SystemClock>,
-    db_name: String,
-    reader: std::io::PipeReader,
-) -> Result<(), Error> {
-    use std::{collections::VecDeque, os::fd::OwnedFd, path::PathBuf};
-
-    use bytes::{Bytes, BytesMut};
-    use reqwest::{Client, Response};
-    use rusty_s3::S3Action as _;
-    use rusty_s3::{actions::CreateMultipartUpload, Bucket, BucketError, Credentials, UrlStyle};
-    use tokio::task::JoinHandle;
-
-    let reader = OwnedFd::from(reader);
-    let reader = tokio::net::unix::pipe::Receiver::from_owned_fd(reader)?;
-    let s3_snapshot_prefix = PathBuf::from(s3_snapshot_prefix);
-    let url =
-        s3_bucket_url.parse().map_err(BucketError::ParseError).map_err(Error::S3BucketError)?;
-    let bucket = Bucket::new(url, UrlStyle::Path, s3_bucket_name, s3_bucket_region)
-        .map_err(Error::S3BucketError)?;
-    let credential = Credentials::new(s3_access_key, s3_secret_key);
-
-    // Note for the future (rust 1.91+): use with_added_extension, it's prettier
-    let object_path = s3_snapshot_prefix.join(format!("{db_name}.snapshot"));
-    // Note: It doesn't work on Windows and if a port to this platform is needed,
-    //       use the slash-path crate or similar to get the correct path separator.
-    let object = object_path.display().to_string();
-
-    let action = bucket.create_multipart_upload(Some(&credential), &object);
-    let url = action.sign(s3_signature_duration);
-
-    let client = Client::new();
-    let resp = client.post(url).send().await.map_err(Error::S3HttpError)?;
-    let status = resp.status();
-
-    let body = match resp.error_for_status_ref() {
-        Ok(_) => resp.text().await.map_err(Error::S3HttpError)?,
-        Err(_) => {
-            return Err(Error::S3Error { status, body: resp.text().await.unwrap_or_default() })
-        }
-    };
-
-    let multipart =
-        CreateMultipartUpload::parse_response(&body).map_err(|e| Error::S3XmlError(Box::new(e)))?;
-    tracing::debug!("Starting the upload of the snapshot to {object}");
-
-    // We use this bumpalo for etags strings.
-    let bump = bumpalo::Bump::new();
-    let mut etags = Vec::<&str>::new();
-    let mut in_flight = VecDeque::<(JoinHandle<reqwest::Result<Response>>, Bytes)>::with_capacity(
-        s3_max_in_flight_parts.get(),
-    );
-
-    // Part numbers start at 1 and cannot be larger than 10k
-    for part_number in 1u16.. {
-        if must_stop_processing.get() {
-            return Err(Error::AbortedTask);
-        }
-
-        let part_upload =
-            bucket.upload_part(Some(&credential), &object, part_number, multipart.upload_id());
-        let url = part_upload.sign(s3_signature_duration);
-
-        // Wait for a buffer to be ready if there are in-flight parts that landed
-        let mut buffer = if in_flight.len() >= s3_max_in_flight_parts.get() {
-            let (handle, buffer) = in_flight.pop_front().expect("At least one in flight request");
-            let resp = join_and_map_error(handle).await?;
-            extract_and_append_etag(&bump, &mut etags, resp.headers())?;
-
-            let mut buffer = match buffer.try_into_mut() {
-                Ok(buffer) => buffer,
-                Err(_) => unreachable!("All bytes references were consumed in the task"),
-            };
-            buffer.clear();
-            buffer
-        } else {
-            BytesMut::with_capacity(s3_multipart_part_size as usize)
-        };
-
-        // If we successfully read enough bytes,
-        // we can continue and send the buffer/part
-        while buffer.len() < (s3_multipart_part_size as usize / 2) {
-            // Wait for the pipe to be readable
-
-            use std::io;
-            reader.readable().await?;
-
-            match reader.try_read_buf(&mut buffer) {
-                Ok(0) => break,
-                // We read some bytes but maybe not enough
-                Ok(_) => continue,
-                // The readiness event is a false positive.
-                Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => continue,
-                Err(e) => return Err(e.into()),
-            }
-        }
-
-        if buffer.is_empty() {
-            // Break the loop if the buffer is
-            // empty after we tried to read bytes
-            break;
-        }
-
-        let body = buffer.freeze();
-        tracing::trace!("Sending part {part_number}");
-        let task = tokio::spawn({
-            let client = client.clone();
-            let body = body.clone();
-            backoff::future::retry(retry_backoff.clone(), move || {
-                let client = client.clone();
-                let url = url.clone();
-                let body = body.clone();
-                async move {
-                    match client.put(url).body(body).send().await {
-                        Ok(resp) if resp.status().is_client_error() => {
-                            resp.error_for_status().map_err(backoff::Error::Permanent)
-                        }
-                        Ok(resp) => Ok(resp),
-                        Err(e) => Err(backoff::Error::transient(e)),
-                    }
-                }
-            })
-        });
-        in_flight.push_back((task, body));
-    }
-
-    for (handle, _buffer) in in_flight {
-        let resp = join_and_map_error(handle).await?;
-        extract_and_append_etag(&bump, &mut etags, resp.headers())?;
-    }
-
-    tracing::debug!("Finalizing the multipart upload");
-
-    let action = bucket.complete_multipart_upload(
-        Some(&credential),
-        &object,
-        multipart.upload_id(),
-        etags.iter().map(AsRef::as_ref),
-    );
-    let url = action.sign(s3_signature_duration);
-    let body = action.body();
-    let resp = backoff::future::retry(retry_backoff, move || {
-        let client = client.clone();
-        let url = url.clone();
-        let body = body.clone();
-        async move {
-            match client.post(url).body(body).send().await {
-                Ok(resp) if resp.status().is_client_error() => {
-                    resp.error_for_status().map_err(backoff::Error::Permanent)
-                }
-                Ok(resp) => Ok(resp),
-                Err(e) => Err(backoff::Error::transient(e)),
-            }
-        }
-    })
-    .await
-    .map_err(Error::S3HttpError)?;
-
-    let status = resp.status();
-    let body = resp.text().await.map_err(|e| Error::S3Error { status, body: e.to_string() })?;
-    if status.is_success() {
-        Ok(())
-    } else {
-        Err(Error::S3Error { status, body })
-    }
-}
-
-#[cfg(unix)]
-async fn join_and_map_error(
-    join_handle: tokio::task::JoinHandle<Result<reqwest::Response, reqwest::Error>>,
-) -> Result<reqwest::Response> {
-    // safety: Panic happens if the task (JoinHandle) was aborted, cancelled, or panicked
-    let request = join_handle.await.unwrap();
-    let resp = request.map_err(Error::S3HttpError)?;
-    match resp.error_for_status_ref() {
-        Ok(_) => Ok(resp),
-        Err(_) => Err(Error::S3Error {
-            status: resp.status(),
-            body: resp.text().await.unwrap_or_default(),
-        }),
-    }
-}
-
-#[cfg(unix)]
-fn extract_and_append_etag<'b>(
-    bump: &'b bumpalo::Bump,
-    etags: &mut Vec<&'b str>,
-    headers: &reqwest::header::HeaderMap,
-) -> Result<()> {
-    use reqwest::header::ETAG;
-
-    let etag = headers.get(ETAG).ok_or_else(|| Error::S3XmlError("Missing ETag header".into()))?;
-    let etag = etag.to_str().map_err(|e| Error::S3XmlError(Box::new(e)))?;
-    etags.push(bump.alloc_str(etag));
-
-    Ok(())
 }

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 25, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 24, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
 3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
@@ -57,7 +57,7 @@ girafo: { number_of_documents: 0, field_distribution: {} }
 [timestamp] [4,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.25.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.24.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", }
 2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", }
 3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", }

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 25, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 24, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,]

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 25, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 24, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 ----------------------------------------------------------------------
 ### Status:

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 25, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 24, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 ----------------------------------------------------------------------
 ### Status:
@@ -37,7 +37,7 @@ catto [1,]
 [timestamp] [0,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.25.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.24.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 25, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 24, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
 ----------------------------------------------------------------------
@@ -40,7 +40,7 @@ doggo [2,]
 [timestamp] [0,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.25.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.24.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 25, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 24, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
 3 {uid: 3, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
@@ -43,7 +43,7 @@ doggo [2,3,]
 [timestamp] [0,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.25.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.24.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]

crates/index-scheduler/src/test_utils.rs

@@ -126,7 +126,7 @@ impl IndexScheduler {
         std::fs::create_dir_all(&options.auth_path).unwrap();
         let auth_env = open_auth_store_env(&options.auth_path).unwrap();
         let index_scheduler =
-            Self::new_test(options, auth_env, version, None, sender, planned_failures).unwrap();
+            Self::new(options, auth_env, version, sender, planned_failures).unwrap();
 
         // To be 100% consistent between all test we're going to start the scheduler right now
         // and ensure it's in the expected starting state.

crates/meilisearch-types/src/error.rs

@@ -258,7 +258,6 @@ InvalidIndexUid                                , InvalidRequest       , BAD_REQU
 InvalidMultiSearchFacets                       , InvalidRequest       , BAD_REQUEST ;
 InvalidMultiSearchFacetsByIndex                , InvalidRequest       , BAD_REQUEST ;
 InvalidMultiSearchFacetOrder                   , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchQueryPersonalization         , InvalidRequest       , BAD_REQUEST ;
 InvalidMultiSearchFederated                    , InvalidRequest       , BAD_REQUEST ;
 InvalidMultiSearchFederationOptions            , InvalidRequest       , BAD_REQUEST ;
 InvalidMultiSearchMaxValuesPerFacet            , InvalidRequest       , BAD_REQUEST ;
@@ -316,8 +315,6 @@ InvalidSearchShowRankingScoreDetails           , InvalidRequest       , BAD_REQU
 InvalidSimilarShowRankingScoreDetails          , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchSort                              , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchDistinct                          , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchPersonalize                       , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchPersonalizeUserContext            , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchMediaAndVector                    , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsDisplayedAttributes             , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsDistinctAttribute               , InvalidRequest       , BAD_REQUEST ;
@@ -393,9 +390,6 @@ TooManyVectors                                 , InvalidRequest       , BAD_REQU
 UnretrievableDocument                          , Internal             , BAD_REQUEST ;
 UnretrievableErrorCode                         , InvalidRequest       , BAD_REQUEST ;
 UnsupportedMediaType                           , InvalidRequest       , UNSUPPORTED_MEDIA_TYPE ;
-InvalidS3SnapshotRequest                       , Internal             , BAD_REQUEST ;
-InvalidS3SnapshotParameters                    , Internal             , BAD_REQUEST ;
-S3SnapshotServerError                          , Internal             , BAD_GATEWAY ;
 
 // Experimental features
 VectorEmbeddingError                           , InvalidRequest       , BAD_REQUEST ;
@@ -685,18 +679,6 @@ impl fmt::Display for deserr_codes::InvalidNetworkSearchApiKey {
     }
 }
 
-impl fmt::Display for deserr_codes::InvalidSearchPersonalize {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "the value of `personalize` is invalid, expected a JSON object with `userContext` string.")
-    }
-}
-
-impl fmt::Display for deserr_codes::InvalidSearchPersonalizeUserContext {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "the value of `userContext` is invalid, expected a string.")
-    }
-}
-
 #[macro_export]
 macro_rules! internal_error {
     ($target:ty : $($other:path), *) => {

crates/meilisearch-types/src/settings.rs

@@ -346,26 +346,24 @@ impl<T> Settings<T> {
                 continue;
             };
 
-            hide_secret(api_key, 0);
+            Self::hide_secret(api_key);
         }
     }
-}
 
-/// Redact a secret string, starting from the `secret_offset`th byte.
-pub fn hide_secret(secret: &mut String, secret_offset: usize) {
-    match secret.len().checked_sub(secret_offset) {
-        None => (),
-        Some(x) if x < 10 => {
-            secret.replace_range(secret_offset.., "XXX...");
-        }
-        Some(x) if x < 20 => {
-            secret.replace_range((secret_offset + 2).., "XXXX...");
-        }
-        Some(x) if x < 30 => {
-            secret.replace_range((secret_offset + 3).., "XXXXX...");
-        }
-        Some(_x) => {
-            secret.replace_range((secret_offset + 5).., "XXXXXX...");
+    fn hide_secret(secret: &mut String) {
+        match secret.len() {
+            x if x < 10 => {
+                secret.replace_range(.., "XXX...");
+            }
+            x if x < 20 => {
+                secret.replace_range(2.., "XXXX...");
+            }
+            x if x < 30 => {
+                secret.replace_range(3.., "XXXXX...");
+            }
+            _x => {
+                secret.replace_range(5.., "XXXXXX...");
+            }
         }
     }
 }

crates/meilisearch-types/src/webhooks.rs

@@ -11,24 +11,6 @@ pub struct Webhook {
     pub headers: BTreeMap<String, String>,
 }
 
-impl Webhook {
-    pub fn redact_authorization_header(&mut self) {
-        // headers are case insensitive, so to make the redaction robust we iterate over qualifying headers
-        // rather than getting one canonical `Authorization` header.
-        for value in self
-            .headers
-            .iter_mut()
-            .filter_map(|(name, value)| name.eq_ignore_ascii_case("authorization").then_some(value))
-        {
-            if value.starts_with("Bearer ") {
-                crate::settings::hide_secret(value, "Bearer ".len());
-            } else {
-                crate::settings::hide_secret(value, 0);
-            }
-        }
-    }
-}
-
 #[derive(Debug, Serialize, Default, Clone, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub struct WebhooksView {

crates/meilisearch/src/analytics/segment_analytics.rs

@@ -208,7 +208,6 @@ struct Infos {
     experimental_no_edition_2024_for_prefix_post_processing: bool,
     experimental_no_edition_2024_for_facet_post_processing: bool,
     experimental_vector_store_setting: bool,
-    experimental_personalization: bool,
     gpu_enabled: bool,
     db_path: bool,
     import_dump: bool,
@@ -218,7 +217,6 @@ struct Infos {
     import_snapshot: bool,
     schedule_snapshot: Option<u64>,
     snapshot_dir: bool,
-    uses_s3_snapshots: bool,
     ignore_missing_snapshot: bool,
     ignore_snapshot_if_db_exists: bool,
     http_addr: bool,
@@ -287,8 +285,6 @@ impl Infos {
             indexer_options,
             config_file_path,
             no_analytics: _,
-            experimental_personalization_api_key,
-            s3_snapshot_options,
         } = options;
 
         let schedule_snapshot = match schedule_snapshot {
@@ -352,7 +348,6 @@ impl Infos {
             import_snapshot: import_snapshot.is_some(),
             schedule_snapshot,
             snapshot_dir: snapshot_dir != PathBuf::from("snapshots/"),
-            uses_s3_snapshots: s3_snapshot_options.is_some(),
             ignore_missing_snapshot,
             ignore_snapshot_if_db_exists,
             http_addr: http_addr != default_http_addr(),
@@ -376,7 +371,6 @@ impl Infos {
             experimental_no_edition_2024_for_settings,
             experimental_no_edition_2024_for_prefix_post_processing,
             experimental_no_edition_2024_for_facet_post_processing,
-            experimental_personalization: experimental_personalization_api_key.is_some(),
         }
     }
 }

crates/meilisearch/src/error.rs

@@ -38,8 +38,6 @@ pub enum MeilisearchHttpError {
     PaginationInFederatedQuery(usize, &'static str),
     #[error("Inside `.queries[{0}]`: Using facet options is not allowed in federated queries.\n - Hint: remove `facets` from query #{0} or remove `federation` from the request\n - Hint: pass `federation.facetsByIndex.{1}: {2:?}` for facets in federated search")]
     FacetsInFederatedQuery(usize, String, Vec<String>),
-    #[error("Inside `.queries[{0}]`: Using `.personalize` is not allowed in federated queries.\n - Hint: remove `personalize` from query #{0} or remove `federation` from the request")]
-    PersonalizationInFederatedQuery(usize),
     #[error("Inconsistent order for values in facet `{facet}`: index `{previous_uid}` orders {previous_facet_order}, but index `{current_uid}` orders {index_facet_order}.\n - Hint: Remove `federation.mergeFacets` or change `faceting.sortFacetValuesBy` to be consistent in settings.")]
     InconsistentFacetOrder {
         facet: String,
@@ -139,9 +137,6 @@ impl ErrorCode for MeilisearchHttpError {
             MeilisearchHttpError::InconsistentFacetOrder { .. } => {
                 Code::InvalidMultiSearchFacetOrder
             }
-            MeilisearchHttpError::PersonalizationInFederatedQuery(_) => {
-                Code::InvalidMultiSearchQueryPersonalization
-            }
             MeilisearchHttpError::InconsistentOriginHeaders { .. } => {
                 Code::InconsistentDocumentChangeHeaders
             }

crates/meilisearch/src/lib.rs

@@ -11,7 +11,6 @@ pub mod middleware;
 pub mod option;
 #[cfg(test)]
 mod option_test;
-pub mod personalization;
 pub mod routes;
 pub mod search;
 pub mod search_queue;
@@ -59,7 +58,6 @@ use tracing::{error, info_span};
 use tracing_subscriber::filter::Targets;
 
 use crate::error::MeilisearchHttpError;
-use crate::personalization::PersonalizationService;
 
 /// Default number of simultaneously opened indexes.
 ///
@@ -130,8 +128,12 @@ pub type LogStderrType = tracing_subscriber::filter::Filtered<
 >;
 
 pub fn create_app(
-    services: ServicesData,
+    index_scheduler: Data<IndexScheduler>,
+    auth_controller: Data<AuthController>,
+    search_queue: Data<SearchQueue>,
     opt: Opt,
+    logs: (LogRouteHandle, LogStderrHandle),
+    analytics: Data<Analytics>,
     enable_dashboard: bool,
 ) -> actix_web::App<
     impl ServiceFactory<
@@ -143,7 +145,17 @@ pub fn create_app(
     >,
 > {
     let app = actix_web::App::new()
-        .configure(|s| configure_data(s, services, &opt))
+        .configure(|s| {
+            configure_data(
+                s,
+                index_scheduler.clone(),
+                auth_controller.clone(),
+                search_queue.clone(),
+                &opt,
+                logs,
+                analytics.clone(),
+            )
+        })
         .configure(routes::configure)
         .configure(|s| dashboard(s, enable_dashboard));
 
@@ -204,10 +216,7 @@ enum OnFailure {
     KeepDb,
 }
 
-pub fn setup_meilisearch(
-    opt: &Opt,
-    handle: tokio::runtime::Handle,
-) -> anyhow::Result<(Arc<IndexScheduler>, Arc<AuthController>)> {
+pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc<IndexScheduler>, Arc<AuthController>)> {
     let index_scheduler_opt = IndexSchedulerOptions {
         version_file_path: opt.db_path.join(VERSION_FILE_NAME),
         auth_path: opt.db_path.join("auth"),
@@ -221,11 +230,7 @@ pub fn setup_meilisearch(
         task_db_size: opt.max_task_db_size.as_u64() as usize,
         index_base_map_size: opt.max_index_size.as_u64() as usize,
         enable_mdb_writemap: opt.experimental_reduce_indexing_memory_usage,
-        indexer_config: Arc::new({
-            let s3_snapshot_options =
-                opt.s3_snapshot_options.clone().map(|opt| opt.try_into()).transpose()?;
-            IndexerConfig { s3_snapshot_options, ..(&opt.indexer_options).try_into()? }
-        }),
+        indexer_config: Arc::new((&opt.indexer_options).try_into()?),
         autobatching_enabled: true,
         cleanup_enabled: !opt.experimental_replication_parameters,
         max_number_of_tasks: 1_000_000,
@@ -251,7 +256,6 @@ pub fn setup_meilisearch(
                     index_scheduler_opt,
                     OnFailure::RemoveDb,
                     binary_version, // the db is empty
-                    handle,
                 )?,
                 Err(e) => {
                     std::fs::remove_dir_all(&opt.db_path)?;
@@ -269,7 +273,7 @@ pub fn setup_meilisearch(
             bail!("snapshot doesn't exist at {}", snapshot_path.display())
         // the snapshot and the db exist, and we can ignore the snapshot because of the ignore_snapshot_if_db_exists flag
         } else {
-            open_or_create_database(opt, index_scheduler_opt, empty_db, binary_version, handle)?
+            open_or_create_database(opt, index_scheduler_opt, empty_db, binary_version)?
         }
     } else if let Some(ref path) = opt.import_dump {
         let src_path_exists = path.exists();
@@ -280,7 +284,6 @@ pub fn setup_meilisearch(
                 index_scheduler_opt,
                 OnFailure::RemoveDb,
                 binary_version, // the db is empty
-                handle,
             )?;
             match import_dump(&opt.db_path, path, &mut index_scheduler, &mut auth_controller) {
                 Ok(()) => (index_scheduler, auth_controller),
@@ -301,10 +304,10 @@ pub fn setup_meilisearch(
         // the dump and the db exist and we can ignore the dump because of the ignore_dump_if_db_exists flag
         // or, the dump is missing but we can ignore that because of the ignore_missing_dump flag
         } else {
-            open_or_create_database(opt, index_scheduler_opt, empty_db, binary_version, handle)?
+            open_or_create_database(opt, index_scheduler_opt, empty_db, binary_version)?
         }
     } else {
-        open_or_create_database(opt, index_scheduler_opt, empty_db, binary_version, handle)?
+        open_or_create_database(opt, index_scheduler_opt, empty_db, binary_version)?
     };
 
     // We create a loop in a thread that registers snapshotCreation tasks
@@ -335,7 +338,6 @@ fn open_or_create_database_unchecked(
     index_scheduler_opt: IndexSchedulerOptions,
     on_failure: OnFailure,
     version: (u32, u32, u32),
-    handle: tokio::runtime::Handle,
 ) -> anyhow::Result<(IndexScheduler, AuthController)> {
     // we don't want to create anything in the data.ms yet, thus we
     // wrap our two builders in a closure that'll be executed later.
@@ -343,7 +345,7 @@ fn open_or_create_database_unchecked(
     let auth_env = open_auth_store_env(&index_scheduler_opt.auth_path).unwrap();
     let auth_controller = AuthController::new(auth_env.clone(), &opt.master_key);
     let index_scheduler_builder = || -> anyhow::Result<_> {
-        Ok(IndexScheduler::new(index_scheduler_opt, auth_env, version, Some(handle))?)
+        Ok(IndexScheduler::new(index_scheduler_opt, auth_env, version)?)
     };
 
     match (
@@ -450,7 +452,6 @@ fn open_or_create_database(
     index_scheduler_opt: IndexSchedulerOptions,
     empty_db: bool,
     binary_version: (u32, u32, u32),
-    handle: tokio::runtime::Handle,
 ) -> anyhow::Result<(IndexScheduler, AuthController)> {
     let version = if !empty_db {
         check_version(opt, &index_scheduler_opt, binary_version)?
@@ -458,7 +459,7 @@ fn open_or_create_database(
         binary_version
     };
 
-    open_or_create_database_unchecked(opt, index_scheduler_opt, OnFailure::KeepDb, version, handle)
+    open_or_create_database_unchecked(opt, index_scheduler_opt, OnFailure::KeepDb, version)
 }
 
 fn import_dump(
@@ -526,11 +527,7 @@ fn import_dump(
     let indexer_config = if base_config.max_threads.is_none() {
         let (thread_pool, _) = default_thread_pool_and_threads();
 
-        let _config = IndexerConfig {
-            thread_pool,
-            s3_snapshot_options: base_config.s3_snapshot_options.clone(),
-            ..*base_config
-        };
+        let _config = IndexerConfig { thread_pool, ..*base_config };
         backup_config = _config;
         &backup_config
     } else {
@@ -678,26 +675,23 @@ fn import_dump(
     Ok(index_scheduler_dump.finish()?)
 }
 
-pub fn configure_data(config: &mut web::ServiceConfig, services: ServicesData, opt: &Opt) {
-    let ServicesData {
-        index_scheduler,
-        auth,
-        search_queue,
-        personalization_service,
-        logs_route_handle,
-        logs_stderr_handle,
-        analytics,
-    } = services;
-
+pub fn configure_data(
+    config: &mut web::ServiceConfig,
+    index_scheduler: Data<IndexScheduler>,
+    auth: Data<AuthController>,
+    search_queue: Data<SearchQueue>,
+    opt: &Opt,
+    (logs_route, logs_stderr): (LogRouteHandle, LogStderrHandle),
+    analytics: Data<Analytics>,
+) {
     let http_payload_size_limit = opt.http_payload_size_limit.as_u64() as usize;
     config
         .app_data(index_scheduler)
         .app_data(auth)
         .app_data(search_queue)
         .app_data(analytics)
-        .app_data(personalization_service)
-        .app_data(logs_route_handle)
-        .app_data(logs_stderr_handle)
+        .app_data(web::Data::new(logs_route))
+        .app_data(web::Data::new(logs_stderr))
         .app_data(web::Data::new(opt.clone()))
         .app_data(
             web::JsonConfig::default()
@@ -758,14 +752,3 @@ pub fn dashboard(config: &mut web::ServiceConfig, enable_frontend: bool) {
 pub fn dashboard(config: &mut web::ServiceConfig, _enable_frontend: bool) {
     config.service(web::resource("/").route(web::get().to(routes::running)));
 }
-
-#[derive(Clone)]
-pub struct ServicesData {
-    pub index_scheduler: Data<IndexScheduler>,
-    pub auth: Data<AuthController>,
-    pub search_queue: Data<SearchQueue>,
-    pub personalization_service: Data<PersonalizationService>,
-    pub logs_route_handle: Data<LogRouteHandle>,
-    pub logs_stderr_handle: Data<LogStderrHandle>,
-    pub analytics: Data<Analytics>,
-}

crates/meilisearch/src/main.rs

@@ -14,11 +14,10 @@ use index_scheduler::IndexScheduler;
 use is_terminal::IsTerminal;
 use meilisearch::analytics::Analytics;
 use meilisearch::option::LogMode;
-use meilisearch::personalization::PersonalizationService;
 use meilisearch::search_queue::SearchQueue;
 use meilisearch::{
     analytics, create_app, setup_meilisearch, LogRouteHandle, LogRouteType, LogStderrHandle,
-    LogStderrType, Opt, ServicesData, SubscriberForSecondLayer,
+    LogStderrType, Opt, SubscriberForSecondLayer,
 };
 use meilisearch_auth::{generate_master_key, AuthController, MASTER_KEY_MIN_SIZE};
 use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
@@ -77,10 +76,7 @@ fn on_panic(info: &std::panic::PanicHookInfo) {
 
 #[actix_web::main]
 async fn main() -> anyhow::Result<()> {
-    // won't panic inside of tokio::main
-    let runtime = tokio::runtime::Handle::current();
-
-    try_main(runtime).await.inspect_err(|error| {
+    try_main().await.inspect_err(|error| {
         tracing::error!(%error);
         let mut current = error.source();
         let mut depth = 0;
@@ -92,7 +88,7 @@ async fn main() -> anyhow::Result<()> {
     })
 }
 
-async fn try_main(runtime: tokio::runtime::Handle) -> anyhow::Result<()> {
+async fn try_main() -> anyhow::Result<()> {
     let (opt, config_read_from) = Opt::try_build()?;
 
     std::panic::set_hook(Box::new(on_panic));
@@ -126,7 +122,7 @@ async fn try_main(runtime: tokio::runtime::Handle) -> anyhow::Result<()> {
         _ => (),
     }
 
-    let (index_scheduler, auth_controller) = setup_meilisearch(&opt, runtime)?;
+    let (index_scheduler, auth_controller) = setup_meilisearch(&opt)?;
 
     let analytics =
         analytics::Analytics::new(&opt, index_scheduler.clone(), auth_controller.clone()).await;
@@ -153,15 +149,8 @@ async fn run_http(
     let enable_dashboard = &opt.env == "development";
     let opt_clone = opt.clone();
     let index_scheduler = Data::from(index_scheduler);
-    let auth = Data::from(auth_controller);
+    let auth_controller = Data::from(auth_controller);
     let analytics = Data::from(analytics);
-    // Create personalization service with API key from options
-    let personalization_service = Data::new(
-        opt.experimental_personalization_api_key
-            .clone()
-            .map(PersonalizationService::cohere)
-            .unwrap_or_else(PersonalizationService::disabled),
-    );
     let search_queue = SearchQueue::new(
         opt.experimental_search_queue_size,
         available_parallelism()
@@ -173,25 +162,21 @@ async fn run_http(
         usize::from(opt.experimental_drop_search_after) as u64
     ));
     let search_queue = Data::new(search_queue);
-    let (logs_route_handle, logs_stderr_handle) = logs;
-    let logs_route_handle = Data::new(logs_route_handle);
-    let logs_stderr_handle = Data::new(logs_stderr_handle);
 
-    let services = ServicesData {
-        index_scheduler,
-        auth,
-        search_queue,
-        personalization_service,
-        logs_route_handle,
-        logs_stderr_handle,
-        analytics,
-    };
-
-    let http_server =
-        HttpServer::new(move || create_app(services.clone(), opt.clone(), enable_dashboard))
-            // Disable signals allows the server to terminate immediately when a user enter CTRL-C
-            .disable_signals()
-            .keep_alive(KeepAlive::Os);
+    let http_server = HttpServer::new(move || {
+        create_app(
+            index_scheduler.clone(),
+            auth_controller.clone(),
+            search_queue.clone(),
+            opt.clone(),
+            logs.clone(),
+            analytics.clone(),
+            enable_dashboard,
+        )
+    })
+    // Disable signals allows the server to terminate immediately when a user enter CTRL-C
+    .disable_signals()
+    .keep_alive(KeepAlive::Os);
 
     if let Some(config) = opt_clone.get_ssl_config()? {
         http_server.bind_rustls_0_23(opt_clone.http_addr, config)?.run().await?;

crates/meilisearch/src/metrics.rs

@@ -114,9 +114,4 @@ lazy_static! {
             "Meilisearch Task Queue Size Until Stop Registering",
         ))
         .expect("Can't create a metric");
-    pub static ref MEILISEARCH_PERSONALIZED_SEARCH_REQUESTS: IntGauge = register_int_gauge!(opts!(
-        "meilisearch_personalized_search_requests",
-        "Meilisearch number of search requests with personalization"
-    ))
-    .expect("Can't create a metric");
 }

crates/meilisearch/src/option.rs

@@ -7,13 +7,12 @@ use std::ops::Deref;
 use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::Arc;
-use std::time::Duration;
 use std::{env, fmt, fs};
 
 use byte_unit::{Byte, ParseError, UnitType};
 use clap::Parser;
 use meilisearch_types::features::InstanceTogglableFeatures;
-use meilisearch_types::milli::update::{IndexerConfig, S3SnapshotOptions};
+use meilisearch_types::milli::update::IndexerConfig;
 use meilisearch_types::milli::ThreadPoolNoAbortBuilder;
 use rustls::server::{ServerSessionMemoryCache, WebPkiClientVerifier};
 use rustls::RootCertStore;
@@ -75,22 +74,6 @@ const MEILI_EXPERIMENTAL_EMBEDDING_CACHE_ENTRIES: &str =
 const MEILI_EXPERIMENTAL_NO_SNAPSHOT_COMPACTION: &str = "MEILI_EXPERIMENTAL_NO_SNAPSHOT_COMPACTION";
 const MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_DUMPS: &str =
     "MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_DUMPS";
-const MEILI_EXPERIMENTAL_PERSONALIZATION_API_KEY: &str =
-    "MEILI_EXPERIMENTAL_PERSONALIZATION_API_KEY";
-
-// Related to S3 snapshots
-const MEILI_S3_BUCKET_URL: &str = "MEILI_S3_BUCKET_URL";
-const MEILI_S3_BUCKET_REGION: &str = "MEILI_S3_BUCKET_REGION";
-const MEILI_S3_BUCKET_NAME: &str = "MEILI_S3_BUCKET_NAME";
-const MEILI_S3_SNAPSHOT_PREFIX: &str = "MEILI_S3_SNAPSHOT_PREFIX";
-const MEILI_S3_ACCESS_KEY: &str = "MEILI_S3_ACCESS_KEY";
-const MEILI_S3_SECRET_KEY: &str = "MEILI_S3_SECRET_KEY";
-const MEILI_EXPERIMENTAL_S3_MAX_IN_FLIGHT_PARTS: &str = "MEILI_EXPERIMENTAL_S3_MAX_IN_FLIGHT_PARTS";
-const MEILI_EXPERIMENTAL_S3_COMPRESSION_LEVEL: &str = "MEILI_EXPERIMENTAL_S3_COMPRESSION_LEVEL";
-const MEILI_EXPERIMENTAL_S3_SIGNATURE_DURATION_SECONDS: &str =
-    "MEILI_EXPERIMENTAL_S3_SIGNATURE_DURATION_SECONDS";
-const MEILI_EXPERIMENTAL_S3_MULTIPART_PART_SIZE: &str = "MEILI_EXPERIMENTAL_S3_MULTIPART_PART_SIZE";
-
 const DEFAULT_CONFIG_FILE_PATH: &str = "./config.toml";
 const DEFAULT_DB_PATH: &str = "./data.ms";
 const DEFAULT_HTTP_ADDR: &str = "localhost:7700";
@@ -100,10 +83,6 @@ const DEFAULT_SNAPSHOT_DIR: &str = "snapshots/";
 const DEFAULT_SNAPSHOT_INTERVAL_SEC: u64 = 86400;
 const DEFAULT_SNAPSHOT_INTERVAL_SEC_STR: &str = "86400";
 const DEFAULT_DUMP_DIR: &str = "dumps/";
-const DEFAULT_S3_SNAPSHOT_MAX_IN_FLIGHT_PARTS: NonZeroUsize = NonZeroUsize::new(10).unwrap();
-const DEFAULT_S3_SNAPSHOT_COMPRESSION_LEVEL: u32 = 0;
-const DEFAULT_S3_SNAPSHOT_SIGNATURE_DURATION_SECONDS: u64 = 8 * 3600; // 8 hours
-const DEFAULT_S3_SNAPSHOT_MULTIPART_PART_SIZE: Byte = Byte::from_u64(375 * 1024 * 1024); // 375 MiB
 
 const MEILI_MAX_INDEXING_MEMORY: &str = "MEILI_MAX_INDEXING_MEMORY";
 const MEILI_MAX_INDEXING_THREADS: &str = "MEILI_MAX_INDEXING_THREADS";
@@ -496,20 +475,10 @@ pub struct Opt {
     #[serde(default)]
     pub experimental_no_snapshot_compaction: bool,
 
-    /// Experimental personalization API key feature.
-    ///
-    /// Sets the API key for personalization features.
-    #[clap(long, env = MEILI_EXPERIMENTAL_PERSONALIZATION_API_KEY)]
-    pub experimental_personalization_api_key: Option<String>,
-
     #[serde(flatten)]
     #[clap(flatten)]
     pub indexer_options: IndexerOpts,
 
-    #[serde(flatten)]
-    #[clap(flatten)]
-    pub s3_snapshot_options: Option<S3SnapshotOpts>,
-
     /// Set the path to a configuration file that should be used to setup the engine.
     /// Format must be TOML.
     #[clap(long)]
@@ -611,8 +580,6 @@ impl Opt {
             experimental_limit_batched_tasks_total_size,
             experimental_embedding_cache_entries,
             experimental_no_snapshot_compaction,
-            experimental_personalization_api_key,
-            s3_snapshot_options,
         } = self;
         export_to_env_if_not_present(MEILI_DB_PATH, db_path);
         export_to_env_if_not_present(MEILI_HTTP_ADDR, http_addr);
@@ -713,22 +680,7 @@ impl Opt {
             MEILI_EXPERIMENTAL_NO_SNAPSHOT_COMPACTION,
             experimental_no_snapshot_compaction.to_string(),
         );
-        if let Some(experimental_personalization_api_key) = experimental_personalization_api_key {
-            export_to_env_if_not_present(
-                MEILI_EXPERIMENTAL_PERSONALIZATION_API_KEY,
-                experimental_personalization_api_key,
-            );
-        }
         indexer_options.export_to_env();
-        if let Some(s3_snapshot_options) = s3_snapshot_options {
-            #[cfg(not(unix))]
-            {
-                let _ = s3_snapshot_options;
-                panic!("S3 snapshot options are not supported on Windows");
-            }
-            #[cfg(unix)]
-            s3_snapshot_options.export_to_env();
-        }
     }
 
     pub fn get_ssl_config(&self) -> anyhow::Result<Option<rustls::ServerConfig>> {
@@ -897,16 +849,6 @@ impl TryFrom<&IndexerOpts> for IndexerConfig {
     type Error = anyhow::Error;
 
     fn try_from(other: &IndexerOpts) -> Result<Self, Self::Error> {
-        let IndexerOpts {
-            max_indexing_memory,
-            max_indexing_threads,
-            skip_index_budget,
-            experimental_no_edition_2024_for_settings,
-            experimental_no_edition_2024_for_dumps,
-            experimental_no_edition_2024_for_prefix_post_processing,
-            experimental_no_edition_2024_for_facet_post_processing,
-        } = other;
-
         let thread_pool = ThreadPoolNoAbortBuilder::new_for_indexing()
             .num_threads(other.max_indexing_threads.unwrap_or_else(|| num_cpus::get() / 2))
             .build()?;
@@ -914,163 +856,21 @@ impl TryFrom<&IndexerOpts> for IndexerConfig {
         Ok(Self {
             thread_pool,
             log_every_n: Some(DEFAULT_LOG_EVERY_N),
-            max_memory: max_indexing_memory.map(|b| b.as_u64() as usize),
-            max_threads: max_indexing_threads.0,
+            max_memory: other.max_indexing_memory.map(|b| b.as_u64() as usize),
+            max_threads: *other.max_indexing_threads,
             max_positions_per_attributes: None,
-            skip_index_budget: *skip_index_budget,
-            experimental_no_edition_2024_for_settings: *experimental_no_edition_2024_for_settings,
-            experimental_no_edition_2024_for_dumps: *experimental_no_edition_2024_for_dumps,
+            skip_index_budget: other.skip_index_budget,
+            experimental_no_edition_2024_for_settings: other
+                .experimental_no_edition_2024_for_settings,
+            experimental_no_edition_2024_for_dumps: other.experimental_no_edition_2024_for_dumps,
             chunk_compression_type: Default::default(),
             chunk_compression_level: Default::default(),
             documents_chunk_size: Default::default(),
             max_nb_chunks: Default::default(),
-            experimental_no_edition_2024_for_prefix_post_processing:
-                *experimental_no_edition_2024_for_prefix_post_processing,
-            experimental_no_edition_2024_for_facet_post_processing:
-                *experimental_no_edition_2024_for_facet_post_processing,
-            s3_snapshot_options: None,
-        })
-    }
-}
-
-#[derive(Debug, Clone, Parser, Deserialize)]
-// This group is a bit tricky but makes it possible to require all listed fields if one of them
-// is specified. It lets us keep an Option for the S3SnapshotOpts configuration.
-// <https://github.com/clap-rs/clap/issues/5092#issuecomment-2616986075>
-#[group(requires_all = ["s3_bucket_url", "s3_bucket_region", "s3_bucket_name", "s3_snapshot_prefix", "s3_access_key", "s3_secret_key"])]
-pub struct S3SnapshotOpts {
-    /// The S3 bucket URL in the format https://s3.<region>.amazonaws.com.
-    #[clap(long, env = MEILI_S3_BUCKET_URL, required = false)]
-    #[serde(default)]
-    pub s3_bucket_url: String,
-
-    /// The region in the format us-east-1.
-    #[clap(long, env = MEILI_S3_BUCKET_REGION, required = false)]
-    #[serde(default)]
-    pub s3_bucket_region: String,
-
-    /// The bucket name.
-    #[clap(long, env = MEILI_S3_BUCKET_NAME, required = false)]
-    #[serde(default)]
-    pub s3_bucket_name: String,
-
-    /// The prefix path where to put the snapshot, uses normal slashes (/).
-    #[clap(long, env = MEILI_S3_SNAPSHOT_PREFIX, required = false)]
-    #[serde(default)]
-    pub s3_snapshot_prefix: String,
-
-    /// The S3 access key.
-    #[clap(long, env = MEILI_S3_ACCESS_KEY, required = false)]
-    #[serde(default)]
-    pub s3_access_key: String,
-
-    /// The S3 secret key.
-    #[clap(long, env = MEILI_S3_SECRET_KEY, required = false)]
-    #[serde(default)]
-    pub s3_secret_key: String,
-
-    /// The maximum number of parts that can be uploaded in parallel.
-    ///
-    /// For more information, see <https://github.com/orgs/meilisearch/discussions/869>.
-    #[clap(long, env = MEILI_EXPERIMENTAL_S3_MAX_IN_FLIGHT_PARTS, default_value_t = default_experimental_s3_snapshot_max_in_flight_parts())]
-    #[serde(default = "default_experimental_s3_snapshot_max_in_flight_parts")]
-    pub experimental_s3_max_in_flight_parts: NonZeroUsize,
-
-    /// The compression level. Defaults to no compression (0).
-    ///
-    /// For more information, see <https://github.com/orgs/meilisearch/discussions/869>.
-    #[clap(long, env = MEILI_EXPERIMENTAL_S3_COMPRESSION_LEVEL, default_value_t = default_experimental_s3_snapshot_compression_level())]
-    #[serde(default = "default_experimental_s3_snapshot_compression_level")]
-    pub experimental_s3_compression_level: u32,
-
-    /// The signature duration for the multipart upload.
-    ///
-    /// For more information, see <https://github.com/orgs/meilisearch/discussions/869>.
-    #[clap(long, env = MEILI_EXPERIMENTAL_S3_SIGNATURE_DURATION_SECONDS, default_value_t = default_experimental_s3_snapshot_signature_duration_seconds())]
-    #[serde(default = "default_experimental_s3_snapshot_signature_duration_seconds")]
-    pub experimental_s3_signature_duration_seconds: u64,
-
-    /// The size of the the multipart parts.
-    ///
-    /// Must not be less than 10MiB and larger than 8GiB. Yes,
-    /// twice the boundaries of the AWS S3 multipart upload
-    /// because we use it a bit differently internally.
-    ///
-    /// For more information, see <https://github.com/orgs/meilisearch/discussions/869>.
-    #[clap(long, env = MEILI_EXPERIMENTAL_S3_MULTIPART_PART_SIZE, default_value_t = default_experimental_s3_snapshot_multipart_part_size())]
-    #[serde(default = "default_experimental_s3_snapshot_multipart_part_size")]
-    pub experimental_s3_multipart_part_size: Byte,
-}
-
-impl S3SnapshotOpts {
-    /// Exports the values to their corresponding env vars if they are not set.
-    pub fn export_to_env(self) {
-        let S3SnapshotOpts {
-            s3_bucket_url,
-            s3_bucket_region,
-            s3_bucket_name,
-            s3_snapshot_prefix,
-            s3_access_key,
-            s3_secret_key,
-            experimental_s3_max_in_flight_parts,
-            experimental_s3_compression_level,
-            experimental_s3_signature_duration_seconds,
-            experimental_s3_multipart_part_size,
-        } = self;
-
-        export_to_env_if_not_present(MEILI_S3_BUCKET_URL, s3_bucket_url);
-        export_to_env_if_not_present(MEILI_S3_BUCKET_REGION, s3_bucket_region);
-        export_to_env_if_not_present(MEILI_S3_BUCKET_NAME, s3_bucket_name);
-        export_to_env_if_not_present(MEILI_S3_SNAPSHOT_PREFIX, s3_snapshot_prefix);
-        export_to_env_if_not_present(MEILI_S3_ACCESS_KEY, s3_access_key);
-        export_to_env_if_not_present(MEILI_S3_SECRET_KEY, s3_secret_key);
-        export_to_env_if_not_present(
-            MEILI_EXPERIMENTAL_S3_MAX_IN_FLIGHT_PARTS,
-            experimental_s3_max_in_flight_parts.to_string(),
-        );
-        export_to_env_if_not_present(
-            MEILI_EXPERIMENTAL_S3_COMPRESSION_LEVEL,
-            experimental_s3_compression_level.to_string(),
-        );
-        export_to_env_if_not_present(
-            MEILI_EXPERIMENTAL_S3_SIGNATURE_DURATION_SECONDS,
-            experimental_s3_signature_duration_seconds.to_string(),
-        );
-        export_to_env_if_not_present(
-            MEILI_EXPERIMENTAL_S3_MULTIPART_PART_SIZE,
-            experimental_s3_multipart_part_size.to_string(),
-        );
-    }
-}
-
-impl TryFrom<S3SnapshotOpts> for S3SnapshotOptions {
-    type Error = anyhow::Error;
-
-    fn try_from(other: S3SnapshotOpts) -> Result<Self, Self::Error> {
-        let S3SnapshotOpts {
-            s3_bucket_url,
-            s3_bucket_region,
-            s3_bucket_name,
-            s3_snapshot_prefix,
-            s3_access_key,
-            s3_secret_key,
-            experimental_s3_max_in_flight_parts,
-            experimental_s3_compression_level,
-            experimental_s3_signature_duration_seconds,
-            experimental_s3_multipart_part_size,
-        } = other;
-
-        Ok(S3SnapshotOptions {
-            s3_bucket_url,
-            s3_bucket_region,
-            s3_bucket_name,
-            s3_snapshot_prefix,
-            s3_access_key,
-            s3_secret_key,
-            s3_max_in_flight_parts: experimental_s3_max_in_flight_parts,
-            s3_compression_level: experimental_s3_compression_level,
-            s3_signature_duration: Duration::from_secs(experimental_s3_signature_duration_seconds),
-            s3_multipart_part_size: experimental_s3_multipart_part_size.as_u64(),
+            experimental_no_edition_2024_for_prefix_post_processing: other
+                .experimental_no_edition_2024_for_prefix_post_processing,
+            experimental_no_edition_2024_for_facet_post_processing: other
+                .experimental_no_edition_2024_for_facet_post_processing,
         })
     }
 }
@@ -1289,22 +1089,6 @@ fn default_snapshot_interval_sec() -> &'static str {
     DEFAULT_SNAPSHOT_INTERVAL_SEC_STR
 }
 
-fn default_experimental_s3_snapshot_max_in_flight_parts() -> NonZeroUsize {
-    DEFAULT_S3_SNAPSHOT_MAX_IN_FLIGHT_PARTS
-}
-
-fn default_experimental_s3_snapshot_compression_level() -> u32 {
-    DEFAULT_S3_SNAPSHOT_COMPRESSION_LEVEL
-}
-
-fn default_experimental_s3_snapshot_signature_duration_seconds() -> u64 {
-    DEFAULT_S3_SNAPSHOT_SIGNATURE_DURATION_SECONDS
-}
-
-fn default_experimental_s3_snapshot_multipart_part_size() -> Byte {
-    DEFAULT_S3_SNAPSHOT_MULTIPART_PART_SIZE
-}
-
 fn default_dump_dir() -> PathBuf {
     PathBuf::from(DEFAULT_DUMP_DIR)
 }

crates/meilisearch/src/personalization/mod.rs

@@ -1,366 +0,0 @@
-use crate::search::{Personalize, SearchResult};
-use meilisearch_types::{
-    error::{Code, ErrorCode, ResponseError},
-    milli::TimeBudget,
-};
-use rand::Rng;
-use reqwest::Client;
-use serde::{Deserialize, Serialize};
-use std::time::Duration;
-use tracing::{debug, info, warn};
-
-const COHERE_API_URL: &str = "https://api.cohere.ai/v1/rerank";
-const MAX_RETRIES: u32 = 10;
-
-#[derive(Debug, thiserror::Error)]
-enum PersonalizationError {
-    #[error("Personalization service: HTTP request failed: {0}")]
-    Request(#[from] reqwest::Error),
-    #[error("Personalization service: Failed to parse response: {0}")]
-    Parse(String),
-    #[error("Personalization service: Cohere API error: {0}")]
-    Api(String),
-    #[error("Personalization service: Unauthorized: invalid API key")]
-    Unauthorized,
-    #[error("Personalization service: Rate limited: too many requests")]
-    RateLimited,
-    #[error("Personalization service: Bad request: {0}")]
-    BadRequest(String),
-    #[error("Personalization service: Internal server error: {0}")]
-    InternalServerError(String),
-    #[error("Personalization service: Network error: {0}")]
-    Network(String),
-    #[error("Personalization service: Deadline exceeded")]
-    DeadlineExceeded,
-    #[error(transparent)]
-    FeatureNotEnabled(#[from] index_scheduler::error::FeatureNotEnabledError),
-}
-
-impl ErrorCode for PersonalizationError {
-    fn error_code(&self) -> Code {
-        match self {
-            PersonalizationError::FeatureNotEnabled { .. } => Code::FeatureNotEnabled,
-            PersonalizationError::Unauthorized => Code::RemoteInvalidApiKey,
-            PersonalizationError::RateLimited => Code::TooManySearchRequests,
-            PersonalizationError::BadRequest(_) => Code::RemoteBadRequest,
-            PersonalizationError::InternalServerError(_) => Code::RemoteRemoteError,
-            PersonalizationError::Network(_) | PersonalizationError::Request(_) => {
-                Code::RemoteCouldNotSendRequest
-            }
-            PersonalizationError::Parse(_) | PersonalizationError::Api(_) => {
-                Code::RemoteBadResponse
-            }
-            PersonalizationError::DeadlineExceeded => Code::Internal, // should not be returned to the client
-        }
-    }
-}
-
-pub struct CohereService {
-    client: Client,
-    api_key: String,
-}
-
-impl CohereService {
-    pub fn new(api_key: String) -> Self {
-        info!("Personalization service initialized with Cohere API");
-        let client = Client::builder()
-            .timeout(Duration::from_secs(30))
-            .build()
-            .expect("Failed to create HTTP client");
-        Self { client, api_key }
-    }
-
-    pub async fn rerank_search_results(
-        &self,
-        search_result: SearchResult,
-        personalize: &Personalize,
-        query: Option<&str>,
-        time_budget: TimeBudget,
-    ) -> Result<SearchResult, ResponseError> {
-        if time_budget.exceeded() {
-            warn!("Could not rerank due to deadline");
-            // If the deadline is exceeded, return the original search result instead of an error
-            return Ok(search_result);
-        }
-
-        // Extract user context from personalization
-        let user_context = personalize.user_context.as_str();
-
-        // Build the prompt by merging query and user context
-        let prompt = match query {
-            Some(q) => format!("User Context: {user_context}\nQuery: {q}"),
-            None => format!("User Context: {user_context}"),
-        };
-
-        // Extract documents for reranking
-        let documents: Vec<String> = search_result
-            .hits
-            .iter()
-            .map(|hit| {
-                // Convert the document to a string representation for reranking
-                serde_json::to_string(&hit.document).unwrap_or_else(|_| "{}".to_string())
-            })
-            .collect();
-
-        if documents.is_empty() {
-            return Ok(search_result);
-        }
-
-        // Call Cohere's rerank API with retry logic
-        let reranked_indices =
-            match self.call_rerank_with_retry(&prompt, &documents, time_budget).await {
-                Ok(indices) => indices,
-                Err(PersonalizationError::DeadlineExceeded) => {
-                    // If the deadline is exceeded, return the original search result instead of an error
-                    return Ok(search_result);
-                }
-                Err(e) => return Err(e.into()),
-            };
-
-        debug!("Cohere rerank successful, reordering {} results", search_result.hits.len());
-
-        // Reorder the hits based on Cohere's reranking
-        let mut reranked_hits = Vec::new();
-        for index in reranked_indices.iter() {
-            if let Some(hit) = search_result.hits.get(*index) {
-                reranked_hits.push(hit.clone());
-            }
-        }
-
-        Ok(SearchResult { hits: reranked_hits, ..search_result })
-    }
-
-    async fn call_rerank_with_retry(
-        &self,
-        query: &str,
-        documents: &[String],
-        time_budget: TimeBudget,
-    ) -> Result<Vec<usize>, PersonalizationError> {
-        let request_body = CohereRerankRequest {
-            query: query.to_string(),
-            documents: documents.to_vec(),
-            model: "rerank-english-v3.0".to_string(),
-        };
-
-        // Retry loop similar to vector extraction
-        for attempt in 0..MAX_RETRIES {
-            let response_result = self.send_rerank_request(&request_body).await;
-
-            let retry_duration = match self.handle_response(response_result).await {
-                Ok(indices) => return Ok(indices),
-                Err(retry) => {
-                    warn!("Cohere rerank attempt #{} failed: {}", attempt, retry.error);
-
-                    if time_budget.exceeded() {
-                        warn!("Could not rerank due to deadline");
-                        return Err(PersonalizationError::DeadlineExceeded);
-                    } else {
-                        match retry.into_duration(attempt) {
-                            Ok(d) => d,
-                            Err(error) => return Err(error),
-                        }
-                    }
-                }
-            };
-
-            // randomly up to double the retry duration
-            let retry_duration = retry_duration
-                + rand::thread_rng().gen_range(std::time::Duration::ZERO..retry_duration);
-
-            warn!("Retrying after {}ms", retry_duration.as_millis());
-            tokio::time::sleep(retry_duration).await;
-        }
-
-        // Final attempt without retry
-        let response_result = self.send_rerank_request(&request_body).await;
-
-        match self.handle_response(response_result).await {
-            Ok(indices) => Ok(indices),
-            Err(retry) => Err(retry.into_error()),
-        }
-    }
-
-    async fn send_rerank_request(
-        &self,
-        request_body: &CohereRerankRequest,
-    ) -> Result<reqwest::Response, reqwest::Error> {
-        self.client
-            .post(COHERE_API_URL)
-            .header("Authorization", format!("Bearer {}", self.api_key))
-            .header("Content-Type", "application/json")
-            .json(request_body)
-            .send()
-            .await
-    }
-
-    async fn handle_response(
-        &self,
-        response_result: Result<reqwest::Response, reqwest::Error>,
-    ) -> Result<Vec<usize>, Retry> {
-        let response = match response_result {
-            Ok(r) => r,
-            Err(e) if e.is_timeout() => {
-                return Err(Retry::retry_later(PersonalizationError::Network(format!(
-                    "Request timeout: {}",
-                    e
-                ))));
-            }
-            Err(e) => {
-                return Err(Retry::retry_later(PersonalizationError::Network(format!(
-                    "Network error: {}",
-                    e
-                ))));
-            }
-        };
-
-        let status = response.status();
-        let status_code = status.as_u16();
-
-        if status.is_success() {
-            let rerank_response: CohereRerankResponse = match response.json().await {
-                Ok(r) => r,
-                Err(e) => {
-                    return Err(Retry::retry_later(PersonalizationError::Parse(format!(
-                        "Failed to parse response: {}",
-                        e
-                    ))));
-                }
-            };
-
-            // Extract indices from rerank results
-            let indices: Vec<usize> =
-                rerank_response.results.iter().map(|result| result.index as usize).collect();
-
-            return Ok(indices);
-        }
-
-        // Handle error status codes
-        let error_body = response.text().await.unwrap_or_else(|_| "Unknown error".to_string());
-
-        let retry = match status_code {
-            401 => Retry::give_up(PersonalizationError::Unauthorized),
-            429 => Retry::rate_limited(PersonalizationError::RateLimited),
-            400 => Retry::give_up(PersonalizationError::BadRequest(error_body)),
-            500..=599 => Retry::retry_later(PersonalizationError::InternalServerError(format!(
-                "Status {}: {}",
-                status_code, error_body
-            ))),
-            402..=499 => Retry::give_up(PersonalizationError::Api(format!(
-                "Status {}: {}",
-                status_code, error_body
-            ))),
-            _ => Retry::retry_later(PersonalizationError::Api(format!(
-                "Unexpected status {}: {}",
-                status_code, error_body
-            ))),
-        };
-
-        Err(retry)
-    }
-}
-
-#[derive(Serialize)]
-struct CohereRerankRequest {
-    query: String,
-    documents: Vec<String>,
-    model: String,
-}
-
-#[derive(Deserialize)]
-struct CohereRerankResponse {
-    results: Vec<CohereRerankResult>,
-}
-
-#[derive(Deserialize)]
-struct CohereRerankResult {
-    index: u32,
-}
-
-// Retry strategy similar to vector extraction
-struct Retry {
-    error: PersonalizationError,
-    strategy: RetryStrategy,
-}
-
-enum RetryStrategy {
-    GiveUp,
-    Retry,
-    RetryAfterRateLimit,
-}
-
-impl Retry {
-    fn give_up(error: PersonalizationError) -> Self {
-        Self { error, strategy: RetryStrategy::GiveUp }
-    }
-
-    fn retry_later(error: PersonalizationError) -> Self {
-        Self { error, strategy: RetryStrategy::Retry }
-    }
-
-    fn rate_limited(error: PersonalizationError) -> Self {
-        Self { error, strategy: RetryStrategy::RetryAfterRateLimit }
-    }
-
-    fn into_duration(self, attempt: u32) -> Result<Duration, PersonalizationError> {
-        match self.strategy {
-            RetryStrategy::GiveUp => Err(self.error),
-            RetryStrategy::Retry => {
-                // Exponential backoff: 10^attempt milliseconds
-                Ok(Duration::from_millis((10u64).pow(attempt)))
-            }
-            RetryStrategy::RetryAfterRateLimit => {
-                // Longer backoff for rate limits: 100ms + exponential
-                Ok(Duration::from_millis(100 + (10u64).pow(attempt)))
-            }
-        }
-    }
-
-    fn into_error(self) -> PersonalizationError {
-        self.error
-    }
-}
-
-pub enum PersonalizationService {
-    Cohere(CohereService),
-    Disabled,
-}
-
-impl PersonalizationService {
-    pub fn cohere(api_key: String) -> Self {
-        // If the API key is empty, consider the personalization service as disabled
-        if api_key.trim().is_empty() {
-            Self::disabled()
-        } else {
-            Self::Cohere(CohereService::new(api_key))
-        }
-    }
-
-    pub fn disabled() -> Self {
-        debug!("Personalization service disabled");
-        Self::Disabled
-    }
-
-    pub async fn rerank_search_results(
-        &self,
-        search_result: SearchResult,
-        personalize: &Personalize,
-        query: Option<&str>,
-        time_budget: TimeBudget,
-    ) -> Result<SearchResult, ResponseError> {
-        match self {
-            Self::Cohere(cohere_service) => {
-                cohere_service
-                    .rerank_search_results(search_result, personalize, query, time_budget)
-                    .await
-            }
-            Self::Disabled => Err(PersonalizationError::FeatureNotEnabled(
-                index_scheduler::error::FeatureNotEnabledError {
-                    disabled_action: "reranking search results",
-                    feature: "personalization",
-                    issue_link: "https://github.com/orgs/meilisearch/discussions/866",
-                },
-            )
-            .into()),
-        }
-    }
-}

crates/meilisearch/src/routes/indexes/facet_search.rs

@@ -343,7 +343,6 @@ impl From<FacetSearchQuery> for SearchQuery {
             hybrid,
             ranking_score_threshold,
             locales,
-            personalize: None,
         }
     }
 }

crates/meilisearch/src/routes/indexes/search.rs

@@ -24,9 +24,9 @@ use crate::metrics::MEILISEARCH_DEGRADED_SEARCH_REQUESTS;
 use crate::routes::indexes::search_analytics::{SearchAggregator, SearchGET, SearchPOST};
 use crate::routes::parse_include_metadata_header;
 use crate::search::{
-    add_search_rules, perform_search, HybridQuery, MatchingStrategy, Personalize,
-    RankingScoreThreshold, RetrieveVectors, SearchKind, SearchParams, SearchQuery, SearchResult,
-    SemanticRatio, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG,
+    add_search_rules, perform_search, HybridQuery, MatchingStrategy, RankingScoreThreshold,
+    RetrieveVectors, SearchKind, SearchParams, SearchQuery, SearchResult, SemanticRatio,
+    DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG,
     DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET, DEFAULT_SEMANTIC_RATIO,
 };
 use crate::search_queue::SearchQueue;
@@ -134,8 +134,6 @@ pub struct SearchQueryGet {
     #[deserr(default, error = DeserrQueryParamError<InvalidSearchLocales>)]
     #[param(value_type = Vec<Locale>, explode = false)]
     pub locales: Option<CS<Locale>>,
-    #[deserr(default, error = DeserrQueryParamError<InvalidSearchPersonalizeUserContext>)]
-    pub personalize_user_context: Option<String>,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, deserr::Deserr)]
@@ -207,9 +205,6 @@ impl TryFrom<SearchQueryGet> for SearchQuery {
             ));
         }
 
-        let personalize =
-            other.personalize_user_context.map(|user_context| Personalize { user_context });
-
         Ok(Self {
             q: other.q,
             // `media` not supported for `GET`
@@ -239,7 +234,6 @@ impl TryFrom<SearchQueryGet> for SearchQuery {
             hybrid,
             ranking_score_threshold: other.ranking_score_threshold.map(|o| o.0),
             locales: other.locales.map(|o| o.into_iter().collect()),
-            personalize,
         })
     }
 }
@@ -328,7 +322,6 @@ pub fn fix_sort_query_parameters(sort_query: &str) -> Vec<String> {
 pub async fn search_with_url_query(
     index_scheduler: GuardedData<ActionPolicy<{ actions::SEARCH }>, Data<IndexScheduler>>,
     search_queue: web::Data<SearchQueue>,
-    personalization_service: web::Data<crate::personalization::PersonalizationService>,
     index_uid: web::Path<String>,
     params: AwebQueryParameter<SearchQueryGet, DeserrQueryParamError>,
     req: HttpRequest,
@@ -349,16 +342,9 @@ pub async fn search_with_url_query(
 
     let index = index_scheduler.index(&index_uid)?;
 
-    // Extract personalization and query string before moving query
-    let personalize = query.personalize.take();
-
     let search_kind =
         search_kind(&query, index_scheduler.get_ref(), index_uid.to_string(), &index)?;
     let retrieve_vector = RetrieveVectors::new(query.retrieve_vectors);
-
-    // Save the query string for personalization if requested
-    let personalize_query = personalize.is_some().then(|| query.q.clone()).flatten();
-
     let permit = search_queue.try_get_search_permit().await?;
     let include_metadata = parse_include_metadata_header(&req);
 
@@ -379,24 +365,12 @@ pub async fn search_with_url_query(
     .await;
     permit.drop().await;
     let search_result = search_result?;
-    if let Ok((search_result, _)) = search_result.as_ref() {
+    if let Ok(ref search_result) = search_result {
         aggregate.succeed(search_result);
     }
     analytics.publish(aggregate, &req);
 
-    let (mut search_result, time_budget) = search_result?;
-
-    // Apply personalization if requested
-    if let Some(personalize) = personalize.as_ref() {
-        search_result = personalization_service
-            .rerank_search_results(
-                search_result,
-                personalize,
-                personalize_query.as_deref(),
-                time_budget,
-            )
-            .await?;
-    }
+    let search_result = search_result?;
 
     debug!(request_uid = ?request_uid, returns = ?search_result, "Search get");
     Ok(HttpResponse::Ok().json(search_result))
@@ -461,7 +435,6 @@ pub async fn search_with_url_query(
 pub async fn search_with_post(
     index_scheduler: GuardedData<ActionPolicy<{ actions::SEARCH }>, Data<IndexScheduler>>,
     search_queue: web::Data<SearchQueue>,
-    personalization_service: web::Data<crate::personalization::PersonalizationService>,
     index_uid: web::Path<String>,
     params: AwebJson<SearchQuery, DeserrJsonError>,
     req: HttpRequest,
@@ -482,18 +455,12 @@ pub async fn search_with_post(
 
     let index = index_scheduler.index(&index_uid)?;
 
-    // Extract personalization and query string before moving query
-    let personalize = query.personalize.take();
-
     let search_kind =
         search_kind(&query, index_scheduler.get_ref(), index_uid.to_string(), &index)?;
     let retrieve_vectors = RetrieveVectors::new(query.retrieve_vectors);
 
     let include_metadata = parse_include_metadata_header(&req);
 
-    // Save the query string for personalization if requested
-    let personalize_query = personalize.is_some().then(|| query.q.clone()).flatten();
-
     let permit = search_queue.try_get_search_permit().await?;
     let search_result = tokio::task::spawn_blocking(move || {
         perform_search(
@@ -512,7 +479,7 @@ pub async fn search_with_post(
     .await;
     permit.drop().await;
     let search_result = search_result?;
-    if let Ok((ref search_result, _)) = search_result {
+    if let Ok(ref search_result) = search_result {
         aggregate.succeed(search_result);
         if search_result.degraded {
             MEILISEARCH_DEGRADED_SEARCH_REQUESTS.inc();
@@ -520,19 +487,7 @@ pub async fn search_with_post(
     }
     analytics.publish(aggregate, &req);
 
-    let (mut search_result, time_budget) = search_result?;
-
-    // Apply personalization if requested
-    if let Some(personalize) = personalize.as_ref() {
-        search_result = personalization_service
-            .rerank_search_results(
-                search_result,
-                personalize,
-                personalize_query.as_deref(),
-                time_budget,
-            )
-            .await?;
-    }
+    let search_result = search_result?;
 
     debug!(request_uid = ?request_uid, returns = ?search_result, "Search post");
     Ok(HttpResponse::Ok().json(search_result))

crates/meilisearch/src/routes/indexes/search_analytics.rs

@@ -7,7 +7,6 @@ use serde_json::{json, Value};
 
 use crate::aggregate_methods;
 use crate::analytics::{Aggregate, AggregateMethod};
-use crate::metrics::MEILISEARCH_PERSONALIZED_SEARCH_REQUESTS;
 use crate::search::{
     SearchQuery, SearchResult, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER,
     DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT,
@@ -96,9 +95,6 @@ pub struct SearchAggregator<Method: AggregateMethod> {
     show_ranking_score_details: bool,
     ranking_score_threshold: bool,
 
-    // personalization
-    total_personalized: usize,
-
     marker: std::marker::PhantomData<Method>,
 }
 
@@ -133,7 +129,6 @@ impl<Method: AggregateMethod> SearchAggregator<Method> {
             hybrid,
             ranking_score_threshold,
             locales,
-            personalize,
         } = query;
 
         let mut ret = Self::default();
@@ -209,12 +204,6 @@ impl<Method: AggregateMethod> SearchAggregator<Method> {
             ret.locales = locales.iter().copied().collect();
         }
 
-        // personalization
-        if personalize.is_some() {
-            ret.total_personalized = 1;
-            MEILISEARCH_PERSONALIZED_SEARCH_REQUESTS.inc();
-        }
-
         ret.highlight_pre_tag = *highlight_pre_tag != DEFAULT_HIGHLIGHT_PRE_TAG();
         ret.highlight_post_tag = *highlight_post_tag != DEFAULT_HIGHLIGHT_POST_TAG();
         ret.crop_marker = *crop_marker != DEFAULT_CROP_MARKER();
@@ -307,7 +296,6 @@ impl<Method: AggregateMethod> Aggregate for SearchAggregator<Method> {
             total_used_negative_operator,
             ranking_score_threshold,
             mut locales,
-            total_personalized,
             marker: _,
         } = *new;
 
@@ -393,9 +381,6 @@ impl<Method: AggregateMethod> Aggregate for SearchAggregator<Method> {
         // locales
         self.locales.append(&mut locales);
 
-        // personalization
-        self.total_personalized = self.total_personalized.saturating_add(total_personalized);
-
         self
     }
 
@@ -441,7 +426,6 @@ impl<Method: AggregateMethod> Aggregate for SearchAggregator<Method> {
             total_used_negative_operator,
             ranking_score_threshold,
             locales,
-            total_personalized,
             marker: _,
         } = *self;
 
@@ -515,9 +499,6 @@ impl<Method: AggregateMethod> Aggregate for SearchAggregator<Method> {
                 "show_ranking_score_details": show_ranking_score_details,
                 "ranking_score_threshold": ranking_score_threshold,
             },
-            "personalization": {
-                "total_personalized": total_personalized,
-            },
         })
     }
 }

crates/meilisearch/src/routes/mod.rs

@@ -41,9 +41,7 @@ use crate::routes::indexes::IndexView;
 use crate::routes::multi_search::SearchResults;
 use crate::routes::network::{Network, Remote};
 use crate::routes::swap_indexes::SwapIndexesPayload;
-use crate::routes::webhooks::{
-    WebhookResults, WebhookSettings, WebhookWithMetadataRedactedAuthorization,
-};
+use crate::routes::webhooks::{WebhookResults, WebhookSettings, WebhookWithMetadata};
 use crate::search::{
     FederatedSearch, FederatedSearchResult, Federation, FederationOptions, MergeFacets,
     SearchQueryWithIndex, SearchResultWithIndex, SimilarQuery, SimilarResult,
@@ -105,7 +103,7 @@ mod webhooks;
         url = "/",
         description = "Local server",
     )),
-    components(schemas(PaginationView<KeyView>, PaginationView<IndexView>, IndexView, DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView<serde_json::Value>, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings<Unchecked>, Settings<Checked>, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind, Network, Remote, FilterableAttributesRule, FilterableAttributesPatterns, AttributePatterns, FilterableAttributesFeatures, FilterFeatures, Export, WebhookSettings, WebhookResults, WebhookWithMetadataRedactedAuthorization, meilisearch_types::milli::vector::VectorStoreBackend))
+    components(schemas(PaginationView<KeyView>, PaginationView<IndexView>, IndexView, DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView<serde_json::Value>, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings<Unchecked>, Settings<Checked>, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind, Network, Remote, FilterableAttributesRule, FilterableAttributesPatterns, AttributePatterns, FilterableAttributesFeatures, FilterFeatures, Export, WebhookSettings, WebhookResults, WebhookWithMetadata, meilisearch_types::milli::vector::VectorStoreBackend))
 )]
 pub struct MeilisearchApi;
 

crates/meilisearch/src/routes/multi_search.rs

@@ -146,7 +146,6 @@ pub struct SearchResults {
 pub async fn multi_search_with_post(
     index_scheduler: GuardedData<ActionPolicy<{ actions::SEARCH }>, Data<IndexScheduler>>,
     search_queue: Data<SearchQueue>,
-    personalization_service: web::Data<crate::personalization::PersonalizationService>,
     params: AwebJson<FederatedSearch, DeserrJsonError>,
     req: HttpRequest,
     analytics: web::Data<Analytics>,
@@ -237,7 +236,7 @@ pub async fn multi_search_with_post(
             // changes.
             let search_results: Result<_, (ResponseError, usize)> = async {
                 let mut search_results = Vec::with_capacity(queries.len());
-                for (query_index, (index_uid, mut query, federation_options)) in queries
+                for (query_index, (index_uid, query, federation_options)) in queries
                     .into_iter()
                     .map(SearchQueryWithIndex::into_index_query_federation)
                     .enumerate()
@@ -270,13 +269,6 @@ pub async fn multi_search_with_post(
                         })
                         .with_index(query_index)?;
 
-                    // Extract personalization and query string before moving query
-                    let personalize = query.personalize.take();
-
-                    // Save the query string for personalization if requested
-                    let personalize_query =
-                        personalize.is_some().then(|| query.q.clone()).flatten();
-
                     let index_uid_str = index_uid.to_string();
 
                     let search_kind = search_kind(
@@ -288,7 +280,7 @@ pub async fn multi_search_with_post(
                     .with_index(query_index)?;
                     let retrieve_vector = RetrieveVectors::new(query.retrieve_vectors);
 
-                    let (mut search_result, time_budget) = tokio::task::spawn_blocking(move || {
+                    let search_result = tokio::task::spawn_blocking(move || {
                         perform_search(
                             SearchParams {
                                 index_uid: index_uid_str.clone(),
@@ -303,25 +295,11 @@ pub async fn multi_search_with_post(
                         )
                     })
                     .await
-                    .with_index(query_index)?
                     .with_index(query_index)?;
 
-                    // Apply personalization if requested
-                    if let Some(personalize) = personalize.as_ref() {
-                        search_result = personalization_service
-                            .rerank_search_results(
-                                search_result,
-                                personalize,
-                                personalize_query.as_deref(),
-                                time_budget,
-                            )
-                            .await
-                            .with_index(query_index)?;
-                    }
-
                     search_results.push(SearchResultWithIndex {
                         index_uid: index_uid.into_inner(),
-                        result: search_result,
+                        result: search_result.with_index(query_index)?,
                     });
                 }
                 Ok(search_results)

crates/meilisearch/src/routes/multi_search_analytics.rs

@@ -67,7 +67,6 @@ impl MultiSearchAggregator {
             hybrid: _,
             ranking_score_threshold: _,
             locales: _,
-            personalize: _,
         } in &federated_search.queries
         {
             if let Some(federation_options) = federation_options {

crates/meilisearch/src/routes/webhooks.rs

@@ -90,7 +90,7 @@ fn deny_immutable_fields_webhook(
 #[derive(Debug, Serialize, ToSchema)]
 #[serde(rename_all = "camelCase")]
 #[schema(rename_all = "camelCase")]
-pub(super) struct WebhookWithMetadataRedactedAuthorization {
+pub(super) struct WebhookWithMetadata {
     uuid: Uuid,
     is_editable: bool,
     #[schema(value_type = WebhookSettings)]
@@ -98,9 +98,8 @@ pub(super) struct WebhookWithMetadataRedactedAuthorization {
     webhook: Webhook,
 }
 
-impl WebhookWithMetadataRedactedAuthorization {
-    pub fn from(uuid: Uuid, mut webhook: Webhook) -> Self {
-        webhook.redact_authorization_header();
+impl WebhookWithMetadata {
+    pub fn from(uuid: Uuid, webhook: Webhook) -> Self {
         Self { uuid, is_editable: uuid != Uuid::nil(), webhook }
     }
 }
@@ -108,7 +107,7 @@ impl WebhookWithMetadataRedactedAuthorization {
 #[derive(Debug, Serialize, ToSchema)]
 #[serde(rename_all = "camelCase")]
 pub(super) struct WebhookResults {
-    results: Vec<WebhookWithMetadataRedactedAuthorization>,
+    results: Vec<WebhookWithMetadata>,
 }
 
 #[utoipa::path(
@@ -151,7 +150,7 @@ async fn get_webhooks(
     let results = webhooks
         .webhooks
         .into_iter()
-        .map(|(uuid, webhook)| WebhookWithMetadataRedactedAuthorization::from(uuid, webhook))
+        .map(|(uuid, webhook)| WebhookWithMetadata::from(uuid, webhook))
         .collect::<Vec<_>>();
     let results = WebhookResults { results };
 
@@ -302,7 +301,7 @@ fn check_changed(uuid: Uuid, webhook: &Webhook) -> Result<(), WebhooksError> {
     tag = "Webhooks",
     security(("Bearer" = ["webhooks.get", "webhooks.*", "*.get", "*"])),
     responses(
-        (status = 200, description = "Webhook found", body = WebhookWithMetadataRedactedAuthorization, content_type = "application/json", example = json!({
+        (status = 200, description = "Webhook found", body = WebhookWithMetadata, content_type = "application/json", example = json!({
             "uuid": "550e8400-e29b-41d4-a716-446655440000",
             "url": "https://your.site/on-tasks-completed",
             "headers": {
@@ -325,7 +324,7 @@ async fn get_webhook(
     let mut webhooks = index_scheduler.webhooks_view();
 
     let webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?;
-    let webhook = WebhookWithMetadataRedactedAuthorization::from(uuid, webhook);
+    let webhook = WebhookWithMetadata::from(uuid, webhook);
 
     debug!(returns = ?webhook, "Get webhook");
     Ok(HttpResponse::Ok().json(webhook))
@@ -338,7 +337,7 @@ async fn get_webhook(
     request_body = WebhookSettings,
     security(("Bearer" = ["webhooks.create", "webhooks.*", "*"])),
     responses(
-        (status = 201, description = "Webhook created successfully", body = WebhookWithMetadataRedactedAuthorization, content_type = "application/json", example = json!({
+        (status = 201, description = "Webhook created successfully", body = WebhookWithMetadata, content_type = "application/json", example = json!({
             "uuid": "550e8400-e29b-41d4-a716-446655440000",
             "url": "https://your.site/on-tasks-completed",
             "headers": {
@@ -384,7 +383,7 @@ async fn post_webhook(
 
     analytics.publish(PostWebhooksAnalytics, &req);
 
-    let response = WebhookWithMetadataRedactedAuthorization::from(uuid, webhook);
+    let response = WebhookWithMetadata::from(uuid, webhook);
     debug!(returns = ?response, "Post webhook");
     Ok(HttpResponse::Created().json(response))
 }
@@ -396,7 +395,7 @@ async fn post_webhook(
     request_body = WebhookSettings,
     security(("Bearer" = ["webhooks.update", "webhooks.*", "*"])),
     responses(
-        (status = 200, description = "Webhook updated successfully", body = WebhookWithMetadataRedactedAuthorization, content_type = "application/json", example = json!({
+        (status = 200, description = "Webhook updated successfully", body = WebhookWithMetadata, content_type = "application/json", example = json!({
             "uuid": "550e8400-e29b-41d4-a716-446655440000",
             "url": "https://your.site/on-tasks-completed",
             "headers": {
@@ -436,7 +435,7 @@ async fn patch_webhook(
 
     analytics.publish(PatchWebhooksAnalytics, &req);
 
-    let response = WebhookWithMetadataRedactedAuthorization::from(uuid, webhook);
+    let response = WebhookWithMetadata::from(uuid, webhook);
     debug!(returns = ?response, "Patch webhook");
     Ok(HttpResponse::Ok().json(response))
 }

crates/meilisearch/src/search/federated/perform.rs

@@ -601,10 +601,6 @@ impl PartitionedQueries {
             .into());
         }
 
-        if federated_query.has_personalize() {
-            return Err(MeilisearchHttpError::PersonalizationInFederatedQuery(query_index).into());
-        }
-
         let (index_uid, query, federation_options) = federated_query.into_index_query_federation();
 
         let federation_options = federation_options.unwrap_or_default();

crates/meilisearch/src/search/mod.rs

@@ -59,13 +59,6 @@ pub const DEFAULT_HIGHLIGHT_POST_TAG: fn() -> String = || "</em>".to_string();
 pub const DEFAULT_SEMANTIC_RATIO: fn() -> SemanticRatio = || SemanticRatio(0.5);
 pub const INCLUDE_METADATA_HEADER: &str = "Meili-Include-Metadata";
 
-#[derive(Clone, Default, PartialEq, Deserr, ToSchema, Debug)]
-#[deserr(error = DeserrJsonError<InvalidSearchPersonalize>, rename_all = camelCase, deny_unknown_fields)]
-pub struct Personalize {
-    #[deserr(error = DeserrJsonError<InvalidSearchPersonalizeUserContext>)]
-    pub user_context: String,
-}
-
 #[derive(Clone, Default, PartialEq, Deserr, ToSchema)]
 #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
 pub struct SearchQuery {
@@ -129,8 +122,6 @@ pub struct SearchQuery {
     pub ranking_score_threshold: Option<RankingScoreThreshold>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchLocales>)]
     pub locales: Option<Vec<Locale>>,
-    #[deserr(default, error = DeserrJsonError<InvalidSearchPersonalize>, default)]
-    pub personalize: Option<Personalize>,
 }
 
 impl From<SearchParameters> for SearchQuery {
@@ -178,7 +169,6 @@ impl From<SearchParameters> for SearchQuery {
             highlight_post_tag: DEFAULT_HIGHLIGHT_POST_TAG(),
             crop_marker: DEFAULT_CROP_MARKER(),
             locales: None,
-            personalize: None,
         }
     }
 }
@@ -260,7 +250,6 @@ impl fmt::Debug for SearchQuery {
             attributes_to_search_on,
             ranking_score_threshold,
             locales,
-            personalize,
         } = self;
 
         let mut debug = f.debug_struct("SearchQuery");
@@ -349,10 +338,6 @@ impl fmt::Debug for SearchQuery {
             debug.field("locales", &locales);
         }
 
-        if let Some(personalize) = personalize {
-            debug.field("personalize", &personalize);
-        }
-
         debug.finish()
     }
 }
@@ -558,9 +543,6 @@ pub struct SearchQueryWithIndex {
     pub ranking_score_threshold: Option<RankingScoreThreshold>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchLocales>, default)]
     pub locales: Option<Vec<Locale>>,
-    #[deserr(default, error = DeserrJsonError<InvalidSearchPersonalize>, default)]
-    #[serde(skip)]
-    pub personalize: Option<Personalize>,
 
     #[deserr(default)]
     pub federation_options: Option<FederationOptions>,
@@ -585,10 +567,6 @@ impl SearchQueryWithIndex {
         self.facets.as_deref().filter(|v| !v.is_empty())
     }
 
-    pub fn has_personalize(&self) -> bool {
-        self.personalize.is_some()
-    }
-
     pub fn from_index_query_federation(
         index_uid: IndexUid,
         query: SearchQuery,
@@ -622,7 +600,6 @@ impl SearchQueryWithIndex {
             attributes_to_search_on,
             ranking_score_threshold,
             locales,
-            personalize,
         } = query;
 
         SearchQueryWithIndex {
@@ -654,7 +631,6 @@ impl SearchQueryWithIndex {
             attributes_to_search_on,
             ranking_score_threshold,
             locales,
-            personalize,
             federation_options,
         }
     }
@@ -690,7 +666,6 @@ impl SearchQueryWithIndex {
             hybrid,
             ranking_score_threshold,
             locales,
-            personalize,
         } = self;
         (
             index_uid,
@@ -722,7 +697,6 @@ impl SearchQueryWithIndex {
                 hybrid,
                 ranking_score_threshold,
                 locales,
-                personalize,
                 // do not use ..Default::default() here,
                 // rather add any missing field from `SearchQuery` to `SearchQueryWithIndex`
             },
@@ -1175,10 +1149,7 @@ pub struct SearchParams {
     pub include_metadata: bool,
 }
 
-pub fn perform_search(
-    params: SearchParams,
-    index: &Index,
-) -> Result<(SearchResult, TimeBudget), ResponseError> {
+pub fn perform_search(params: SearchParams, index: &Index) -> Result<SearchResult, ResponseError> {
     let SearchParams {
         index_uid,
         query,
@@ -1197,7 +1168,7 @@ pub fn perform_search(
     };
 
     let (search, is_finite_pagination, max_total_hits, offset) =
-        prepare_search(index, &rtxn, &query, &search_kind, time_budget.clone(), features)?;
+        prepare_search(index, &rtxn, &query, &search_kind, time_budget, features)?;
 
     let (
         milli::SearchResult {
@@ -1255,7 +1226,6 @@ pub fn perform_search(
         attributes_to_search_on: _,
         filter: _,
         distinct: _,
-        personalize: _,
     } = query;
 
     let format = AttributesFormat {
@@ -1321,7 +1291,7 @@ pub fn perform_search(
         request_uid: Some(request_uid),
         metadata,
     };
-    Ok((result, time_budget))
+    Ok(result)
 }
 
 #[derive(Debug, Clone, Default, Serialize, Deserialize, ToSchema)]

crates/meilisearch/tests/common/server.rs

@@ -49,8 +49,8 @@ impl Server<Owned> {
         }
 
         let options = default_settings(dir.path());
-        let handle = tokio::runtime::Handle::current();
-        let (index_scheduler, auth) = setup_meilisearch(&options, handle).unwrap();
+
+        let (index_scheduler, auth) = setup_meilisearch(&options).unwrap();
         let service = Service { index_scheduler, auth, options, api_key: None };
 
         Server { service, _dir: Some(dir), _marker: PhantomData }
@@ -65,9 +65,7 @@ impl Server<Owned> {
 
         options.master_key = Some("MASTER_KEY".to_string());
 
-        let handle = tokio::runtime::Handle::current();
-
-        let (index_scheduler, auth) = setup_meilisearch(&options, handle).unwrap();
+        let (index_scheduler, auth) = setup_meilisearch(&options).unwrap();
         let service = Service { index_scheduler, auth, options, api_key: None };
 
         Server { service, _dir: Some(dir), _marker: PhantomData }
@@ -80,9 +78,7 @@ impl Server<Owned> {
     }
 
     pub async fn new_with_options(options: Opt) -> Result<Self, anyhow::Error> {
-        let handle = tokio::runtime::Handle::current();
-
-        let (index_scheduler, auth) = setup_meilisearch(&options, handle)?;
+        let (index_scheduler, auth) = setup_meilisearch(&options)?;
         let service = Service { index_scheduler, auth, options, api_key: None };
 
         Ok(Server { service, _dir: None, _marker: PhantomData })
@@ -221,9 +217,8 @@ impl Server<Shared> {
         }
 
         let options = default_settings(dir.path());
-        let handle = tokio::runtime::Handle::current();
 
-        let (index_scheduler, auth) = setup_meilisearch(&options, handle).unwrap();
+        let (index_scheduler, auth) = setup_meilisearch(&options).unwrap();
         let service = Service { index_scheduler, auth, api_key: None, options };
 
         Server { service, _dir: Some(dir), _marker: PhantomData }

crates/meilisearch/tests/common/service.rs

@@ -10,9 +10,8 @@ use actix_web::test::TestRequest;
 use actix_web::web::Data;
 use index_scheduler::IndexScheduler;
 use meilisearch::analytics::Analytics;
-use meilisearch::personalization::PersonalizationService;
 use meilisearch::search_queue::SearchQueue;
-use meilisearch::{create_app, Opt, ServicesData, SubscriberForSecondLayer};
+use meilisearch::{create_app, Opt, SubscriberForSecondLayer};
 use meilisearch_auth::AuthController;
 use tracing::level_filters::LevelFilter;
 use tracing_subscriber::Layer;
@@ -136,24 +135,14 @@ impl Service {
             self.options.experimental_search_queue_size,
             NonZeroUsize::new(1).unwrap(),
         );
-        let personalization_service = self
-            .options
-            .experimental_personalization_api_key
-            .clone()
-            .map(PersonalizationService::cohere)
-            .unwrap_or_else(PersonalizationService::disabled);
 
         actix_web::test::init_service(create_app(
-            ServicesData {
-                index_scheduler: self.index_scheduler.clone().into(),
-                auth: self.auth.clone().into(),
-                search_queue: Data::new(search_queue),
-                personalization_service: Data::new(personalization_service),
-                logs_route_handle: Data::new(route_layer_handle),
-                logs_stderr_handle: Data::new(stderr_layer_handle),
-                analytics: Data::new(Analytics::no_analytics()),
-            },
+            self.index_scheduler.clone().into(),
+            self.auth.clone().into(),
+            Data::new(search_queue),
             self.options.clone(),
+            (route_layer_handle, stderr_layer_handle),
+            Data::new(Analytics::no_analytics()),
             true,
         ))
         .await

crates/meilisearch/tests/features/mod.rs

@@ -207,118 +207,3 @@ async fn errors() {
     }
     "###);
 }
-
-#[actix_rt::test]
-async fn search_with_personalization_without_enabling_the_feature() {
-    let server = Server::new().await;
-    let index = server.unique_index();
-
-    // Create the index and add some documents
-    let (task, _code) = index.create(None).await;
-    server.wait_task(task.uid()).await.succeeded();
-
-    let (task, _code) = index
-        .add_documents(
-            json!([
-                {"id": 1, "title": "The Dark Knight", "genre": "Action"},
-                {"id": 2, "title": "Inception", "genre": "Sci-Fi"},
-                {"id": 3, "title": "The Matrix", "genre": "Sci-Fi"}
-            ]),
-            None,
-        )
-        .await;
-    server.wait_task(task.uid()).await.succeeded();
-
-    // Try to search with personalization - should return feature_not_enabled error
-    let (response, code) = index
-        .search_post(json!({
-            "q": "movie",
-            "personalize": {
-                "userContext": "I love science fiction movies"
-            }
-        }))
-        .await;
-
-    meili_snap::snapshot!(code, @"400 Bad Request");
-    meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
-    {
-      "message": "reranking search results requires enabling the `personalization` experimental feature. See https://github.com/orgs/meilisearch/discussions/866",
-      "code": "feature_not_enabled",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
-    }
-    "###);
-}
-
-#[actix_rt::test]
-async fn multi_search_with_personalization_without_enabling_the_feature() {
-    let server = Server::new().await;
-    let index = server.unique_index();
-
-    // Create the index and add some documents
-    let (task, _code) = index.create(None).await;
-    server.wait_task(task.uid()).await.succeeded();
-
-    let (task, _code) = index
-        .add_documents(
-            json!([
-                {"id": 1, "title": "The Dark Knight", "genre": "Action"},
-                {"id": 2, "title": "Inception", "genre": "Sci-Fi"},
-                {"id": 3, "title": "The Matrix", "genre": "Sci-Fi"}
-            ]),
-            None,
-        )
-        .await;
-    server.wait_task(task.uid()).await.succeeded();
-
-    // Try to multi-search with personalization - should return feature_not_enabled error
-    let (response, code) = server
-        .multi_search(json!({
-            "queries": [
-                {
-                    "indexUid": index.uid,
-                    "q": "movie",
-                    "personalize": {
-                        "userContext": "I love science fiction movies"
-                    }
-                }
-            ]
-        }))
-        .await;
-
-    meili_snap::snapshot!(code, @"400 Bad Request");
-    meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
-    {
-      "message": "Inside `.queries[0]`: reranking search results requires enabling the `personalization` experimental feature. See https://github.com/orgs/meilisearch/discussions/866",
-      "code": "feature_not_enabled",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
-    }
-    "###);
-
-    // Try to federated search with personalization - should return feature_not_enabled error
-    let (response, code) = server
-        .multi_search(json!({
-          "federation": {},
-            "queries": [
-                {
-                    "indexUid": index.uid,
-                    "q": "movie",
-                    "personalize": {
-                        "userContext": "I love science fiction movies"
-                    }
-                }
-            ]
-        }))
-        .await;
-
-    meili_snap::snapshot!(code, @"400 Bad Request");
-    meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
-    {
-      "message": "Inside `.queries[0]`: Using `.personalize` is not allowed in federated queries.\n - Hint: remove `personalize` from query #0 or remove `federation` from the request",
-      "code": "invalid_multi_search_query_personalization",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#invalid_multi_search_query_personalization"
-    }
-    "###);
-}

crates/meilisearch/tests/logs/mod.rs

@@ -8,9 +8,8 @@ use actix_web::http::header::ContentType;
 use actix_web::web::Data;
 use meili_snap::snapshot;
 use meilisearch::analytics::Analytics;
-use meilisearch::personalization::PersonalizationService;
 use meilisearch::search_queue::SearchQueue;
-use meilisearch::{create_app, Opt, ServicesData, SubscriberForSecondLayer};
+use meilisearch::{create_app, Opt, SubscriberForSecondLayer};
 use tracing::level_filters::LevelFilter;
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::Layer;
@@ -51,16 +50,12 @@ async fn basic_test_log_stream_route() {
     );
 
     let app = actix_web::test::init_service(create_app(
-        ServicesData {
-            index_scheduler: server.service.index_scheduler.clone().into(),
-            auth: server.service.auth.clone().into(),
-            search_queue: Data::new(search_queue),
-            personalization_service: Data::new(PersonalizationService::disabled()),
-            logs_route_handle: Data::new(route_layer_handle),
-            logs_stderr_handle: Data::new(stderr_layer_handle),
-            analytics: Data::new(Analytics::no_analytics()),
-        },
+        server.service.index_scheduler.clone().into(),
+        server.service.auth.clone().into(),
+        Data::new(search_queue),
         server.service.options.clone(),
+        (route_layer_handle, stderr_layer_handle),
+        Data::new(Analytics::no_analytics()),
         true,
     ))
     .await;

crates/meilisearch/tests/search/errors.rs

@@ -1,11 +1,7 @@
 use meili_snap::*;
-use meilisearch::Opt;
-use tempfile::TempDir;
 
 use super::test_settings_documents_indexing_swapping_and_search;
-use crate::common::{
-    default_settings, shared_does_not_exists_index, Server, DOCUMENTS, NESTED_DOCUMENTS,
-};
+use crate::common::{shared_does_not_exists_index, Server, DOCUMENTS, NESTED_DOCUMENTS};
 use crate::json;
 
 #[actix_rt::test]
@@ -1324,98 +1320,3 @@ async fn search_with_contains_without_enabling_the_feature() {
     }
     "#);
 }
-
-#[actix_rt::test]
-#[ignore]
-async fn search_with_personalization_invalid_api_key() {
-    // Create a server with a fake personalization API key
-    let dir = TempDir::new().unwrap();
-    let options = Opt {
-        experimental_personalization_api_key: Some("fake-api-key-12345".to_string()),
-        ..default_settings(dir.path())
-    };
-    let server = Server::new_with_options(options).await.unwrap();
-    let index = server.unique_index();
-
-    // Create the index and add some documents
-    let (task, _code) = index.create(None).await;
-    server.wait_task(task.uid()).await.succeeded();
-
-    let (task, _code) = index
-        .add_documents(
-            json!([
-                {"id": 1, "title": "The Dark Knight", "genre": "Action"},
-                {"id": 2, "title": "Inception", "genre": "Sci-Fi"},
-                {"id": 3, "title": "The Matrix", "genre": "Sci-Fi"}
-            ]),
-            None,
-        )
-        .await;
-    server.wait_task(task.uid()).await.succeeded();
-
-    // Try to search with personalization - should return remote_invalid_api_key error
-    let (response, code) = index
-        .search_post(json!({
-            "q": "the",
-            "personalize": {
-                "userContext": "I love science fiction movies"
-            }
-        }))
-        .await;
-
-    snapshot!(code, @"403 Forbidden");
-    snapshot!(json_string!(response), @r#"
-    {
-      "message": "Personalization service: Unauthorized: invalid API key",
-      "code": "remote_invalid_api_key",
-      "type": "auth",
-      "link": "https://docs.meilisearch.com/errors#remote_invalid_api_key"
-    }
-    "#);
-}
-
-#[actix_rt::test]
-async fn search_with_personalization_no_user_context() {
-    // Create a server with a fake personalization API key
-    let dir = TempDir::new().unwrap();
-    let options = Opt {
-        experimental_personalization_api_key: Some("fake-api-key-12345".to_string()),
-        ..default_settings(dir.path())
-    };
-    let server = Server::new_with_options(options).await.unwrap();
-    let index = server.unique_index();
-
-    // Create the index and add some documents
-    let (task, _code) = index.create(None).await;
-    server.wait_task(task.uid()).await.succeeded();
-
-    let (task, _code) = index
-        .add_documents(
-            json!([
-                {"id": 1, "title": "The Dark Knight", "genre": "Action"},
-                {"id": 2, "title": "Inception", "genre": "Sci-Fi"},
-                {"id": 3, "title": "The Matrix", "genre": "Sci-Fi"}
-            ]),
-            None,
-        )
-        .await;
-    server.wait_task(task.uid()).await.succeeded();
-
-    // Try to search with personalization - should return remote_invalid_api_key error
-    let (response, code) = index
-        .search_post(json!({
-            "q": "the",
-            "personalize": {}
-        }))
-        .await;
-
-    snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "Missing field `userContext` inside `.personalize`",
-      "code": "invalid_search_personalize",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#invalid_search_personalize"
-    }
-    "###);
-}

crates/meilisearch/tests/tasks/webhook.rs

@@ -82,7 +82,7 @@ async fn cli_only() {
 
     let (webhooks, code) = server.get_webhooks().await;
     snapshot!(code, @"200 OK");
-    snapshot!(webhooks, @r###"
+    snapshot!(webhooks, @r#"
     {
       "results": [
         {
@@ -90,12 +90,12 @@ async fn cli_only() {
           "isEditable": false,
           "url": "https://example-cli.com/",
           "headers": {
-            "Authorization": "Bearer a-XXXX..."
+            "Authorization": "Bearer a-secret-token"
           }
         }
       ]
     }
-    "###);
+    "#);
 }
 
 #[actix_web::test]
@@ -233,7 +233,7 @@ async fn cli_with_dumps() {
 
     let (webhooks, code) = server.get_webhooks().await;
     snapshot!(code, @"200 OK");
-    snapshot!(webhooks, @r###"
+    snapshot!(webhooks, @r#"
     {
       "results": [
         {
@@ -241,7 +241,7 @@ async fn cli_with_dumps() {
           "isEditable": false,
           "url": "http://defined-in-test-cli.com/",
           "headers": {
-            "Authorization": "Bearer a-secXXXXXX..."
+            "Authorization": "Bearer a-secret-token-defined-in-test-cli"
           }
         },
         {
@@ -255,7 +255,7 @@ async fn cli_with_dumps() {
           "isEditable": true,
           "url": "https://example.com/hook",
           "headers": {
-            "authorization": "XXX..."
+            "authorization": "TOKEN"
           }
         },
         {
@@ -266,7 +266,7 @@ async fn cli_with_dumps() {
         }
       ]
     }
-    "###);
+    "#);
 }
 
 #[actix_web::test]
@@ -367,30 +367,30 @@ async fn post_get_delete() {
         }))
         .await;
     snapshot!(code, @"201 Created");
-    snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r###"
+    snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#"
     {
       "uuid": "[uuid]",
       "isEditable": true,
       "url": "https://example.com/hook",
       "headers": {
-        "authorization": "XXX..."
+        "authorization": "TOKEN"
       }
     }
-    "###);
+    "#);
 
     let uuid = value.get("uuid").unwrap().as_str().unwrap();
     let (value, code) = server.get_webhook(uuid).await;
     snapshot!(code, @"200 OK");
-    snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r###"
+    snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#"
     {
       "uuid": "[uuid]",
       "isEditable": true,
       "url": "https://example.com/hook",
       "headers": {
-        "authorization": "XXX..."
+        "authorization": "TOKEN"
       }
     }
-    "###);
+    "#);
 
     let (_value, code) = server.delete_webhook(uuid).await;
     snapshot!(code, @"204 No Content");
@@ -430,31 +430,31 @@ async fn create_and_patch() {
     let (value, code) =
         server.patch_webhook(&uuid, json!({ "headers": { "authorization": "TOKEN" } })).await;
     snapshot!(code, @"200 OK");
-    snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r###"
+    snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#"
     {
       "uuid": "[uuid]",
       "isEditable": true,
       "url": "https://example.com/hook",
       "headers": {
-        "authorization": "XXX..."
+        "authorization": "TOKEN"
       }
     }
-    "###);
+    "#);
 
     let (value, code) =
         server.patch_webhook(&uuid, json!({ "headers": { "authorization2": "TOKEN" } })).await;
     snapshot!(code, @"200 OK");
-    snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r###"
+    snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#"
     {
       "uuid": "[uuid]",
       "isEditable": true,
       "url": "https://example.com/hook",
       "headers": {
-        "authorization": "XXX...",
+        "authorization": "TOKEN",
         "authorization2": "TOKEN"
       }
     }
-    "###);
+    "#);
 
     let (value, code) =
         server.patch_webhook(&uuid, json!({ "headers": { "authorization": null } })).await;

crates/meilisearch/tests/upgrade/mod.rs

@@ -43,7 +43,7 @@ async fn version_too_old() {
     std::fs::write(db_path.join("VERSION"), "1.11.9999").unwrap();
     let options = Opt { experimental_dumpless_upgrade: true, ..default_settings };
     let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err();
-    snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.25.0");
+    snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.24.0");
 }
 
 #[actix_rt::test]
@@ -58,7 +58,7 @@ async fn version_requires_downgrade() {
     std::fs::write(db_path.join("VERSION"), format!("{major}.{minor}.{patch}")).unwrap();
     let options = Opt { experimental_dumpless_upgrade: true, ..default_settings };
     let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err();
-    snapshot!(err, @"Database version 1.25.1 is higher than the Meilisearch version 1.25.0. Downgrade is not supported");
+    snapshot!(err, @"Database version 1.24.1 is higher than the Meilisearch version 1.24.0. Downgrade is not supported");
 }
 
 #[actix_rt::test]

crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap

@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
       "progress": null,
       "details": {
         "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.25.0"
+        "upgradeTo": "v1.24.0"
       },
       "stats": {
         "totalNbTasks": 1,

crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap

@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
       "progress": null,
       "details": {
         "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.25.0"
+        "upgradeTo": "v1.24.0"
       },
       "stats": {
         "totalNbTasks": 1,

crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap

@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
       "progress": null,
       "details": {
         "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.25.0"
+        "upgradeTo": "v1.24.0"
       },
       "stats": {
         "totalNbTasks": 1,

crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap

@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
       "canceledBy": null,
       "details": {
         "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.25.0"
+        "upgradeTo": "v1.24.0"
       },
       "error": null,
       "duration": "[duration]",

crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap

@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
       "canceledBy": null,
       "details": {
         "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.25.0"
+        "upgradeTo": "v1.24.0"
       },
       "error": null,
       "duration": "[duration]",

crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap

@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
       "canceledBy": null,
       "details": {
         "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.25.0"
+        "upgradeTo": "v1.24.0"
       },
       "error": null,
       "duration": "[duration]",

crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap

@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
       "progress": null,
       "details": {
         "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.25.0"
+        "upgradeTo": "v1.24.0"
       },
       "stats": {
         "totalNbTasks": 1,

crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap

@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
       "canceledBy": null,
       "details": {
         "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.25.0"
+        "upgradeTo": "v1.24.0"
       },
       "error": null,
       "duration": "[duration]",

crates/meilitool/src/upgrade/v1_12.rs

@@ -68,7 +68,7 @@ fn convert_update_files(db_path: &Path) -> anyhow::Result<()> {
 
     for uuid in file_store.all_uuids().context("while retrieving uuids from file store")? {
         let uuid = uuid.context("while retrieving uuid from file store")?;
-        let update_file_path = file_store.update_path(uuid);
+        let update_file_path = file_store.get_update_path(uuid);
         let update_file = file_store
             .get_update(uuid)
             .with_context(|| format!("while getting update file for uuid {uuid:?}"))?;

crates/milli/Cargo.toml

@@ -18,7 +18,7 @@ bincode = "1.3.3"
 bstr = "1.12.0"
 bytemuck = { version = "1.23.1", features = ["extern_crate_alloc"] }
 byteorder = "1.5.0"
-charabia = { version = "0.9.8", default-features = false }
+charabia = { version = "0.9.7", default-features = false }
 cellulite = "0.3.1-nested-rtxns-2"
 concat-arrays = "0.1.2"
 convert_case = "0.8.0"
@@ -34,7 +34,7 @@ grenad = { version = "0.5.0", default-features = false, features = [
     "rayon",
     "tempfile",
 ] }
-heed = { version = "0.22.1-nested-rtxns-6", default-features = false, features = [
+heed = { version = "0.22.1-nested-rtxns", default-features = false, features = [
     "serde-json",
     "serde-bincode",
 ] }

crates/milli/src/index.rs

@@ -425,10 +425,6 @@ impl Index {
         self.env.info().map_size
     }
 
-    pub fn try_clone_inner_file(&self) -> heed::Result<File> {
-        self.env.try_clone_inner_file()
-    }
-
     pub fn copy_to_file(&self, file: &mut File, option: CompactionOption) -> Result<()> {
         self.env.copy_to_file(file, option).map_err(Into::into)
     }

crates/milli/src/update/index_documents/extract/extract_vector_points.rs

@@ -1192,12 +1192,12 @@ pub fn extract_embeddings_from_fragments<R: io::Read + io::Seek>(
                 Metadata { docid, external_docid: "", extractor_id },
                 value,
                 unused_vectors_distribution,
-            )?;
+            );
         }
     }
 
     // send last chunk
-    let on_embed = session.drain(unused_vectors_distribution)?;
+    let on_embed = session.drain(unused_vectors_distribution);
     on_embed.finish()
 }
 

crates/milli/src/update/indexer_config.rs

@@ -1,6 +1,3 @@
-use std::num::NonZeroUsize;
-use std::time::Duration;
-
 use grenad::CompressionType;
 
 use super::GrenadParameters;
@@ -23,7 +20,6 @@ pub struct IndexerConfig {
     pub experimental_no_edition_2024_for_dumps: bool,
     pub experimental_no_edition_2024_for_prefix_post_processing: bool,
     pub experimental_no_edition_2024_for_facet_post_processing: bool,
-    pub s3_snapshot_options: Option<S3SnapshotOptions>,
 }
 
 impl IndexerConfig {
@@ -41,20 +37,6 @@ impl IndexerConfig {
     }
 }
 
-#[derive(Debug, Clone)]
-pub struct S3SnapshotOptions {
-    pub s3_bucket_url: String,
-    pub s3_bucket_region: String,
-    pub s3_bucket_name: String,
-    pub s3_snapshot_prefix: String,
-    pub s3_access_key: String,
-    pub s3_secret_key: String,
-    pub s3_max_in_flight_parts: NonZeroUsize,
-    pub s3_compression_level: u32,
-    pub s3_signature_duration: Duration,
-    pub s3_multipart_part_size: u64,
-}
-
 /// By default use only 1 thread for indexing in tests
 #[cfg(test)]
 pub fn default_thread_pool_and_threads() -> (ThreadPoolNoAbort, Option<usize>) {
@@ -94,7 +76,6 @@ impl Default for IndexerConfig {
             experimental_no_edition_2024_for_dumps: false,
             experimental_no_edition_2024_for_prefix_post_processing: false,
             experimental_no_edition_2024_for_facet_post_processing: false,
-            s3_snapshot_options: None,
         }
     }
 }

crates/milli/src/update/mod.rs

@@ -5,7 +5,7 @@ pub use self::concurrent_available_ids::ConcurrentAvailableIds;
 pub use self::facet::bulk::FacetsUpdateBulk;
 pub use self::facet::incremental::FacetsUpdateIncrementalInner;
 pub use self::index_documents::{request_threads, *};
-pub use self::indexer_config::{default_thread_pool_and_threads, IndexerConfig, S3SnapshotOptions};
+pub use self::indexer_config::{default_thread_pool_and_threads, IndexerConfig};
 pub use self::new::ChannelCongestion;
 pub use self::settings::{validate_embedding_settings, Setting, Settings};
 pub use self::update_step::UpdateIndexingStep;

crates/milli/src/update/new/extract/vectors/mod.rs

@@ -684,7 +684,7 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> {
                                     metadata,
                                     input,
                                     unused_vectors_distribution,
-                                )?;
+                                );
                             }
                             ExtractorDiff::Unchanged => { /* nothing to do */ }
                         }
@@ -724,7 +724,7 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> {
                         if old_is_user_provided || full_reindex {
                             session.on_embed_mut().clear_vectors(docid);
                         }
-                        session.request_embedding(metadata, input, unused_vectors_distribution)?;
+                        session.request_embedding(metadata, input, unused_vectors_distribution);
                     }
                     ExtractorDiff::Unchanged => { /* do nothing */ }
                 }
@@ -764,7 +764,8 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> {
                     document_template,
                     doc_alloc,
                     new_fields_ids_map,
-                );
+                )
+                .ignore_errors();
 
                 update_autogenerated(
                     docid,
@@ -850,7 +851,8 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> {
                     document_template,
                     doc_alloc,
                     new_fields_ids_map,
-                );
+                )
+                .ignore_errors();
 
                 insert_autogenerated(
                     docid,
@@ -885,10 +887,10 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> {
     pub fn drain(self, unused_vectors_distribution: &UnusedVectorsDistributionBump) -> Result<()> {
         match self.kind {
             ChunkType::DocumentTemplate { document_template: _, session } => {
-                session.drain(unused_vectors_distribution)?;
+                session.drain(unused_vectors_distribution);
             }
             ChunkType::Fragments { fragments: _, session } => {
-                session.drain(unused_vectors_distribution)?;
+                session.drain(unused_vectors_distribution);
             }
         }
         Ok(())
@@ -1036,7 +1038,7 @@ where
                 Metadata { docid, external_docid, extractor_id: extractor.extractor_id() };
 
             if let Some(new_rendered) = new_rendered {
-                session.request_embedding(metadata, new_rendered, unused_vectors_distribution)?
+                session.request_embedding(metadata, new_rendered, unused_vectors_distribution)
             } else {
                 // remove any existing embedding
                 OnEmbed::process_embedding_response(
@@ -1072,7 +1074,7 @@ where
                 Metadata { docid, external_docid, extractor_id: extractor.extractor_id() },
                 new_rendered,
                 unused_vectors_distribution,
-            )?;
+            );
         }
     }
 

crates/milli/src/update/new/words_prefix_docids.rs

@@ -4,7 +4,7 @@ use std::io::{BufReader, BufWriter, Read, Seek, Write};
 use std::iter;
 
 use hashbrown::HashMap;
-use heed::types::{Bytes, DecodeIgnore, Str};
+use heed::types::{Bytes, DecodeIgnore};
 use heed::{BytesDecode, Database, Error, RoTxn, RwTxn};
 use rayon::iter::{IndexedParallelIterator as _, IntoParallelIterator, ParallelIterator as _};
 use roaring::MultiOps;
@@ -16,29 +16,22 @@ use crate::heed_codec::StrBEU16Codec;
 use crate::update::GrenadParameters;
 use crate::{CboRoaringBitmapCodec, Index, Prefix, Result};
 
-struct WordPrefixDocids<'i> {
-    index: &'i Index,
+struct WordPrefixDocids {
     database: Database<Bytes, CboRoaringBitmapCodec>,
     prefix_database: Database<Bytes, CboRoaringBitmapCodec>,
     max_memory_by_thread: Option<usize>,
-    /// Do not use an experimental LMDB feature to read uncommitted data in parallel.
-    no_experimental_post_processing: bool,
 }
 
-impl<'i> WordPrefixDocids<'i> {
+impl WordPrefixDocids {
     fn new(
-        index: &'i Index,
         database: Database<Bytes, CboRoaringBitmapCodec>,
         prefix_database: Database<Bytes, CboRoaringBitmapCodec>,
         grenad_parameters: &GrenadParameters,
-    ) -> WordPrefixDocids<'i> {
+    ) -> WordPrefixDocids {
         WordPrefixDocids {
-            index,
             database,
             prefix_database,
             max_memory_by_thread: grenad_parameters.max_memory_by_thread(),
-            no_experimental_post_processing: grenad_parameters
-                .experimental_no_edition_2024_for_prefix_post_processing,
         }
     }
 
@@ -49,77 +42,7 @@ impl<'i> WordPrefixDocids<'i> {
         prefix_to_delete: &BTreeSet<Prefix>,
     ) -> Result<()> {
         delete_prefixes(wtxn, &self.prefix_database, prefix_to_delete)?;
-        if self.no_experimental_post_processing {
-            self.recompute_modified_prefixes(wtxn, prefix_to_compute)
-        } else {
-            self.recompute_modified_prefixes_no_frozen(wtxn, prefix_to_compute)
-        }
-    }
-
-    #[tracing::instrument(level = "trace", skip_all, target = "indexing::prefix")]
-    fn recompute_modified_prefixes_no_frozen(
-        &self,
-        wtxn: &mut RwTxn,
-        prefix_to_compute: &BTreeSet<Prefix>,
-    ) -> Result<()> {
-        let thread_count = rayon::current_num_threads();
-        let rtxns = iter::repeat_with(|| self.index.env.nested_read_txn(wtxn))
-            .take(thread_count)
-            .collect::<heed::Result<Vec<_>>>()?;
-
-        let outputs = rtxns
-            .into_par_iter()
-            .enumerate()
-            .map(|(thread_id, rtxn)| {
-                // `indexes` represent offsets at which prefixes computations were stored in the `file`.
-                let mut indexes = Vec::new();
-                let mut file = BufWriter::new(spooled_tempfile(
-                    self.max_memory_by_thread.unwrap_or(usize::MAX),
-                ));
-
-                let mut buffer = Vec::new();
-                for (prefix_index, prefix) in prefix_to_compute.iter().enumerate() {
-                    // Is prefix for another thread?
-                    if prefix_index % thread_count != thread_id {
-                        continue;
-                    }
-
-                    let output = self
-                        .database
-                        .prefix_iter(&rtxn, prefix.as_bytes())?
-                        .remap_types::<Str, CboRoaringBitmapCodec>()
-                        .map(|result| result.map(|(_word, bitmap)| bitmap))
-                        .union()?;
-
-                    buffer.clear();
-                    CboRoaringBitmapCodec::serialize_into_vec(&output, &mut buffer);
-                    indexes.push(PrefixEntry { prefix, serialized_length: buffer.len() });
-                    file.write_all(&buffer)?;
-                }
-
-                Ok((indexes, file))
-            })
-            .collect::<Result<Vec<_>>>()?;
-
-        // We iterate over all the collected and serialized bitmaps through
-        // the files and entries to eventually put them in the final database.
-        let mut buffer = Vec::new();
-        for (index, file) in outputs {
-            let mut file = file.into_inner().map_err(|e| e.into_error())?;
-            file.rewind()?;
-            let mut file = BufReader::new(file);
-            for PrefixEntry { prefix, serialized_length } in index {
-                buffer.resize(serialized_length, 0);
-                file.read_exact(&mut buffer)?;
-                self.prefix_database.remap_data_type::<Bytes>().put(
-                    wtxn,
-                    prefix.as_bytes(),
-                    &buffer,
-                )?;
-            }
-        }
-
-        Ok(())
+        self.recompute_modified_prefixes(wtxn, prefix_to_compute)
     }
 
     #[tracing::instrument(level = "trace", skip_all, target = "indexing::prefix")]
@@ -540,7 +463,6 @@ pub fn compute_word_prefix_docids(
     grenad_parameters: &GrenadParameters,
 ) -> Result<()> {
     WordPrefixDocids::new(
-        index,
         index.word_docids.remap_key_type(),
         index.word_prefix_docids.remap_key_type(),
         grenad_parameters,
@@ -557,7 +479,6 @@ pub fn compute_exact_word_prefix_docids(
     grenad_parameters: &GrenadParameters,
 ) -> Result<()> {
     WordPrefixDocids::new(
-        index,
         index.exact_word_docids.remap_key_type(),
         index.exact_word_prefix_docids.remap_key_type(),
         grenad_parameters,

crates/milli/src/update/upgrade/mod.rs

@@ -42,7 +42,6 @@ const UPGRADE_FUNCTIONS: &[&dyn UpgradeIndex] = &[
     &ToTargetNoOp { target: (1, 22, 0) },
     &ToTargetNoOp { target: (1, 23, 0) },
     &ToTargetNoOp { target: (1, 24, 0) },
-    &ToTargetNoOp { target: (1, 25, 0) },
     // This is the last upgrade function, it will be called when the index is up to date.
     // any other upgrade function should be added before this one.
     &ToCurrentNoOp {},
@@ -78,7 +77,6 @@ const fn start(from: (u32, u32, u32)) -> Option<usize> {
         (1, 22, _) => function_index!(12),
         (1, 23, _) => function_index!(13),
         (1, 24, _) => function_index!(14),
-        (1, 25, _) => function_index!(15),
         // We deliberately don't add a placeholder with (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) here to force manually
         // considering dumpless upgrade.
         (_major, _minor, _patch) => return None,

crates/milli/src/vector/embedder/rest.rs

@@ -91,6 +91,7 @@ struct EmbedderData {
     request: RequestData,
     response: Response,
     configuration_source: ConfigurationSource,
+    max_retry_duration: std::time::Duration,
 }
 
 #[derive(Debug)]
@@ -182,10 +183,15 @@ impl Embedder {
     ) -> Result<Self, NewEmbedderError> {
         let bearer = options.api_key.as_deref().map(|api_key| format!("Bearer {api_key}"));
 
+        let timeout = std::env::var("MEILI_EXPERIMENTAL_REST_EMBEDDER_TIMEOUT_SECONDS")
+            .ok()
+            .map(|p| p.parse().unwrap())
+            .unwrap_or(30);
+
         let client = ureq::AgentBuilder::new()
             .max_idle_connections(REQUEST_PARALLELISM * 2)
             .max_idle_connections_per_host(REQUEST_PARALLELISM * 2)
-            .timeout(std::time::Duration::from_secs(30))
+            .timeout(std::time::Duration::from_secs(timeout))
             .build();
 
         let request = RequestData::new(
@@ -196,6 +202,14 @@ impl Embedder {
 
         let response = Response::new(options.response, &request)?;
 
+        let max_retry_duration =
+            std::env::var("MEILI_EXPERIMENTAL_REST_EMBEDDER_MAX_RETRY_DURATION_SECONDS")
+                .ok()
+                .map(|p| p.parse().unwrap())
+                .unwrap_or(60);
+
+        let max_retry_duration = std::time::Duration::from_secs(max_retry_duration);
+
         let data = EmbedderData {
             client,
             bearer,
@@ -204,6 +218,7 @@ impl Embedder {
             response,
             configuration_source,
             headers: options.headers,
+            max_retry_duration,
         };
 
         let dimensions = if let Some(dimensions) = options.dimensions {
@@ -457,7 +472,7 @@ where
             }
         }?;
 
-        let retry_duration = retry_duration.min(std::time::Duration::from_secs(60)); // don't wait more than a minute
+        let retry_duration = retry_duration.min(data.max_retry_duration); // don't wait more than the max duration
 
         // randomly up to double the retry duration
         let retry_duration = retry_duration

crates/milli/src/vector/session.rs

@@ -5,7 +5,7 @@ use serde_json::Value;
 use super::error::EmbedError;
 use super::{Embedder, Embedding};
 use crate::progress::EmbedderStats;
-use crate::{DocumentId, Result, ThreadPoolNoAbort};
+use crate::{DocumentId, ThreadPoolNoAbort};
 type ExtractorId = u8;
 
 #[derive(Clone, Copy)]
@@ -108,32 +108,28 @@ impl<'doc, C: OnEmbed<'doc>, I: Input> EmbedSession<'doc, C, I> {
         metadata: Metadata<'doc>,
         rendered: I,
         unused_vectors_distribution: &C::ErrorMetadata,
-    ) -> Result<()> {
+    ) {
         if self.inputs.len() < self.inputs.capacity() {
             self.inputs.push(rendered);
             self.metadata.push(metadata);
-            return Ok(());
+            return;
         }
 
         self.embed_chunks(unused_vectors_distribution)
     }
 
-    pub fn drain(mut self, unused_vectors_distribution: &C::ErrorMetadata) -> Result<C> {
-        self.embed_chunks(unused_vectors_distribution)?;
-        Ok(self.on_embed)
+    pub fn drain(mut self, unused_vectors_distribution: &C::ErrorMetadata) -> C {
+        self.embed_chunks(unused_vectors_distribution);
+        self.on_embed
     }
 
     #[allow(clippy::too_many_arguments)]
-    fn embed_chunks(&mut self, unused_vectors_distribution: &C::ErrorMetadata) -> Result<()> {
+    fn embed_chunks(&mut self, _unused_vectors_distribution: &C::ErrorMetadata) {
         if self.inputs.is_empty() {
-            return Ok(());
+            return;
         }
-        let res = match I::embed_ref(
-            self.inputs.as_slice(),
-            self.embedder,
-            self.threads,
-            self.embedder_stats,
-        ) {
+        match I::embed_ref(self.inputs.as_slice(), self.embedder, self.threads, self.embedder_stats)
+        {
             Ok(embeddings) => {
                 for (metadata, embedding) in self.metadata.iter().copied().zip(embeddings) {
                     self.on_embed.process_embedding_response(EmbeddingResponse {
@@ -141,27 +137,37 @@ impl<'doc, C: OnEmbed<'doc>, I: Input> EmbedSession<'doc, C, I> {
                         embedding: Some(embedding),
                     });
                 }
-                Ok(())
             }
             Err(error) => {
-                // reset metadata and inputs, and send metadata to the error processing.
-                let doc_alloc = self.metadata.bump();
-                let metadata = std::mem::replace(
-                    &mut self.metadata,
-                    BVec::with_capacity_in(self.inputs.capacity(), doc_alloc),
+                tracing::warn!(
+                    %error,
+                    "error embedding batch of documents, retrying one by one"
                 );
-                self.inputs.clear();
-                return Err(self.on_embed.process_embedding_error(
-                    error,
-                    self.embedder_name,
-                    unused_vectors_distribution,
-                    metadata,
-                ));
+                // retry with one call per input
+                for (metadata, input) in self.metadata.iter().copied().zip(self.inputs.chunks(1)) {
+                    match I::embed_ref(input, self.embedder, self.threads, self.embedder_stats) {
+                        Ok(mut embeddings) => {
+                            let Some(embedding) = embeddings.pop() else {
+                                continue;
+                            };
+                            self.on_embed.process_embedding_response(EmbeddingResponse {
+                                metadata,
+                                embedding: Some(embedding),
+                            })
+                        }
+                        Err(err) => {
+                            tracing::warn!(
+                                docid = metadata.external_docid,
+                                %err,
+                                "error embedding document"
+                            );
+                        }
+                    }
+                }
             }
         };
         self.inputs.clear();
         self.metadata.clear();
-        res
     }
 
     pub(crate) fn embedder_name(&self) -> &'doc str {