diff --git a/Cargo.lock b/Cargo.lock index e73f48127..ddd6a1e96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,7 +27,7 @@ checksum = "f9e772b3bcafe335042b5db010ab7c09013dad6eac4915c91d8d50902769f331" dependencies = [ "actix-utils", "actix-web", - "derive_more", + "derive_more 0.99.17", "futures-util", "log", "once_cell", @@ -36,24 +36,24 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.9.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d48f96fc3003717aeb9856ca3d02a8c7de502667ad76eeacd830b48d2e91fac4" +checksum = "0fa882656b67966045e4152c634051e70346939fced7117d5f0b52146a7c74c9" dependencies = [ "actix-codec", "actix-rt", "actix-service", "actix-tls", "actix-utils", - "ahash 0.8.11", "base64 0.22.1", "bitflags 2.9.0", - "brotli", + "brotli 7.0.0", "bytes", "bytestring", - "derive_more", + "derive_more 2.0.1", "encoding_rs", "flate2", + "foldhash", "futures-core", "h2 0.3.26", "http 0.2.11", @@ -65,7 +65,7 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "rand", + "rand 0.9.0", "sha1", "smallvec", "tokio", @@ -168,9 +168,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.9.0" +version = "4.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9180d76e5cc7ccbc4d60a506f2c727730b154010262df5b910eb17dbe4b8cb38" +checksum = "92a9ac2e5a0d74d27551b65d10e2ed5c04e9935308c6b07c5f1325223dedfd00" dependencies = [ "actix-codec", "actix-http", @@ -182,13 +182,13 @@ dependencies = [ "actix-tls", "actix-utils", "actix-web-codegen", - "ahash 0.8.11", "bytes", "bytestring", "cfg-if", "cookie", - "derive_more", + "derive_more 2.0.1", "encoding_rs", + "foldhash", "futures-core", "futures-util", "impl-more", @@ -205,6 +205,7 @@ dependencies = [ "smallvec", "socket2 0.5.5", "time", + "tracing", "url", ] @@ -258,7 +259,7 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ - "getrandom", + "getrandom 0.2.15", "once_cell", "version_check", ] @@ -271,10 +272,10 @@ checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "const-random", - "getrandom", + "getrandom 0.2.15", "once_cell", "version_check", - "zerocopy", + "zerocopy 0.7.32", ] [[package]] @@ -405,7 +406,7 @@ dependencies = [ "nohash", "ordered-float", "page_size", - "rand", + "rand 0.8.5", "rayon", "roaring", "tempfile", @@ -498,8 +499,8 @@ dependencies = [ "memmap2", "milli", "mimalloc", - "rand", - "rand_chacha", + "rand 0.8.5", + "rand_chacha 0.3.1", "reqwest", "roaring", "serde_json", @@ -654,6 +655,17 @@ dependencies = [ "brotli-decompressor", ] +[[package]] +name = "brotli" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + [[package]] name = "brotli-decompressor" version = "4.0.1" @@ -832,7 +844,7 @@ dependencies = [ "memmap2", "num-traits", "num_cpus", - "rand", + "rand 0.8.5", "rand_distr", "rayon", "safetensors", @@ -878,7 +890,7 @@ dependencies = [ "candle-nn", "fancy-regex", "num-traits", - "rand", + "rand 0.8.5", "rayon", "serde", "serde_json", @@ -1143,7 +1155,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom", + "getrandom 0.2.15", "once_cell", "tiny-keccak", ] @@ -1544,6 +1556,27 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_more" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "unicode-xid", +] + [[package]] name = "deserr" version = "0.6.3" @@ -2216,10 +2249,22 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + [[package]] name = "gimli" version = "0.27.3" @@ -2312,7 +2357,7 @@ dependencies = [ "cfg-if", "crunchy", "num-traits", - "rand", + "rand 0.8.5", "rand_distr", ] @@ -2442,7 +2487,7 @@ dependencies = [ "http 1.2.0", "indicatif", "log", - "rand", + "rand 0.8.5", "serde", "serde_json", "thiserror 1.0.69", @@ -3578,7 +3623,7 @@ dependencies = [ "actix-web", "anyhow", "async-trait", - "brotli", + "brotli 6.0.0", "bstr", "build-info", "byte-unit", @@ -3619,7 +3664,7 @@ dependencies = [ "pin-project-lite", "platform-dirs", "prometheus", - "rand", + "rand 0.8.5", "rayon", "regex", "reqwest", @@ -3668,7 +3713,7 @@ dependencies = [ "hmac", "maplit", "meilisearch-types", - "rand", + "rand 0.8.5", "roaring", "serde", "serde_json", @@ -3798,7 +3843,7 @@ dependencies = [ "obkv", "once_cell", "ordered-float", - "rand", + "rand 0.8.5", "rayon", "rayon-par-bridge", "rhai", @@ -3882,7 +3927,7 @@ checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.48.0", ] @@ -3893,7 +3938,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -4338,7 +4383,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ "phf_shared", - "rand", + "rand 0.8.5", ] [[package]] @@ -4605,7 +4650,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ "bytes", - "rand", + "rand 0.8.5", "ring", "rustc-hash 2.1.0", "rustls", @@ -4637,6 +4682,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + [[package]] name = "radium" version = "0.7.0" @@ -4650,8 +4701,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", + "zerocopy 0.8.24", ] [[package]] @@ -4661,7 +4723,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -4670,7 +4742,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.2", ] [[package]] @@ -4680,7 +4761,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" dependencies = [ "num-traits", - "rand", + "rand 0.8.5", ] [[package]] @@ -4762,7 +4843,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom", + "getrandom 0.2.15", "redox_syscall 0.2.16", "thiserror 1.0.69", ] @@ -4892,7 +4973,7 @@ checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", "untrusted", "windows-sys 0.52.0", @@ -4960,7 +5041,7 @@ dependencies = [ "borsh", "bytes", "num-traits", - "rand", + "rand 0.8.5", "rkyv", "serde", "serde_json", @@ -5576,7 +5657,7 @@ checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" dependencies = [ "cfg-if", "fastrand", - "getrandom", + "getrandom 0.2.15", "once_cell", "rustix", "windows-sys 0.52.0", @@ -5751,7 +5832,7 @@ dependencies = [ "aho-corasick", "derive_builder 0.12.0", "esaxx-rs", - "getrandom", + "getrandom 0.2.15", "itertools 0.12.1", "lazy_static", "log", @@ -5759,7 +5840,7 @@ dependencies = [ "monostate", "onig", "paste", - "rand", + "rand 0.8.5", "rayon", "rayon-cond", "regex", @@ -6122,6 +6203,12 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "unicode_categories" version = "0.1.1" @@ -6238,7 +6325,7 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ - "getrandom", + "getrandom 0.2.15", "serde", ] @@ -6334,6 +6421,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasm-bindgen" version = "0.2.92" @@ -6803,6 +6899,15 @@ dependencies = [ "url", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags 2.9.0", +] + [[package]] name = "write16" version = "1.0.0" @@ -6905,7 +7010,16 @@ version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" dependencies = [ - "zerocopy-derive", + "zerocopy-derive 0.7.32", +] + +[[package]] +name = "zerocopy" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +dependencies = [ + "zerocopy-derive 0.8.24", ] [[package]] @@ -6919,6 +7033,17 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "zerocopy-derive" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "zerofrom" version = "0.1.3" @@ -7017,7 +7142,7 @@ dependencies = [ "lzma-rs", "memchr", "pbkdf2", - "rand", + "rand 0.8.5", "sha1", "thiserror 2.0.9", "time", diff --git a/crates/dump/src/lib.rs b/crates/dump/src/lib.rs index 4e2d6ac2f..d61305c0a 100644 --- a/crates/dump/src/lib.rs +++ b/crates/dump/src/lib.rs @@ -213,6 +213,7 @@ impl From for KindDump { KindDump::DumpCreation { keys, instance_uid } } KindWithContent::SnapshotCreation => KindDump::SnapshotCreation, + KindWithContent::SnapshotCreationWithParams { .. } => KindDump::SnapshotCreation, KindWithContent::UpgradeDatabase { from: version } => { KindDump::UpgradeDatabase { from: version } } diff --git a/crates/index-scheduler/src/dump.rs b/crates/index-scheduler/src/dump.rs index ca26e50c8..7f2704352 100644 --- a/crates/index-scheduler/src/dump.rs +++ b/crates/index-scheduler/src/dump.rs @@ -210,7 +210,10 @@ impl<'a> Dump<'a> { KindDump::DumpCreation { keys, instance_uid } => { KindWithContent::DumpCreation { keys, instance_uid } } - KindDump::SnapshotCreation => KindWithContent::SnapshotCreation, + KindDump::SnapshotCreation => KindWithContent::SnapshotCreationWithParams { + compaction: false, + compression: true, + }, KindDump::UpgradeDatabase { from } => KindWithContent::UpgradeDatabase { from }, }, }; diff --git a/crates/index-scheduler/src/scheduler/autobatcher.rs b/crates/index-scheduler/src/scheduler/autobatcher.rs index 605bf80dd..fee9f93cf 100644 --- a/crates/index-scheduler/src/scheduler/autobatcher.rs +++ b/crates/index-scheduler/src/scheduler/autobatcher.rs @@ -5,9 +5,10 @@ tasks affecting a single index into a [batch](crate::batch::Batch). The main function of the autobatcher is [`next_autobatch`]. */ -use meilisearch_types::tasks::TaskId; use std::ops::ControlFlow::{self, Break, Continue}; +use meilisearch_types::tasks::TaskId; + use crate::KindWithContent; /// Succinctly describes a task's [`Kind`](meilisearch_types::tasks::Kind) @@ -71,7 +72,8 @@ impl From for AutobatchKind { | KindWithContent::TaskDeletion { .. } | KindWithContent::DumpCreation { .. } | KindWithContent::UpgradeDatabase { .. } - | KindWithContent::SnapshotCreation => { + | KindWithContent::SnapshotCreation + | KindWithContent::SnapshotCreationWithParams { .. } => { panic!("The autobatcher should never be called with tasks that don't apply to an index.") } } diff --git a/crates/index-scheduler/src/scheduler/create_batch.rs b/crates/index-scheduler/src/scheduler/create_batch.rs index 10f480d12..b0a3c10c0 100644 --- a/crates/index-scheduler/src/scheduler/create_batch.rs +++ b/crates/index-scheduler/src/scheduler/create_batch.rs @@ -23,7 +23,11 @@ pub(crate) enum Batch { task: Task, }, TaskDeletions(Vec), - SnapshotCreation(Vec), + SnapshotCreation { + tasks: Vec, + compression: bool, + compaction: bool, + }, Dump(Task), IndexOperation { op: IndexOperation, @@ -106,7 +110,7 @@ impl Batch { | Batch::IndexUpdate { task, .. } => { RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap() } - Batch::SnapshotCreation(tasks) + Batch::SnapshotCreation { tasks, .. } | Batch::TaskDeletions(tasks) | Batch::UpgradeDatabase { tasks } | Batch::IndexDeletion { tasks, .. } => { @@ -140,7 +144,7 @@ impl Batch { match self { TaskCancelation { .. } | TaskDeletions(_) - | SnapshotCreation(_) + | SnapshotCreation { .. } | Dump(_) | UpgradeDatabase { .. } | IndexSwap { .. } => None, @@ -160,7 +164,7 @@ impl fmt::Display for Batch { match self { Batch::TaskCancelation { .. } => f.write_str("TaskCancelation")?, Batch::TaskDeletions(_) => f.write_str("TaskDeletion")?, - Batch::SnapshotCreation(_) => f.write_str("SnapshotCreation")?, + Batch::SnapshotCreation { .. } => f.write_str("SnapshotCreation")?, Batch::Dump(_) => f.write_str("Dump")?, Batch::IndexOperation { op, .. } => write!(f, "{op}")?, Batch::IndexCreation { .. } => f.write_str("IndexCreation")?, @@ -478,7 +482,17 @@ impl IndexScheduler { if !to_snapshot.is_empty() { let mut tasks = self.queue.tasks.get_existing_tasks(rtxn, to_snapshot)?; current_batch.processing(&mut tasks); - return Ok(Some((Batch::SnapshotCreation(tasks), current_batch))); + let (compaction, compression) = match &tasks.last().unwrap().kind { + KindWithContent::SnapshotCreation => (false, true), + KindWithContent::SnapshotCreationWithParams { compaction, compression } => { + (*compaction, *compression) + } + _ => unreachable!(), + }; + return Ok(Some(( + Batch::SnapshotCreation { compaction, compression, tasks }, + current_batch, + ))); } // 4. we batch the dumps. diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index 8f3987bf6..e5bd91e3c 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -117,9 +117,9 @@ impl IndexScheduler { } Ok((tasks, None)) } - Batch::SnapshotCreation(tasks) => { - self.process_snapshot(progress, tasks).map(|tasks| (tasks, None)) - } + Batch::SnapshotCreation { tasks, compression, compaction } => self + .process_snapshot(progress, tasks, compaction, compression) + .map(|tasks| (tasks, None)), Batch::Dump(task) => { self.process_dump_creation(progress, task).map(|tasks| (tasks, None)) } diff --git a/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs b/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs index f1bafed01..69745a9c6 100644 --- a/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs +++ b/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs @@ -15,8 +15,13 @@ impl IndexScheduler { &self, progress: Progress, mut tasks: Vec, + compaction: bool, + compression: bool, ) -> Result> { + tracing::debug!(compaction, compression, "process snapshot"); progress.update_progress(SnapshotCreationProgress::StartTheSnapshotCreation); + let compaction = + if compaction { CompactionOption::Enabled } else { CompactionOption::Disabled }; fs::create_dir_all(&self.scheduler.snapshots_path)?; let temp_snapshot_dir = tempfile::tempdir()?; @@ -41,7 +46,7 @@ impl IndexScheduler { progress.update_progress(SnapshotCreationProgress::SnapshotTheIndexScheduler); let dst = temp_snapshot_dir.path().join("tasks"); fs::create_dir_all(&dst)?; - self.env.copy_to_path(dst.join("data.mdb"), CompactionOption::Enabled)?; + self.env.copy_to_path(dst.join("data.mdb"), compaction)?; // 2.2 Create a read transaction on the index-scheduler let rtxn = self.env.read_txn()?; @@ -80,7 +85,7 @@ impl IndexScheduler { let dst = temp_snapshot_dir.path().join("indexes").join(uuid.to_string()); fs::create_dir_all(&dst)?; index - .copy_to_path(dst.join("data.mdb"), CompactionOption::Enabled) + .copy_to_path(dst.join("data.mdb"), compaction) .map_err(|e| Error::from_milli(e, Some(name.to_string())))?; } @@ -103,7 +108,7 @@ impl IndexScheduler { // 5.2 Tarball the content of the snapshot in a tempfile with a .snapshot extension let snapshot_path = self.scheduler.snapshots_path.join(format!("{}.snapshot", db_name)); let temp_snapshot_file = tempfile::NamedTempFile::new_in(&self.scheduler.snapshots_path)?; - compression::to_tar_gz(temp_snapshot_dir.path(), temp_snapshot_file.path())?; + compression::to_tar_gz(temp_snapshot_dir.path(), temp_snapshot_file.path(), compression)?; let file = temp_snapshot_file.persist(snapshot_path)?; // 5.3 Change the permission to make the snapshot readonly diff --git a/crates/index-scheduler/src/utils.rs b/crates/index-scheduler/src/utils.rs index 42bf253ad..11e4759a9 100644 --- a/crates/index-scheduler/src/utils.rs +++ b/crates/index-scheduler/src/utils.rs @@ -266,6 +266,7 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) { | K::DumpCreation { .. } | K::UpgradeDatabase { .. } | K::SnapshotCreation => (), + K::SnapshotCreationWithParams { .. } => (), }; if let Some(Details::IndexSwap { swaps }) = &mut task.details { for IndexSwap { indexes: (lhs, rhs) } in swaps.iter_mut() { diff --git a/crates/meilisearch-types/src/compression.rs b/crates/meilisearch-types/src/compression.rs index 1d364b815..b4b303c30 100644 --- a/crates/meilisearch-types/src/compression.rs +++ b/crates/meilisearch-types/src/compression.rs @@ -7,9 +7,14 @@ use flate2::write::GzEncoder; use flate2::Compression; use tar::{Archive, Builder}; -pub fn to_tar_gz(src: impl AsRef, dest: impl AsRef) -> anyhow::Result<()> { +pub fn to_tar_gz( + src: impl AsRef, + dest: impl AsRef, + compression: bool, +) -> anyhow::Result<()> { + let compression = if compression { Compression::default() } else { Compression::none() }; let mut f = File::create(dest)?; - let gz_encoder = GzEncoder::new(&mut f, Compression::default()); + let gz_encoder = GzEncoder::new(&mut f, compression); let mut tar_encoder = Builder::new(gz_encoder); tar_encoder.append_dir_all(".", src)?; let gz_encoder = tar_encoder.into_inner()?; diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index 859563d8a..3b1f7eda1 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -328,6 +328,7 @@ InvalidSettingsDictionary , InvalidRequest , BAD_REQUEST ; InvalidSettingsSynonyms , InvalidRequest , BAD_REQUEST ; InvalidSettingsTypoTolerance , InvalidRequest , BAD_REQUEST ; InvalidSettingsLocalizedAttributes , InvalidRequest , BAD_REQUEST ; +InvalidSnapshotOptions , InvalidRequest , BAD_REQUEST ; InvalidState , Internal , INTERNAL_SERVER_ERROR ; InvalidStoreFile , Internal , INTERNAL_SERVER_ERROR ; InvalidSwapDuplicateIndexFound , InvalidRequest , BAD_REQUEST ; diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index 6b237ee1f..e72e0fde0 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -48,6 +48,7 @@ impl Task { match &self.kind { DumpCreation { .. } | SnapshotCreation + | SnapshotCreationWithParams { .. } | TaskCancelation { .. } | TaskDeletion { .. } | UpgradeDatabase { .. } @@ -86,6 +87,7 @@ impl Task { | KindWithContent::TaskDeletion { .. } | KindWithContent::DumpCreation { .. } | KindWithContent::SnapshotCreation + | KindWithContent::SnapshotCreationWithParams { .. } | KindWithContent::UpgradeDatabase { .. } => None, } } @@ -152,6 +154,10 @@ pub enum KindWithContent { instance_uid: Option, }, SnapshotCreation, + SnapshotCreationWithParams { + compaction: bool, + compression: bool, + }, UpgradeDatabase { from: (u32, u32, u32), }, @@ -180,6 +186,7 @@ impl KindWithContent { KindWithContent::TaskDeletion { .. } => Kind::TaskDeletion, KindWithContent::DumpCreation { .. } => Kind::DumpCreation, KindWithContent::SnapshotCreation => Kind::SnapshotCreation, + KindWithContent::SnapshotCreationWithParams { .. } => Kind::SnapshotCreation, KindWithContent::UpgradeDatabase { .. } => Kind::UpgradeDatabase, } } @@ -190,6 +197,7 @@ impl KindWithContent { match self { DumpCreation { .. } | SnapshotCreation + | SnapshotCreationWithParams { .. } | TaskCancelation { .. } | TaskDeletion { .. } | UpgradeDatabase { .. } => vec![], @@ -269,6 +277,7 @@ impl KindWithContent { }), KindWithContent::DumpCreation { .. } => Some(Details::Dump { dump_uid: None }), KindWithContent::SnapshotCreation => None, + KindWithContent::SnapshotCreationWithParams { .. } => None, KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase { from: (from.0, from.1, from.2), to: ( @@ -335,6 +344,7 @@ impl KindWithContent { }), KindWithContent::DumpCreation { .. } => Some(Details::Dump { dump_uid: None }), KindWithContent::SnapshotCreation => None, + KindWithContent::SnapshotCreationWithParams { .. } => None, KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase { from: *from, to: ( @@ -383,6 +393,7 @@ impl From<&KindWithContent> for Option
{ }), KindWithContent::DumpCreation { .. } => Some(Details::Dump { dump_uid: None }), KindWithContent::SnapshotCreation => None, + KindWithContent::SnapshotCreationWithParams { .. } => None, KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase { from: *from, to: ( diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index 2a32a6be8..7794ed9f8 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -315,9 +315,14 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Arc< .name(String::from("register-snapshot-tasks")) .spawn(move || loop { thread::sleep(snapshot_delay); - if let Err(e) = - index_scheduler.register(KindWithContent::SnapshotCreation, None, false) - { + if let Err(e) = index_scheduler.register( + KindWithContent::SnapshotCreationWithParams { + compaction: false, + compression: true, + }, + None, + false, + ) { error!("Error while registering snapshot: {}", e); } }) diff --git a/crates/meilisearch/src/routes/snapshot.rs b/crates/meilisearch/src/routes/snapshot.rs index de7ecc37f..c31cd6fdc 100644 --- a/crates/meilisearch/src/routes/snapshot.rs +++ b/crates/meilisearch/src/routes/snapshot.rs @@ -1,6 +1,8 @@ use actix_web::web::Data; -use actix_web::{web, HttpRequest, HttpResponse}; +use actix_web::{web, FromRequest, HttpRequest, HttpResponse}; +use deserr::actix_web::AwebJson; use index_scheduler::IndexScheduler; +use meilisearch_types::deserr::DeserrJsonError; use meilisearch_types::error::ResponseError; use meilisearch_types::tasks::KindWithContent; use tracing::debug; @@ -42,6 +44,7 @@ crate::empty_analytics!(SnapshotAnalytics, "Snapshot Created"); path = "", tag = "Snapshots", security(("Bearer" = ["snapshots.create", "snapshots.*", "*"])), + request_body = SnapshotOptions, responses( (status = 202, description = "Snapshot is being created", body = SummarizedTaskView, content_type = "application/json", example = json!( { @@ -64,13 +67,29 @@ crate::empty_analytics!(SnapshotAnalytics, "Snapshot Created"); )] pub async fn create_snapshot( index_scheduler: GuardedData, Data>, + snapshot_options: Option, req: HttpRequest, opt: web::Data, analytics: web::Data, ) -> Result { analytics.publish(SnapshotAnalytics::default(), &req); - let task = KindWithContent::SnapshotCreation; + let task = match snapshot_options { + Some(snapshot_options) if !snapshot_options.is_empty() => { + let mut payload = actix_web::dev::Payload::from(snapshot_options); + let snapshot_options: AwebJson = + match AwebJson::from_request(&req, &mut payload).await { + Ok(snapshot_options) => snapshot_options, + Err(error) => { + return Err(ResponseError::from_msg(format!("{error}\n - note: POST /snapshots without a body to use default parameters"), meilisearch_types::error::Code::InvalidSnapshotOptions)); + } + }; + let SnapshotOptions { compaction, compression } = snapshot_options.into_inner(); + + KindWithContent::SnapshotCreationWithParams { compaction, compression } + } + _ => KindWithContent::SnapshotCreationWithParams { compaction: false, compression: true }, + }; let uid = get_task_id(&req, &opt)?; let dry_run = is_dry_run(&req, &opt)?; let task: SummarizedTaskView = @@ -81,3 +100,12 @@ pub async fn create_snapshot( debug!(returns = ?task, "Create snapshot"); Ok(HttpResponse::Accepted().json(task)) } + +#[derive(Clone, Copy, deserr::Deserr, utoipa::ToSchema)] +#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +struct SnapshotOptions { + #[deserr(default)] + compaction: bool, + #[deserr(default)] + compression: bool, +}