Snapshot takes compression and compaction options

crates/dump/src/lib.rs

@@ -213,6 +213,7 @@ impl From<KindWithContent> for KindDump {
                 KindDump::DumpCreation { keys, instance_uid }
             }
             KindWithContent::SnapshotCreation => KindDump::SnapshotCreation,
+            KindWithContent::SnapshotCreationWithParams { .. } => KindDump::SnapshotCreation,
             KindWithContent::UpgradeDatabase { from: version } => {
                 KindDump::UpgradeDatabase { from: version }
             }

crates/index-scheduler/src/dump.rs

@@ -210,7 +210,10 @@ impl<'a> Dump<'a> {
                 KindDump::DumpCreation { keys, instance_uid } => {
                     KindWithContent::DumpCreation { keys, instance_uid }
                 }
-                KindDump::SnapshotCreation => KindWithContent::SnapshotCreation,
+                KindDump::SnapshotCreation => KindWithContent::SnapshotCreationWithParams {
+                    compaction: false,
+                    compression: true,
+                },
                 KindDump::UpgradeDatabase { from } => KindWithContent::UpgradeDatabase { from },
             },
         };

crates/index-scheduler/src/scheduler/autobatcher.rs

@@ -5,9 +5,10 @@ tasks affecting a single index into a [batch](crate::batch::Batch).
 The main function of the autobatcher is [`next_autobatch`].
 */
 
-use meilisearch_types::tasks::TaskId;
 use std::ops::ControlFlow::{self, Break, Continue};
 
+use meilisearch_types::tasks::TaskId;
+
 use crate::KindWithContent;
 
 /// Succinctly describes a task's [`Kind`](meilisearch_types::tasks::Kind)
@@ -71,7 +72,8 @@ impl From<KindWithContent> for AutobatchKind {
             | KindWithContent::TaskDeletion { .. }
             | KindWithContent::DumpCreation { .. }
             | KindWithContent::UpgradeDatabase { .. }
-            | KindWithContent::SnapshotCreation => {
+            | KindWithContent::SnapshotCreation
+            | KindWithContent::SnapshotCreationWithParams { .. } => {
                 panic!("The autobatcher should never be called with tasks that don't apply to an index.")
             }
         }

crates/index-scheduler/src/scheduler/create_batch.rs

@@ -23,7 +23,11 @@ pub(crate) enum Batch {
         task: Task,
     },
     TaskDeletions(Vec<Task>),
-    SnapshotCreation(Vec<Task>),
+    SnapshotCreation {
+        tasks: Vec<Task>,
+        compression: bool,
+        compaction: bool,
+    },
     Dump(Task),
     IndexOperation {
         op: IndexOperation,
@@ -106,7 +110,7 @@ impl Batch {
             | Batch::IndexUpdate { task, .. } => {
                 RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap()
             }
-            Batch::SnapshotCreation(tasks)
+            Batch::SnapshotCreation { tasks, .. }
             | Batch::TaskDeletions(tasks)
             | Batch::UpgradeDatabase { tasks }
             | Batch::IndexDeletion { tasks, .. } => {
@@ -140,7 +144,7 @@ impl Batch {
         match self {
             TaskCancelation { .. }
             | TaskDeletions(_)
-            | SnapshotCreation(_)
+            | SnapshotCreation { .. }
             | Dump(_)
             | UpgradeDatabase { .. }
             | IndexSwap { .. } => None,
@@ -160,7 +164,7 @@ impl fmt::Display for Batch {
         match self {
             Batch::TaskCancelation { .. } => f.write_str("TaskCancelation")?,
             Batch::TaskDeletions(_) => f.write_str("TaskDeletion")?,
-            Batch::SnapshotCreation(_) => f.write_str("SnapshotCreation")?,
+            Batch::SnapshotCreation { .. } => f.write_str("SnapshotCreation")?,
             Batch::Dump(_) => f.write_str("Dump")?,
             Batch::IndexOperation { op, .. } => write!(f, "{op}")?,
             Batch::IndexCreation { .. } => f.write_str("IndexCreation")?,
@@ -478,7 +482,17 @@ impl IndexScheduler {
         if !to_snapshot.is_empty() {
             let mut tasks = self.queue.tasks.get_existing_tasks(rtxn, to_snapshot)?;
             current_batch.processing(&mut tasks);
-            return Ok(Some((Batch::SnapshotCreation(tasks), current_batch)));
+            let (compaction, compression) = match &tasks.last().unwrap().kind {
+                KindWithContent::SnapshotCreation => (false, true),
+                KindWithContent::SnapshotCreationWithParams { compaction, compression } => {
+                    (*compaction, *compression)
+                }
+                _ => unreachable!(),
+            };
+            return Ok(Some((
+                Batch::SnapshotCreation { compaction, compression, tasks },
+                current_batch,
+            )));
         }
 
         // 4. we batch the dumps.

crates/index-scheduler/src/scheduler/process_batch.rs

@@ -117,9 +117,9 @@ impl IndexScheduler {
                 }
                 Ok((tasks, None))
             }
-            Batch::SnapshotCreation(tasks) => {
-                self.process_snapshot(progress, tasks).map(|tasks| (tasks, None))
-            }
+            Batch::SnapshotCreation { tasks, compression, compaction } => self
+                .process_snapshot(progress, tasks, compaction, compression)
+                .map(|tasks| (tasks, None)),
             Batch::Dump(task) => {
                 self.process_dump_creation(progress, task).map(|tasks| (tasks, None))
             }

crates/index-scheduler/src/scheduler/process_snapshot_creation.rs

@@ -15,8 +15,13 @@ impl IndexScheduler {
         &self,
         progress: Progress,
         mut tasks: Vec<Task>,
+        compaction: bool,
+        compression: bool,
     ) -> Result<Vec<Task>> {
+        tracing::debug!(compaction, compression, "process snapshot");
         progress.update_progress(SnapshotCreationProgress::StartTheSnapshotCreation);
+        let compaction =
+            if compaction { CompactionOption::Enabled } else { CompactionOption::Disabled };
 
         fs::create_dir_all(&self.scheduler.snapshots_path)?;
         let temp_snapshot_dir = tempfile::tempdir()?;
@@ -41,7 +46,7 @@ impl IndexScheduler {
         progress.update_progress(SnapshotCreationProgress::SnapshotTheIndexScheduler);
         let dst = temp_snapshot_dir.path().join("tasks");
         fs::create_dir_all(&dst)?;
-        self.env.copy_to_path(dst.join("data.mdb"), CompactionOption::Enabled)?;
+        self.env.copy_to_path(dst.join("data.mdb"), compaction)?;
 
         // 2.2 Create a read transaction on the index-scheduler
         let rtxn = self.env.read_txn()?;
@@ -80,7 +85,7 @@ impl IndexScheduler {
             let dst = temp_snapshot_dir.path().join("indexes").join(uuid.to_string());
             fs::create_dir_all(&dst)?;
             index
-                .copy_to_path(dst.join("data.mdb"), CompactionOption::Enabled)
+                .copy_to_path(dst.join("data.mdb"), compaction)
                 .map_err(|e| Error::from_milli(e, Some(name.to_string())))?;
         }
 
@@ -103,7 +108,7 @@ impl IndexScheduler {
         // 5.2 Tarball the content of the snapshot in a tempfile with a .snapshot extension
         let snapshot_path = self.scheduler.snapshots_path.join(format!("{}.snapshot", db_name));
         let temp_snapshot_file = tempfile::NamedTempFile::new_in(&self.scheduler.snapshots_path)?;
-        compression::to_tar_gz(temp_snapshot_dir.path(), temp_snapshot_file.path())?;
+        compression::to_tar_gz(temp_snapshot_dir.path(), temp_snapshot_file.path(), compression)?;
         let file = temp_snapshot_file.persist(snapshot_path)?;
 
         // 5.3 Change the permission to make the snapshot readonly

crates/index-scheduler/src/utils.rs

@@ -266,6 +266,7 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) {
         | K::DumpCreation { .. }
         | K::UpgradeDatabase { .. }
         | K::SnapshotCreation => (),
+        K::SnapshotCreationWithParams { .. } => (),
     };
     if let Some(Details::IndexSwap { swaps }) = &mut task.details {
         for IndexSwap { indexes: (lhs, rhs) } in swaps.iter_mut() {

crates/meilisearch-types/src/compression.rs

@@ -7,9 +7,14 @@ use flate2::write::GzEncoder;
 use flate2::Compression;
 use tar::{Archive, Builder};
 
-pub fn to_tar_gz(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> anyhow::Result<()> {
+pub fn to_tar_gz(
+    src: impl AsRef<Path>,
+    dest: impl AsRef<Path>,
+    compression: bool,
+) -> anyhow::Result<()> {
+    let compression = if compression { Compression::default() } else { Compression::none() };
     let mut f = File::create(dest)?;
-    let gz_encoder = GzEncoder::new(&mut f, Compression::default());
+    let gz_encoder = GzEncoder::new(&mut f, compression);
     let mut tar_encoder = Builder::new(gz_encoder);
     tar_encoder.append_dir_all(".", src)?;
     let gz_encoder = tar_encoder.into_inner()?;

crates/meilisearch-types/src/error.rs

@@ -328,6 +328,7 @@ InvalidSettingsDictionary             , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsSynonyms               , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsTypoTolerance          , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsLocalizedAttributes    , InvalidRequest       , BAD_REQUEST ;
+InvalidSnapshotOptions                , InvalidRequest       , BAD_REQUEST ;
 InvalidState                          , Internal             , INTERNAL_SERVER_ERROR ;
 InvalidStoreFile                      , Internal             , INTERNAL_SERVER_ERROR ;
 InvalidSwapDuplicateIndexFound        , InvalidRequest       , BAD_REQUEST ;

crates/meilisearch-types/src/tasks.rs

@@ -48,6 +48,7 @@ impl Task {
         match &self.kind {
             DumpCreation { .. }
             | SnapshotCreation
+            | SnapshotCreationWithParams { .. }
             | TaskCancelation { .. }
             | TaskDeletion { .. }
             | UpgradeDatabase { .. }
@@ -86,6 +87,7 @@ impl Task {
             | KindWithContent::TaskDeletion { .. }
             | KindWithContent::DumpCreation { .. }
             | KindWithContent::SnapshotCreation
+            | KindWithContent::SnapshotCreationWithParams { .. }
             | KindWithContent::UpgradeDatabase { .. } => None,
         }
     }
@@ -152,6 +154,10 @@ pub enum KindWithContent {
         instance_uid: Option<InstanceUid>,
     },
     SnapshotCreation,
+    SnapshotCreationWithParams {
+        compaction: bool,
+        compression: bool,
+    },
     UpgradeDatabase {
         from: (u32, u32, u32),
     },
@@ -180,6 +186,7 @@ impl KindWithContent {
             KindWithContent::TaskDeletion { .. } => Kind::TaskDeletion,
             KindWithContent::DumpCreation { .. } => Kind::DumpCreation,
             KindWithContent::SnapshotCreation => Kind::SnapshotCreation,
+            KindWithContent::SnapshotCreationWithParams { .. } => Kind::SnapshotCreation,
             KindWithContent::UpgradeDatabase { .. } => Kind::UpgradeDatabase,
         }
     }
@@ -190,6 +197,7 @@ impl KindWithContent {
         match self {
             DumpCreation { .. }
             | SnapshotCreation
+            | SnapshotCreationWithParams { .. }
             | TaskCancelation { .. }
             | TaskDeletion { .. }
             | UpgradeDatabase { .. } => vec![],
@@ -269,6 +277,7 @@ impl KindWithContent {
             }),
             KindWithContent::DumpCreation { .. } => Some(Details::Dump { dump_uid: None }),
             KindWithContent::SnapshotCreation => None,
+            KindWithContent::SnapshotCreationWithParams { .. } => None,
             KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase {
                 from: (from.0, from.1, from.2),
                 to: (
@@ -335,6 +344,7 @@ impl KindWithContent {
             }),
             KindWithContent::DumpCreation { .. } => Some(Details::Dump { dump_uid: None }),
             KindWithContent::SnapshotCreation => None,
+            KindWithContent::SnapshotCreationWithParams { .. } => None,
             KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase {
                 from: *from,
                 to: (
@@ -383,6 +393,7 @@ impl From<&KindWithContent> for Option<Details> {
             }),
             KindWithContent::DumpCreation { .. } => Some(Details::Dump { dump_uid: None }),
             KindWithContent::SnapshotCreation => None,
+            KindWithContent::SnapshotCreationWithParams { .. } => None,
             KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase {
                 from: *from,
                 to: (

crates/meilisearch/src/lib.rs

@@ -315,9 +315,14 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc<IndexScheduler>, Arc<
             .name(String::from("register-snapshot-tasks"))
             .spawn(move || loop {
                 thread::sleep(snapshot_delay);
-                if let Err(e) =
-                    index_scheduler.register(KindWithContent::SnapshotCreation, None, false)
-                {
+                if let Err(e) = index_scheduler.register(
+                    KindWithContent::SnapshotCreationWithParams {
+                        compaction: false,
+                        compression: true,
+                    },
+                    None,
+                    false,
+                ) {
                     error!("Error while registering snapshot: {}", e);
                 }
             })

crates/meilisearch/src/routes/snapshot.rs

@@ -1,6 +1,8 @@
 use actix_web::web::Data;
-use actix_web::{web, HttpRequest, HttpResponse};
+use actix_web::{web, FromRequest, HttpRequest, HttpResponse};
+use deserr::actix_web::AwebJson;
 use index_scheduler::IndexScheduler;
+use meilisearch_types::deserr::DeserrJsonError;
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::tasks::KindWithContent;
 use tracing::debug;
@@ -42,6 +44,7 @@ crate::empty_analytics!(SnapshotAnalytics, "Snapshot Created");
     path = "",
     tag = "Snapshots",
     security(("Bearer" = ["snapshots.create", "snapshots.*", "*"])),
+    request_body = SnapshotOptions,
     responses(
         (status = 202, description = "Snapshot is being created", body = SummarizedTaskView, content_type = "application/json", example = json!(
             {
@@ -64,13 +67,29 @@ crate::empty_analytics!(SnapshotAnalytics, "Snapshot Created");
 )]
 pub async fn create_snapshot(
     index_scheduler: GuardedData<ActionPolicy<{ actions::SNAPSHOTS_CREATE }>, Data<IndexScheduler>>,
+    snapshot_options: Option<actix_web::web::Bytes>,
     req: HttpRequest,
     opt: web::Data<Opt>,
     analytics: web::Data<Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     analytics.publish(SnapshotAnalytics::default(), &req);
 
-    let task = KindWithContent::SnapshotCreation;
+    let task = match snapshot_options {
+        Some(snapshot_options) if !snapshot_options.is_empty() => {
+            let mut payload = actix_web::dev::Payload::from(snapshot_options);
+            let snapshot_options: AwebJson<SnapshotOptions, DeserrJsonError> =
+                match AwebJson::from_request(&req, &mut payload).await {
+                    Ok(snapshot_options) => snapshot_options,
+                    Err(error) => {
+                        return Err(ResponseError::from_msg(format!("{error}\n  - note: POST /snapshots without a body to use default parameters"), meilisearch_types::error::Code::InvalidSnapshotOptions));
+                    }
+                };
+            let SnapshotOptions { compaction, compression } = snapshot_options.into_inner();
+
+            KindWithContent::SnapshotCreationWithParams { compaction, compression }
+        }
+        _ => KindWithContent::SnapshotCreationWithParams { compaction: false, compression: true },
+    };
     let uid = get_task_id(&req, &opt)?;
     let dry_run = is_dry_run(&req, &opt)?;
     let task: SummarizedTaskView =
@@ -81,3 +100,12 @@ pub async fn create_snapshot(
     debug!(returns = ?task, "Create snapshot");
     Ok(HttpResponse::Accepted().json(task))
 }
+
+#[derive(Clone, Copy, deserr::Deserr, utoipa::ToSchema)]
+#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
+struct SnapshotOptions {
+    #[deserr(default)]
+    compaction: bool,
+    #[deserr(default)]
+    compression: bool,
+}