make scheduler accept multiple batch handlers

meilisearch-lib/src/dump/actor.rs

@@ -0,0 +1,191 @@
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use async_stream::stream;
+use futures::{lock::Mutex, stream::StreamExt};
+use log::{error, trace};
+use time::macros::format_description;
+use time::OffsetDateTime;
+use tokio::sync::{mpsc, oneshot, RwLock};
+
+use super::error::{DumpActorError, Result};
+use super::{DumpInfo, DumpJob, DumpMsg, DumpStatus};
+use crate::tasks::Scheduler;
+use crate::update_file_store::UpdateFileStore;
+
+pub const CONCURRENT_DUMP_MSG: usize = 10;
+
+pub struct DumpActor {
+    inbox: Option<mpsc::Receiver<DumpMsg>>,
+    update_file_store: UpdateFileStore,
+    scheduler: Arc<RwLock<Scheduler>>,
+    dump_path: PathBuf,
+    analytics_path: PathBuf,
+    lock: Arc<Mutex<()>>,
+    dump_infos: Arc<RwLock<HashMap<String, DumpInfo>>>,
+    update_db_size: usize,
+    index_db_size: usize,
+}
+
+/// Generate uid from creation date
+fn generate_uid() -> String {
+    OffsetDateTime::now_utc()
+        .format(format_description!(
+            "[year repr:full][month repr:numerical][day padding:zero]-[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]"
+        ))
+        .unwrap()
+}
+
+impl DumpActor {
+    pub fn new(
+        inbox: mpsc::Receiver<DumpMsg>,
+        update_file_store: UpdateFileStore,
+        scheduler: Arc<RwLock<Scheduler>>,
+        dump_path: impl AsRef<Path>,
+        analytics_path: impl AsRef<Path>,
+        index_db_size: usize,
+        update_db_size: usize,
+    ) -> Self {
+        let dump_infos = Arc::new(RwLock::new(HashMap::new()));
+        let lock = Arc::new(Mutex::new(()));
+        Self {
+            inbox: Some(inbox),
+            scheduler,
+            update_file_store,
+            dump_path: dump_path.as_ref().into(),
+            analytics_path: analytics_path.as_ref().into(),
+            dump_infos,
+            lock,
+            index_db_size,
+            update_db_size,
+        }
+    }
+
+    pub async fn run(mut self) {
+        trace!("Started dump actor.");
+
+        let mut inbox = self
+            .inbox
+            .take()
+            .expect("Dump Actor must have a inbox at this point.");
+
+        let stream = stream! {
+            loop {
+                match inbox.recv().await {
+                    Some(msg) => yield msg,
+                    None => break,
+                }
+            }
+        };
+
+        stream
+            .for_each_concurrent(Some(CONCURRENT_DUMP_MSG), |msg| self.handle_message(msg))
+            .await;
+
+        error!("Dump actor stopped.");
+    }
+
+    async fn handle_message(&self, msg: DumpMsg) {
+        use DumpMsg::*;
+
+        match msg {
+            CreateDump { ret } => {
+                let _ = self.handle_create_dump(ret).await;
+            }
+            DumpInfo { ret, uid } => {
+                let _ = ret.send(self.handle_dump_info(uid).await);
+            }
+        }
+    }
+
+    async fn handle_create_dump(&self, ret: oneshot::Sender<Result<DumpInfo>>) {
+        let uid = generate_uid();
+        let info = DumpInfo::new(uid.clone(), DumpStatus::InProgress);
+
+        let _lock = match self.lock.try_lock() {
+            Some(lock) => lock,
+            None => {
+                ret.send(Err(DumpActorError::DumpAlreadyRunning))
+                    .expect("Dump actor is dead");
+                return;
+            }
+        };
+
+        self.dump_infos
+            .write()
+            .await
+            .insert(uid.clone(), info.clone());
+
+        ret.send(Ok(info)).expect("Dump actor is dead");
+
+        let task = DumpJob {
+            dump_path: self.dump_path.clone(),
+            db_path: self.analytics_path.clone(),
+            update_file_store: self.update_file_store.clone(),
+            scheduler: self.scheduler.clone(),
+            uid: uid.clone(),
+            update_db_size: self.update_db_size,
+            index_db_size: self.index_db_size,
+        };
+
+        let task_result = tokio::task::spawn_local(task.run()).await;
+
+        let mut dump_infos = self.dump_infos.write().await;
+        let dump_infos = dump_infos
+            .get_mut(&uid)
+            .expect("dump entry deleted while lock was acquired");
+
+        match task_result {
+            Ok(Ok(())) => {
+                dump_infos.done();
+                trace!("Dump succeed");
+            }
+            Ok(Err(e)) => {
+                dump_infos.with_error(e.to_string());
+                error!("Dump failed: {}", e);
+            }
+            Err(_) => {
+                dump_infos.with_error("Unexpected error while performing dump.".to_string());
+                error!("Dump panicked. Dump status set to failed");
+            }
+        };
+    }
+
+    async fn handle_dump_info(&self, uid: String) -> Result<DumpInfo> {
+        match self.dump_infos.read().await.get(&uid) {
+            Some(info) => Ok(info.clone()),
+            _ => Err(DumpActorError::DumpDoesNotExist(uid)),
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_generate_uid() {
+        let current = OffsetDateTime::now_utc();
+
+        let uid = generate_uid();
+        let (date, time) = uid.split_once('-').unwrap();
+
+        let date = time::Date::parse(
+            date,
+            &format_description!("[year repr:full][month repr:numerical][day padding:zero]"),
+        )
+        .unwrap();
+        let time = time::Time::parse(
+            time,
+            &format_description!(
+                "[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]"
+            ),
+        )
+        .unwrap();
+        let datetime = time::PrimitiveDateTime::new(date, time);
+        let datetime = datetime.assume_utc();
+
+        assert!(current - datetime < time::Duration::SECOND);
+    }
+}

meilisearch-lib/src/dump/compat/mod.rs

@@ -0,0 +1,16 @@
+pub mod v2;
+pub mod v3;
+
+/// Parses the v1 version of the Asc ranking rules `asc(price)`and returns the field name.
+pub fn asc_ranking_rule(text: &str) -> Option<&str> {
+    text.split_once("asc(")
+        .and_then(|(_, tail)| tail.rsplit_once(')'))
+        .map(|(field, _)| field)
+}
+
+/// Parses the v1 version of the Desc ranking rules `desc(price)`and returns the field name.
+pub fn desc_ranking_rule(text: &str) -> Option<&str> {
+    text.split_once("desc(")
+        .and_then(|(_, tail)| tail.rsplit_once(')'))
+        .map(|(field, _)| field)
+}

meilisearch-lib/src/dump/compat/v2.rs

@@ -0,0 +1,152 @@
+use anyhow::bail;
+use meilisearch_error::Code;
+use milli::update::IndexDocumentsMethod;
+use serde::{Deserialize, Serialize};
+use time::OffsetDateTime;
+use uuid::Uuid;
+
+use crate::index::{Settings, Unchecked};
+
+#[derive(Serialize, Deserialize)]
+pub struct UpdateEntry {
+    pub uuid: Uuid,
+    pub update: UpdateStatus,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum UpdateFormat {
+    Json,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct DocumentAdditionResult {
+    pub nb_documents: usize,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum UpdateResult {
+    DocumentsAddition(DocumentAdditionResult),
+    DocumentDeletion { deleted: u64 },
+    Other,
+}
+
+#[allow(clippy::large_enum_variant)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum UpdateMeta {
+    DocumentsAddition {
+        method: IndexDocumentsMethod,
+        format: UpdateFormat,
+        primary_key: Option<String>,
+    },
+    ClearDocuments,
+    DeleteDocuments {
+        ids: Vec<String>,
+    },
+    Settings(Settings<Unchecked>),
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct Enqueued {
+    pub update_id: u64,
+    pub meta: UpdateMeta,
+    #[serde(with = "time::serde::rfc3339")]
+    pub enqueued_at: OffsetDateTime,
+    pub content: Option<Uuid>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct Processed {
+    pub success: UpdateResult,
+    #[serde(with = "time::serde::rfc3339")]
+    pub processed_at: OffsetDateTime,
+    #[serde(flatten)]
+    pub from: Processing,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct Processing {
+    #[serde(flatten)]
+    pub from: Enqueued,
+    #[serde(with = "time::serde::rfc3339")]
+    pub started_processing_at: OffsetDateTime,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct Aborted {
+    #[serde(flatten)]
+    pub from: Enqueued,
+    #[serde(with = "time::serde::rfc3339")]
+    pub aborted_at: OffsetDateTime,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Failed {
+    #[serde(flatten)]
+    pub from: Processing,
+    pub error: ResponseError,
+    #[serde(with = "time::serde::rfc3339")]
+    pub failed_at: OffsetDateTime,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(tag = "status", rename_all = "camelCase")]
+pub enum UpdateStatus {
+    Processing(Processing),
+    Enqueued(Enqueued),
+    Processed(Processed),
+    Aborted(Aborted),
+    Failed(Failed),
+}
+
+type StatusCode = ();
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct ResponseError {
+    #[serde(skip)]
+    pub code: StatusCode,
+    pub message: String,
+    pub error_code: String,
+    pub error_type: String,
+    pub error_link: String,
+}
+
+pub fn error_code_from_str(s: &str) -> anyhow::Result<Code> {
+    let code = match s {
+        "index_creation_failed" => Code::CreateIndex,
+        "index_already_exists" => Code::IndexAlreadyExists,
+        "index_not_found" => Code::IndexNotFound,
+        "invalid_index_uid" => Code::InvalidIndexUid,
+        "invalid_state" => Code::InvalidState,
+        "missing_primary_key" => Code::MissingPrimaryKey,
+        "primary_key_already_present" => Code::PrimaryKeyAlreadyPresent,
+        "invalid_request" => Code::InvalidRankingRule,
+        "max_fields_limit_exceeded" => Code::MaxFieldsLimitExceeded,
+        "missing_document_id" => Code::MissingDocumentId,
+        "invalid_facet" => Code::Filter,
+        "invalid_filter" => Code::Filter,
+        "invalid_sort" => Code::Sort,
+        "bad_parameter" => Code::BadParameter,
+        "bad_request" => Code::BadRequest,
+        "document_not_found" => Code::DocumentNotFound,
+        "internal" => Code::Internal,
+        "invalid_geo_field" => Code::InvalidGeoField,
+        "invalid_token" => Code::InvalidToken,
+        "missing_authorization_header" => Code::MissingAuthorizationHeader,
+        "payload_too_large" => Code::PayloadTooLarge,
+        "unretrievable_document" => Code::RetrieveDocument,
+        "search_error" => Code::SearchDocuments,
+        "unsupported_media_type" => Code::UnsupportedMediaType,
+        "dump_already_in_progress" => Code::DumpAlreadyInProgress,
+        "dump_process_failed" => Code::DumpProcessFailed,
+        _ => bail!("unknow error code."),
+    };
+
+    Ok(code)
+}

meilisearch-lib/src/dump/compat/v3.rs

@@ -0,0 +1,204 @@
+use meilisearch_error::{Code, ResponseError};
+use milli::update::IndexDocumentsMethod;
+use serde::{Deserialize, Serialize};
+use time::OffsetDateTime;
+use uuid::Uuid;
+
+use crate::index::{Settings, Unchecked};
+use crate::index_resolver::IndexUid;
+use crate::tasks::task::{DocumentDeletion, Task, TaskContent, TaskEvent, TaskId, TaskResult};
+
+use super::v2;
+
+#[derive(Serialize, Deserialize)]
+pub struct DumpEntry {
+    pub uuid: Uuid,
+    pub uid: String,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct UpdateEntry {
+    pub uuid: Uuid,
+    pub update: UpdateStatus,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(tag = "status", rename_all = "camelCase")]
+pub enum UpdateStatus {
+    Processing(Processing),
+    Enqueued(Enqueued),
+    Processed(Processed),
+    Failed(Failed),
+}
+
+impl From<v2::UpdateResult> for TaskResult {
+    fn from(other: v2::UpdateResult) -> Self {
+        match other {
+            v2::UpdateResult::DocumentsAddition(result) => TaskResult::DocumentAddition {
+                indexed_documents: result.nb_documents as u64,
+            },
+            v2::UpdateResult::DocumentDeletion { deleted } => TaskResult::DocumentDeletion {
+                deleted_documents: deleted,
+            },
+            v2::UpdateResult::Other => TaskResult::Other,
+        }
+    }
+}
+
+#[allow(clippy::large_enum_variant)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum Update {
+    DeleteDocuments(Vec<String>),
+    DocumentAddition {
+        primary_key: Option<String>,
+        method: IndexDocumentsMethod,
+        content_uuid: Uuid,
+    },
+    Settings(Settings<Unchecked>),
+    ClearDocuments,
+}
+
+impl From<Update> for TaskContent {
+    fn from(other: Update) -> Self {
+        match other {
+            Update::DeleteDocuments(ids) => {
+                TaskContent::DocumentDeletion(DocumentDeletion::Ids(ids))
+            }
+            Update::DocumentAddition {
+                primary_key,
+                method,
+                ..
+            } => TaskContent::DocumentAddition {
+                content_uuid: Uuid::default(),
+                merge_strategy: method,
+                primary_key,
+                // document count is unknown for legacy updates
+                documents_count: 0,
+                allow_index_creation: true,
+            },
+            Update::Settings(settings) => TaskContent::SettingsUpdate {
+                settings,
+                // There is no way to know now, so we assume it isn't
+                is_deletion: false,
+                allow_index_creation: true,
+            },
+            Update::ClearDocuments => TaskContent::DocumentDeletion(DocumentDeletion::Clear),
+        }
+    }
+}
+
+#[allow(clippy::large_enum_variant)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum UpdateMeta {
+    DocumentsAddition {
+        method: IndexDocumentsMethod,
+        primary_key: Option<String>,
+    },
+    ClearDocuments,
+    DeleteDocuments {
+        ids: Vec<String>,
+    },
+    Settings(Settings<Unchecked>),
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct Enqueued {
+    pub update_id: u64,
+    pub meta: Update,
+    #[serde(with = "time::serde::rfc3339")]
+    pub enqueued_at: OffsetDateTime,
+}
+
+impl Enqueued {
+    fn update_task(self, task: &mut Task) {
+        // we do not erase the `TaskId` that was given to us.
+        task.content = self.meta.into();
+        task.events.push(TaskEvent::Created(self.enqueued_at));
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct Processed {
+    pub success: v2::UpdateResult,
+    #[serde(with = "time::serde::rfc3339")]
+    pub processed_at: OffsetDateTime,
+    #[serde(flatten)]
+    pub from: Processing,
+}
+
+impl Processed {
+    fn update_task(self, task: &mut Task) {
+        self.from.update_task(task);
+
+        let event = TaskEvent::Succeded {
+            result: TaskResult::from(self.success),
+            timestamp: self.processed_at,
+        };
+        task.events.push(event);
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct Processing {
+    #[serde(flatten)]
+    pub from: Enqueued,
+    #[serde(with = "time::serde::rfc3339")]
+    pub started_processing_at: OffsetDateTime,
+}
+
+impl Processing {
+    fn update_task(self, task: &mut Task) {
+        self.from.update_task(task);
+
+        let event = TaskEvent::Processing(self.started_processing_at);
+        task.events.push(event);
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Failed {
+    #[serde(flatten)]
+    pub from: Processing,
+    pub msg: String,
+    pub code: Code,
+    #[serde(with = "time::serde::rfc3339")]
+    pub failed_at: OffsetDateTime,
+}
+
+impl Failed {
+    fn update_task(self, task: &mut Task) {
+        self.from.update_task(task);
+
+        let event = TaskEvent::Failed {
+            error: ResponseError::from_msg(self.msg, self.code),
+            timestamp: self.failed_at,
+        };
+        task.events.push(event);
+    }
+}
+
+impl From<(UpdateStatus, String, TaskId)> for Task {
+    fn from((update, uid, task_id): (UpdateStatus, String, TaskId)) -> Self {
+        // Dummy task
+        let mut task = Task {
+            id: task_id,
+            index_uid: Some(IndexUid::new(uid).unwrap()),
+            content: TaskContent::IndexDeletion,
+            events: Vec::new(),
+        };
+
+        match update {
+            UpdateStatus::Processing(u) => u.update_task(&mut task),
+            UpdateStatus::Enqueued(u) => u.update_task(&mut task),
+            UpdateStatus::Processed(u) => u.update_task(&mut task),
+            UpdateStatus::Failed(u) => u.update_task(&mut task),
+        }
+
+        task
+    }
+}

meilisearch-lib/src/dump/error.rs

@@ -0,0 +1,41 @@
+use meilisearch_auth::error::AuthControllerError;
+use meilisearch_error::{internal_error, Code, ErrorCode};
+
+use crate::{index_resolver::error::IndexResolverError, tasks::error::TaskError};
+
+pub type Result<T> = std::result::Result<T, DumpActorError>;
+
+#[derive(thiserror::Error, Debug)]
+pub enum DumpActorError {
+    #[error("A dump is already processing. You must wait until the current process is finished before requesting another dump.")]
+    DumpAlreadyRunning,
+    #[error("Dump `{0}` not found.")]
+    DumpDoesNotExist(String),
+    #[error("An internal error has occurred. `{0}`.")]
+    Internal(Box<dyn std::error::Error + Send + Sync + 'static>),
+    #[error("{0}")]
+    IndexResolver(#[from] IndexResolverError),
+}
+
+internal_error!(
+    DumpActorError: milli::heed::Error,
+    std::io::Error,
+    tokio::task::JoinError,
+    tokio::sync::oneshot::error::RecvError,
+    serde_json::error::Error,
+    tempfile::PersistError,
+    fs_extra::error::Error,
+    AuthControllerError,
+    TaskError
+);
+
+impl ErrorCode for DumpActorError {
+    fn error_code(&self) -> Code {
+        match self {
+            DumpActorError::DumpAlreadyRunning => Code::DumpAlreadyInProgress,
+            DumpActorError::DumpDoesNotExist(_) => Code::DumpNotFound,
+            DumpActorError::Internal(_) => Code::Internal,
+            DumpActorError::IndexResolver(e) => e.error_code(),
+        }
+    }
+}

meilisearch-lib/src/dump/handle_impl.rs

@@ -0,0 +1,26 @@
+use tokio::sync::{mpsc, oneshot};
+
+use super::error::Result;
+use super::{DumpActorHandle, DumpInfo, DumpMsg};
+
+#[derive(Clone)]
+pub struct DumpActorHandleImpl {
+    pub sender: mpsc::Sender<DumpMsg>,
+}
+
+#[async_trait::async_trait]
+impl DumpActorHandle for DumpActorHandleImpl {
+    async fn create_dump(&self) -> Result<DumpInfo> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = DumpMsg::CreateDump { ret };
+        let _ = self.sender.send(msg).await;
+        receiver.await.expect("IndexActor has been killed")
+    }
+
+    async fn dump_info(&self, uid: String) -> Result<DumpInfo> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = DumpMsg::DumpInfo { ret, uid };
+        let _ = self.sender.send(msg).await;
+        receiver.await.expect("IndexActor has been killed")
+    }
+}

meilisearch-lib/src/dump/loaders/mod.rs

@@ -0,0 +1,3 @@
+pub mod v2;
+pub mod v3;
+pub mod v4;

meilisearch-lib/src/dump/loaders/v1.rs

@@ -0,0 +1,24 @@
+use std::path::Path;
+
+use serde::{Deserialize, Serialize};
+
+use crate::index_controller::IndexMetadata;
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct MetadataV1 {
+    pub db_version: String,
+    indexes: Vec<IndexMetadata>,
+}
+
+impl MetadataV1 {
+    #[allow(dead_code, unreachable_code, unused_variables)]
+    pub fn load_dump(
+        self,
+        src: impl AsRef<Path>,
+        dst: impl AsRef<Path>,
+        size: usize,
+        indexer_options: &IndexerOpts,
+    ) -> anyhow::Result<()> {
+        anyhow::bail!("The version 1 of the dumps is not supported anymore. You can re-export your dump from a version between 0.21 and 0.24, or start fresh from a version 0.25 onwards.")
+}

meilisearch-lib/src/dump/loaders/v2.rs

@@ -0,0 +1,216 @@
+use std::fs::{File, OpenOptions};
+use std::io::Write;
+use std::path::{Path, PathBuf};
+
+use serde_json::{Deserializer, Value};
+use tempfile::NamedTempFile;
+
+use crate::dump::compat::{self, v2, v3};
+use crate::dump::Metadata;
+use crate::options::IndexerOpts;
+
+/// The dump v2 reads the dump folder and patches all the needed file to make it compatible with a
+/// dump v3, then calls the dump v3 to actually handle the dump.
+pub fn load_dump(
+    meta: Metadata,
+    src: impl AsRef<Path>,
+    dst: impl AsRef<Path>,
+    index_db_size: usize,
+    update_db_size: usize,
+    indexing_options: &IndexerOpts,
+) -> anyhow::Result<()> {
+    log::info!("Patching dump V2 to dump V3...");
+    let indexes_path = src.as_ref().join("indexes");
+
+    let dir_entries = std::fs::read_dir(indexes_path)?;
+    for entry in dir_entries {
+        let entry = entry?;
+
+        // rename the index folder
+        let path = entry.path();
+        let new_path = patch_index_uuid_path(&path).expect("invalid index folder.");
+
+        std::fs::rename(path, &new_path)?;
+
+        let settings_path = new_path.join("meta.json");
+
+        patch_settings(settings_path)?;
+    }
+
+    let update_dir = src.as_ref().join("updates");
+    let update_path = update_dir.join("data.jsonl");
+    patch_updates(update_dir, update_path)?;
+
+    super::v3::load_dump(
+        meta,
+        src,
+        dst,
+        index_db_size,
+        update_db_size,
+        indexing_options,
+    )
+}
+
+fn patch_index_uuid_path(path: &Path) -> Option<PathBuf> {
+    let uuid = path.file_name()?.to_str()?.trim_start_matches("index-");
+    let new_path = path.parent()?.join(uuid);
+    Some(new_path)
+}
+
+fn patch_settings(path: impl AsRef<Path>) -> anyhow::Result<()> {
+    let mut meta_file = File::open(&path)?;
+    let mut meta: Value = serde_json::from_reader(&mut meta_file)?;
+
+    // We first deserialize the dump meta into a serde_json::Value and change
+    // the custom ranking rules settings from the old format to the new format.
+    if let Some(ranking_rules) = meta.pointer_mut("/settings/rankingRules") {
+        patch_custom_ranking_rules(ranking_rules);
+    }
+
+    let mut meta_file = OpenOptions::new().truncate(true).write(true).open(path)?;
+
+    serde_json::to_writer(&mut meta_file, &meta)?;
+
+    Ok(())
+}
+
+fn patch_updates(dir: impl AsRef<Path>, path: impl AsRef<Path>) -> anyhow::Result<()> {
+    let mut output_update_file = NamedTempFile::new_in(&dir)?;
+    let update_file = File::open(&path)?;
+
+    let stream = Deserializer::from_reader(update_file).into_iter::<v2::UpdateEntry>();
+
+    for update in stream {
+        let update_entry = update?;
+
+        let update_entry = v3::UpdateEntry::from(update_entry);
+
+        serde_json::to_writer(&mut output_update_file, &update_entry)?;
+        output_update_file.write_all(b"\n")?;
+    }
+
+    output_update_file.flush()?;
+    output_update_file.persist(path)?;
+
+    Ok(())
+}
+
+/// Converts the ranking rules from the format `asc(_)`, `desc(_)` to the format `_:asc`, `_:desc`.
+///
+/// This is done for compatibility reasons, and to avoid a new dump version,
+/// since the new syntax was introduced soon after the new dump version.
+fn patch_custom_ranking_rules(ranking_rules: &mut Value) {
+    *ranking_rules = match ranking_rules.take() {
+        Value::Array(values) => values
+            .into_iter()
+            .filter_map(|value| match value {
+                Value::String(s) if s.starts_with("asc") => compat::asc_ranking_rule(&s)
+                    .map(|f| format!("{}:asc", f))
+                    .map(Value::String),
+                Value::String(s) if s.starts_with("desc") => compat::desc_ranking_rule(&s)
+                    .map(|f| format!("{}:desc", f))
+                    .map(Value::String),
+                otherwise => Some(otherwise),
+            })
+            .collect(),
+        otherwise => otherwise,
+    }
+}
+
+impl From<v2::UpdateEntry> for v3::UpdateEntry {
+    fn from(v2::UpdateEntry { uuid, update }: v2::UpdateEntry) -> Self {
+        let update = match update {
+            v2::UpdateStatus::Processing(meta) => v3::UpdateStatus::Processing(meta.into()),
+            v2::UpdateStatus::Enqueued(meta) => v3::UpdateStatus::Enqueued(meta.into()),
+            v2::UpdateStatus::Processed(meta) => v3::UpdateStatus::Processed(meta.into()),
+            v2::UpdateStatus::Aborted(_) => unreachable!("Updates could never be aborted."),
+            v2::UpdateStatus::Failed(meta) => v3::UpdateStatus::Failed(meta.into()),
+        };
+
+        Self { uuid, update }
+    }
+}
+
+impl From<v2::Failed> for v3::Failed {
+    fn from(other: v2::Failed) -> Self {
+        let v2::Failed {
+            from,
+            error,
+            failed_at,
+        } = other;
+
+        Self {
+            from: from.into(),
+            msg: error.message,
+            code: v2::error_code_from_str(&error.error_code)
+                .expect("Invalid update: Invalid error code"),
+            failed_at,
+        }
+    }
+}
+
+impl From<v2::Processing> for v3::Processing {
+    fn from(other: v2::Processing) -> Self {
+        let v2::Processing {
+            from,
+            started_processing_at,
+        } = other;
+
+        Self {
+            from: from.into(),
+            started_processing_at,
+        }
+    }
+}
+
+impl From<v2::Enqueued> for v3::Enqueued {
+    fn from(other: v2::Enqueued) -> Self {
+        let v2::Enqueued {
+            update_id,
+            meta,
+            enqueued_at,
+            content,
+        } = other;
+
+        let meta = match meta {
+            v2::UpdateMeta::DocumentsAddition {
+                method,
+                primary_key,
+                ..
+            } => {
+                v3::Update::DocumentAddition {
+                    primary_key,
+                    method,
+                    // Just ignore if the uuid is no present. If it is needed later, an error will
+                    // be thrown.
+                    content_uuid: content.unwrap_or_default(),
+                }
+            }
+            v2::UpdateMeta::ClearDocuments => v3::Update::ClearDocuments,
+            v2::UpdateMeta::DeleteDocuments { ids } => v3::Update::DeleteDocuments(ids),
+            v2::UpdateMeta::Settings(settings) => v3::Update::Settings(settings),
+        };
+
+        Self {
+            update_id,
+            meta,
+            enqueued_at,
+        }
+    }
+}
+
+impl From<v2::Processed> for v3::Processed {
+    fn from(other: v2::Processed) -> Self {
+        let v2::Processed {
+            from,
+            success,
+            processed_at,
+        } = other;
+
+        Self {
+            success,
+            processed_at,
+            from: from.into(),
+        }
+    }
+}

meilisearch-lib/src/dump/loaders/v3.rs

@@ -0,0 +1,136 @@
+use std::collections::HashMap;
+use std::fs::{self, File};
+use std::io::{BufReader, BufWriter, Write};
+use std::path::Path;
+
+use anyhow::Context;
+use fs_extra::dir::{self, CopyOptions};
+use log::info;
+use tempfile::tempdir;
+use uuid::Uuid;
+
+use crate::dump::compat::v3;
+use crate::dump::Metadata;
+use crate::index_resolver::meta_store::{DumpEntry, IndexMeta};
+use crate::options::IndexerOpts;
+use crate::tasks::task::{Task, TaskId};
+
+/// dump structure for V3:
+/// .
+/// ├── indexes
+/// │   └── 25f10bb8-6ea8-42f0-bd48-ad5857f77648
+/// │       ├── documents.jsonl
+/// │       └── meta.json
+/// ├── index_uuids
+/// │   └── data.jsonl
+/// ├── metadata.json
+/// └── updates
+///     └── data.jsonl
+
+pub fn load_dump(
+    meta: Metadata,
+    src: impl AsRef<Path>,
+    dst: impl AsRef<Path>,
+    index_db_size: usize,
+    meta_env_size: usize,
+    indexing_options: &IndexerOpts,
+) -> anyhow::Result<()> {
+    info!("Patching dump V3 to dump V4...");
+
+    let patched_dir = tempdir()?;
+
+    let options = CopyOptions::default();
+    dir::copy(src.as_ref().join("indexes"), patched_dir.path(), &options)?;
+    dir::copy(
+        src.as_ref().join("index_uuids"),
+        patched_dir.path(),
+        &options,
+    )?;
+
+    let uuid_map = patch_index_meta(
+        src.as_ref().join("index_uuids/data.jsonl"),
+        patched_dir.path(),
+    )?;
+
+    fs::copy(
+        src.as_ref().join("metadata.json"),
+        patched_dir.path().join("metadata.json"),
+    )?;
+
+    patch_updates(&src, patched_dir.path(), uuid_map)?;
+
+    super::v4::load_dump(
+        meta,
+        patched_dir.path(),
+        dst,
+        index_db_size,
+        meta_env_size,
+        indexing_options,
+    )
+}
+
+fn patch_index_meta(
+    path: impl AsRef<Path>,
+    dst: impl AsRef<Path>,
+) -> anyhow::Result<HashMap<Uuid, String>> {
+    let file = BufReader::new(File::open(path)?);
+    let dst = dst.as_ref().join("index_uuids");
+    fs::create_dir_all(&dst)?;
+    let mut dst_file = File::create(dst.join("data.jsonl"))?;
+
+    let map = serde_json::Deserializer::from_reader(file)
+        .into_iter::<v3::DumpEntry>()
+        .try_fold(HashMap::new(), |mut map, entry| -> anyhow::Result<_> {
+            let entry = entry?;
+            map.insert(entry.uuid, entry.uid.clone());
+            let meta = IndexMeta {
+                uuid: entry.uuid,
+                // This is lost information, we patch it to 0;
+                creation_task_id: 0,
+            };
+            let entry = DumpEntry {
+                uid: entry.uid,
+                index_meta: meta,
+            };
+            serde_json::to_writer(&mut dst_file, &entry)?;
+            dst_file.write_all(b"\n")?;
+            Ok(map)
+        })?;
+
+    dst_file.flush()?;
+
+    Ok(map)
+}
+
+fn patch_updates(
+    src: impl AsRef<Path>,
+    dst: impl AsRef<Path>,
+    uuid_map: HashMap<Uuid, String>,
+) -> anyhow::Result<()> {
+    let dst = dst.as_ref().join("updates");
+    fs::create_dir_all(&dst)?;
+
+    let mut dst_file = BufWriter::new(File::create(dst.join("data.jsonl"))?);
+    let src_file = BufReader::new(File::open(src.as_ref().join("updates/data.jsonl"))?);
+
+    serde_json::Deserializer::from_reader(src_file)
+        .into_iter::<v3::UpdateEntry>()
+        .enumerate()
+        .try_for_each(|(task_id, entry)| -> anyhow::Result<()> {
+            let entry = entry?;
+            let name = uuid_map
+                .get(&entry.uuid)
+                .with_context(|| format!("Unknown index uuid: {}", entry.uuid))?
+                .clone();
+            serde_json::to_writer(
+                &mut dst_file,
+                &Task::from((entry.update, name, task_id as TaskId)),
+            )?;
+            dst_file.write_all(b"\n")?;
+            Ok(())
+        })?;
+
+    dst_file.flush()?;
+
+    Ok(())
+}

meilisearch-lib/src/dump/loaders/v4.rs

@@ -0,0 +1,48 @@
+use std::path::Path;
+use std::sync::Arc;
+
+use log::info;
+use meilisearch_auth::AuthController;
+use milli::heed::EnvOpenOptions;
+
+use crate::analytics;
+use crate::dump::Metadata;
+use crate::index_resolver::IndexResolver;
+use crate::options::IndexerOpts;
+use crate::tasks::TaskStore;
+use crate::update_file_store::UpdateFileStore;
+
+pub fn load_dump(
+    meta: Metadata,
+    src: impl AsRef<Path>,
+    dst: impl AsRef<Path>,
+    index_db_size: usize,
+    meta_env_size: usize,
+    indexing_options: &IndexerOpts,
+) -> anyhow::Result<()> {
+    info!(
+        "Loading dump from {}, dump database version: {}, dump version: V4",
+        meta.dump_date, meta.db_version
+    );
+
+    let mut options = EnvOpenOptions::new();
+    options.map_size(meta_env_size);
+    options.max_dbs(100);
+    let env = Arc::new(options.open(&dst)?);
+
+    IndexResolver::load_dump(
+        src.as_ref(),
+        &dst,
+        index_db_size,
+        env.clone(),
+        indexing_options,
+    )?;
+    UpdateFileStore::load_dump(src.as_ref(), &dst)?;
+    TaskStore::load_dump(&src, env)?;
+    AuthController::load_dump(&src, &dst)?;
+    analytics::copy_user_id(src.as_ref(), dst.as_ref());
+
+    info!("Loading indexes.");
+
+    Ok(())
+}

meilisearch-lib/src/dump/message.rs

@@ -0,0 +1,14 @@
+use tokio::sync::oneshot;
+
+use super::error::Result;
+use super::DumpInfo;
+
+pub enum DumpMsg {
+    CreateDump {
+        ret: oneshot::Sender<Result<DumpInfo>>,
+    },
+    DumpInfo {
+        uid: String,
+        ret: oneshot::Sender<Result<DumpInfo>>,
+    },
+}

meilisearch-lib/src/dump/mod.rs

@@ -0,0 +1,506 @@
+use std::fs::File;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use anyhow::bail;
+use log::info;
+use serde::{Deserialize, Serialize};
+use time::OffsetDateTime;
+
+pub use actor::DumpActor;
+pub use handle_impl::*;
+pub use message::DumpMsg;
+use tempfile::TempDir;
+use tokio::sync::RwLock;
+
+use crate::compression::from_tar_gz;
+use crate::options::IndexerOpts;
+use crate::tasks::Scheduler;
+use crate::update_file_store::UpdateFileStore;
+use error::Result;
+
+use self::loaders::{v2, v3, v4};
+
+mod actor;
+mod compat;
+pub mod error;
+mod handle_impl;
+mod loaders;
+mod message;
+
+const META_FILE_NAME: &str = "metadata.json";
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct Metadata {
+    db_version: String,
+    index_db_size: usize,
+    update_db_size: usize,
+    #[serde(with = "time::serde::rfc3339")]
+    dump_date: OffsetDateTime,
+}
+
+impl Metadata {
+    pub fn new(index_db_size: usize, update_db_size: usize) -> Self {
+        Self {
+            db_version: env!("CARGO_PKG_VERSION").to_string(),
+            index_db_size,
+            update_db_size,
+            dump_date: OffsetDateTime::now_utc(),
+        }
+    }
+}
+
+#[async_trait::async_trait]
+#[cfg_attr(test, mockall::automock)]
+pub trait DumpActorHandle {
+    /// Start the creation of a dump
+    /// Implementation: [handle_impl::DumpActorHandleImpl::create_dump]
+    async fn create_dump(&self) -> Result<DumpInfo>;
+
+    /// Return the status of an already created dump
+    /// Implementation: [handle_impl::DumpActorHandleImpl::dump_info]
+    async fn dump_info(&self, uid: String) -> Result<DumpInfo>;
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct MetadataV1 {
+    pub db_version: String,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(tag = "dumpVersion")]
+pub enum MetadataVersion {
+    V1(MetadataV1),
+    V2(Metadata),
+    V3(Metadata),
+    V4(Metadata),
+}
+
+impl MetadataVersion {
+    pub fn load_dump(
+        self,
+        src: impl AsRef<Path>,
+        dst: impl AsRef<Path>,
+        index_db_size: usize,
+        meta_env_size: usize,
+        indexing_options: &IndexerOpts,
+    ) -> anyhow::Result<()> {
+        match self {
+            MetadataVersion::V1(_meta) => {
+                anyhow::bail!("The version 1 of the dumps is not supported anymore. You can re-export your dump from a version between 0.21 and 0.24, or start fresh from a version 0.25 onwards.")
+            }
+            MetadataVersion::V2(meta) => v2::load_dump(
+                meta,
+                src,
+                dst,
+                index_db_size,
+                meta_env_size,
+                indexing_options,
+            )?,
+            MetadataVersion::V3(meta) => v3::load_dump(
+                meta,
+                src,
+                dst,
+                index_db_size,
+                meta_env_size,
+                indexing_options,
+            )?,
+            MetadataVersion::V4(meta) => v4::load_dump(
+                meta,
+                src,
+                dst,
+                index_db_size,
+                meta_env_size,
+                indexing_options,
+            )?,
+        }
+
+        Ok(())
+    }
+
+    pub fn new_v4(index_db_size: usize, update_db_size: usize) -> Self {
+        let meta = Metadata::new(index_db_size, update_db_size);
+        Self::V4(meta)
+    }
+
+    pub fn db_version(&self) -> &str {
+        match self {
+            Self::V1(meta) => &meta.db_version,
+            Self::V2(meta) | Self::V3(meta) | Self::V4(meta) => &meta.db_version,
+        }
+    }
+
+    pub fn version(&self) -> &str {
+        match self {
+            MetadataVersion::V1(_) => "V1",
+            MetadataVersion::V2(_) => "V2",
+            MetadataVersion::V3(_) => "V3",
+            MetadataVersion::V4(_) => "V4",
+        }
+    }
+
+    pub fn dump_date(&self) -> Option<&OffsetDateTime> {
+        match self {
+            MetadataVersion::V1(_) => None,
+            MetadataVersion::V2(meta) | MetadataVersion::V3(meta) | MetadataVersion::V4(meta) => {
+                Some(&meta.dump_date)
+            }
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+#[serde(rename_all = "snake_case")]
+pub enum DumpStatus {
+    Done,
+    InProgress,
+    Failed,
+}
+
+#[derive(Debug, Serialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct DumpInfo {
+    pub uid: String,
+    pub status: DumpStatus,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error: Option<String>,
+    #[serde(with = "time::serde::rfc3339")]
+    started_at: OffsetDateTime,
+    #[serde(
+        skip_serializing_if = "Option::is_none",
+        with = "time::serde::rfc3339::option"
+    )]
+    finished_at: Option<OffsetDateTime>,
+}
+
+impl DumpInfo {
+    pub fn new(uid: String, status: DumpStatus) -> Self {
+        Self {
+            uid,
+            status,
+            error: None,
+            started_at: OffsetDateTime::now_utc(),
+            finished_at: None,
+        }
+    }
+
+    pub fn with_error(&mut self, error: String) {
+        self.status = DumpStatus::Failed;
+        self.finished_at = Some(OffsetDateTime::now_utc());
+        self.error = Some(error);
+    }
+
+    pub fn done(&mut self) {
+        self.finished_at = Some(OffsetDateTime::now_utc());
+        self.status = DumpStatus::Done;
+    }
+
+    pub fn dump_already_in_progress(&self) -> bool {
+        self.status == DumpStatus::InProgress
+    }
+}
+
+pub fn load_dump(
+    dst_path: impl AsRef<Path>,
+    src_path: impl AsRef<Path>,
+    ignore_dump_if_db_exists: bool,
+    ignore_missing_dump: bool,
+    index_db_size: usize,
+    update_db_size: usize,
+    indexer_opts: &IndexerOpts,
+) -> anyhow::Result<()> {
+    let empty_db = crate::is_empty_db(&dst_path);
+    let src_path_exists = src_path.as_ref().exists();
+
+    if empty_db && src_path_exists {
+        let (tmp_src, tmp_dst, meta) = extract_dump(&dst_path, &src_path)?;
+        meta.load_dump(
+            tmp_src.path(),
+            tmp_dst.path(),
+            index_db_size,
+            update_db_size,
+            indexer_opts,
+        )?;
+        persist_dump(&dst_path, tmp_dst)?;
+        Ok(())
+    } else if !empty_db && !ignore_dump_if_db_exists {
+        bail!(
+            "database already exists at {:?}, try to delete it or rename it",
+            dst_path
+                .as_ref()
+                .canonicalize()
+                .unwrap_or_else(|_| dst_path.as_ref().to_owned())
+        )
+    } else if !src_path_exists && !ignore_missing_dump {
+        bail!("dump doesn't exist at {:?}", src_path.as_ref())
+    } else {
+        // there is nothing to do
+        Ok(())
+    }
+}
+
+fn extract_dump(
+    dst_path: impl AsRef<Path>,
+    src_path: impl AsRef<Path>,
+) -> anyhow::Result<(TempDir, TempDir, MetadataVersion)> {
+    // Setup a temp directory path in the same path as the database, to prevent cross devices
+    // references.
+    let temp_path = dst_path
+        .as_ref()
+        .parent()
+        .map(ToOwned::to_owned)
+        .unwrap_or_else(|| ".".into());
+
+    let tmp_src = tempfile::tempdir_in(temp_path)?;
+    let tmp_src_path = tmp_src.path();
+
+    from_tar_gz(&src_path, tmp_src_path)?;
+
+    let meta_path = tmp_src_path.join(META_FILE_NAME);
+    let mut meta_file = File::open(&meta_path)?;
+    let meta: MetadataVersion = serde_json::from_reader(&mut meta_file)?;
+
+    if !dst_path.as_ref().exists() {
+        std::fs::create_dir_all(dst_path.as_ref())?;
+    }
+
+    let tmp_dst = tempfile::tempdir_in(dst_path.as_ref())?;
+
+    info!(
+        "Loading dump {}, dump database version: {}, dump version: {}",
+        meta.dump_date()
+            .map(|t| format!("from {}", t))
+            .unwrap_or_else(String::new),
+        meta.db_version(),
+        meta.version()
+    );
+
+    Ok((tmp_src, tmp_dst, meta))
+}
+
+fn persist_dump(dst_path: impl AsRef<Path>, tmp_dst: TempDir) -> anyhow::Result<()> {
+    let persisted_dump = tmp_dst.into_path();
+
+    // Delete everything in the `data.ms` except the tempdir.
+    if dst_path.as_ref().exists() {
+        for file in dst_path.as_ref().read_dir().unwrap() {
+            let file = file.unwrap().path();
+            if file.file_name() == persisted_dump.file_name() {
+                continue;
+            }
+
+            if file.is_file() {
+                std::fs::remove_file(&file)?;
+            } else {
+                std::fs::remove_dir_all(&file)?;
+            }
+        }
+    }
+
+    // Move the whole content of the tempdir into the `data.ms`.
+    for file in persisted_dump.read_dir().unwrap() {
+        let file = file.unwrap().path();
+
+        std::fs::rename(&file, &dst_path.as_ref().join(file.file_name().unwrap()))?;
+    }
+
+    // Delete the empty tempdir.
+    std::fs::remove_dir_all(&persisted_dump)?;
+
+    Ok(())
+}
+
+pub struct DumpJob {
+    dump_path: PathBuf,
+    db_path: PathBuf,
+    update_file_store: UpdateFileStore,
+    scheduler: Arc<RwLock<Scheduler>>,
+    uid: String,
+    update_db_size: usize,
+    index_db_size: usize,
+}
+
+impl DumpJob {
+    async fn run(self) -> Result<()> {
+        // trace!("Performing dump.");
+        //
+        // create_dir_all(&self.dump_path).await?;
+        //
+        // let temp_dump_dir = tokio::task::spawn_blocking(tempfile::TempDir::new).await??;
+        // let temp_dump_path = temp_dump_dir.path().to_owned();
+        //
+        // let meta = MetadataVersion::new_v4(self.index_db_size, self.update_db_size);
+        // let meta_path = temp_dump_path.join(META_FILE_NAME);
+        // let mut meta_file = File::create(&meta_path)?;
+        // serde_json::to_writer(&mut meta_file, &meta)?;
+        // analytics::copy_user_id(&self.db_path, &temp_dump_path);
+        //
+        // create_dir_all(&temp_dump_path.join("indexes")).await?;
+        //
+        // let (sender, receiver) = oneshot::channel();
+        //
+        // self.scheduler
+        //     .write()
+        //     .await
+        //     .schedule_job(Job::Dump {
+        //         ret: sender,
+        //         path: temp_dump_path.clone(),
+        //     })
+        //     .await;
+        //
+        // // wait until the job has started performing before finishing the dump process
+        // let sender = receiver.await??;
+        //
+        // AuthController::dump(&self.db_path, &temp_dump_path)?;
+        //
+        // //TODO(marin): this is not right, the scheduler should dump itself, not do it here...
+        // self.scheduler
+        //     .read()
+        //     .await
+        //     .dump(&temp_dump_path, self.update_file_store.clone())
+        //     .await?;
+        //
+        // let dump_path = tokio::task::spawn_blocking(move || -> Result<PathBuf> {
+        //     // for now we simply copy the updates/updates_files
+        //     // FIXME: We may copy more files than necessary, if new files are added while we are
+        //     // performing the dump. We need a way to filter them out.
+        //
+        //     let temp_dump_file = tempfile::NamedTempFile::new_in(&self.dump_path)?;
+        //     to_tar_gz(temp_dump_path, temp_dump_file.path())
+        //         .map_err(|e| DumpActorError::Internal(e.into()))?;
+        //
+        //     let dump_path = self.dump_path.join(self.uid).with_extension("dump");
+        //     temp_dump_file.persist(&dump_path)?;
+        //
+        //     Ok(dump_path)
+        // })
+        // .await??;
+        //
+        // // notify the update loop that we are finished performing the dump.
+        // let _ = sender.send(());
+        //
+        // info!("Created dump in {:?}.", dump_path);
+        //
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use nelson::Mocker;
+    use once_cell::sync::Lazy;
+
+    use super::*;
+    use crate::index_resolver::error::IndexResolverError;
+    use crate::options::SchedulerConfig;
+    use crate::tasks::error::Result as TaskResult;
+    use crate::tasks::task::{Task, TaskId};
+    use crate::tasks::{BatchHandler, TaskFilter, TaskStore};
+    use crate::update_file_store::UpdateFileStore;
+
+    fn setup() {
+        static SETUP: Lazy<()> = Lazy::new(|| {
+            if cfg!(windows) {
+                std::env::set_var("TMP", ".");
+            } else {
+                std::env::set_var("TMPDIR", ".");
+            }
+        });
+
+        // just deref to make sure the env is setup
+        *SETUP
+    }
+
+    #[actix_rt::test]
+    async fn test_dump_normal() {
+        setup();
+
+        let tmp = tempfile::tempdir().unwrap();
+
+        let mocker = Mocker::default();
+        let update_file_store = UpdateFileStore::mock(mocker);
+
+        let mut performer = BatchHandler::new();
+        performer
+            .expect_process_job()
+            .once()
+            .returning(|j| match j {
+                Job::Dump { ret, .. } => {
+                    let (sender, _receiver) = oneshot::channel();
+                    ret.send(Ok(sender)).unwrap();
+                }
+                _ => unreachable!(),
+            });
+        let performer = Arc::new(performer);
+        let mocker = Mocker::default();
+        mocker
+            .when::<(&Path, UpdateFileStore), TaskResult<()>>("dump")
+            .then(|_| Ok(()));
+        mocker
+            .when::<(Option<TaskId>, Option<TaskFilter>, Option<usize>), TaskResult<Vec<Task>>>(
+                "list_tasks",
+            )
+            .then(|_| Ok(Vec::new()));
+        let store = TaskStore::mock(mocker);
+        let config = SchedulerConfig::default();
+
+        let scheduler = Scheduler::new(store, performer, config).unwrap();
+
+        let task = DumpJob {
+            dump_path: tmp.path().into(),
+            // this should do nothing
+            update_file_store,
+            db_path: tmp.path().into(),
+            uid: String::from("test"),
+            update_db_size: 4096 * 10,
+            index_db_size: 4096 * 10,
+            scheduler,
+        };
+
+        task.run().await.unwrap();
+    }
+
+    #[actix_rt::test]
+    async fn error_performing_dump() {
+        let tmp = tempfile::tempdir().unwrap();
+
+        let mocker = Mocker::default();
+        let file_store = UpdateFileStore::mock(mocker);
+
+        let mocker = Mocker::default();
+        mocker
+            .when::<(Option<TaskId>, Option<TaskFilter>, Option<usize>), TaskResult<Vec<Task>>>(
+                "list_tasks",
+            )
+            .then(|_| Ok(Vec::new()));
+        let task_store = TaskStore::mock(mocker);
+        let mut performer = BatchHandler::new();
+        performer
+            .expect_process_job()
+            .once()
+            .returning(|job| match job {
+                Job::Dump { ret, .. } => drop(ret.send(Err(IndexResolverError::BadlyFormatted(
+                    "blabla".to_string(),
+                )))),
+                _ => unreachable!(),
+            });
+        let performer = Arc::new(performer);
+
+        let scheduler = Scheduler::new(task_store, performer, SchedulerConfig::default()).unwrap();
+
+        let task = DumpJob {
+            dump_path: tmp.path().into(),
+            // this should do nothing
+            db_path: tmp.path().into(),
+            update_file_store: file_store,
+            uid: String::from("test"),
+            update_db_size: 4096 * 10,
+            index_db_size: 4096 * 10,
+            scheduler,
+        };
+
+        assert!(task.run().await.is_err());
+    }
+}