WIP: rebasing on master

meilisearch-http/src/index_controller/update_actor/actor.rs

@@ -71,11 +71,16 @@ where
                 Some(Delete { uuid, ret }) => {
                     let _ = ret.send(self.handle_delete(uuid).await);
                 }
-                Some(Snapshot { uuids, path, ret }) => {
-                    let _ = ret.send(self.handle_snapshot(uuids, path).await);
+                Some(Snapshot { uuid, path, ret }) => {
+                    let _ = ret.send(self.handle_snapshot(uuid, path).await);
+                }
+                Some(Dump { uuid, path, ret }) => {
+                    let _ = ret.send(self.handle_dump(uuid, path).await);
                 }
                 Some(GetInfo { ret }) => {
                     let _ = ret.send(self.handle_get_info().await);
+                Some(GetSize { uuid, ret }) => {
+                    let _ = ret.send(self.handle_get_size(uuid).await);
                 }
                 None => break,
             }
@@ -194,9 +199,51 @@ where
     }
 
     async fn handle_delete(&self, uuid: Uuid) -> Result<()> {
-        let store = self.store.clone();
+        let store = self.store.delete(uuid).await?;
 
-        tokio::task::spawn_blocking(move || store.delete_all(uuid))
+        if let Some(store) = store {
+            tokio::task::spawn(async move {
+                let store = get_arc_ownership_blocking(store).await;
+                tokio::task::spawn_blocking(move || {
+                    store.prepare_for_closing().wait();
+                    info!("Update store {} was closed.", uuid);
+                });
+            });
+        }
+
+        Ok(())
+    }
+
+    async fn handle_create(&self, uuid: Uuid) -> Result<()> {
+        let _ = self.store.get_or_create(uuid).await?;
+        Ok(())
+    }
+
+        Ok(())
+    }
+
+    async fn handle_create(&self, uuid: Uuid) -> Result<()> {
+        let _ = self.store.get_or_create(uuid).await?;
+        Ok(())
+    }
+
+    async fn handle_snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> {
+        let index_handle = self.index_handle.clone();
+        if let Some(update_store) = self.store.get(uuid).await? {
+            tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
+                // acquire write lock to prevent further writes during snapshot
+                // the update lock must be acquired BEFORE the write lock to prevent dead lock
+                let _lock = update_store.update_lock.lock();
+                let mut txn = update_store.env.write_txn()?;
+
+                // create db snapshot
+                update_store.snapshot(&mut txn, &path, uuid)?;
+
+                futures::executor::block_on(
+                    async move { index_handle.snapshot(uuid, path).await },
+                )?;
+                Ok(())
+            })
             .await
             .map_err(|e| UpdateError::Error(e.into()))?
             .map_err(|e| UpdateError::Error(e.into()))?;
@@ -245,4 +292,42 @@ where
 
         Ok(info)
     }
+
+    async fn handle_dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> {
+        let index_handle = self.index_handle.clone();
+        if let Some(update_store) = self.store.get(uuid).await? {
+            tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
+                // acquire write lock to prevent further writes during the dump
+                // the update lock must be acquired BEFORE the write lock to prevent dead lock
+                let _lock = update_store.update_lock.lock();
+                let mut txn = update_store.env.write_txn()?;
+
+                // create db dump
+                update_store.dump(&mut txn, &path, uuid)?;
+
+                futures::executor::block_on(
+                    async move { index_handle.dump(uuid, path).await },
+                )?;
+                Ok(())
+            })
+            .await
+            .map_err(|e| UpdateError::Error(e.into()))?
+            .map_err(|e| UpdateError::Error(e.into()))?;
+        }
+
+        Ok(())
+    }
+
+    async fn handle_get_size(&self, uuid: Uuid) -> Result<u64> {
+        let size = match self.store.get(uuid).await? {
+            Some(update_store) => tokio::task::spawn_blocking(move || -> anyhow::Result<u64> {
+                let txn = update_store.env.read_txn()?;
+
+                update_store.get_size(&txn)
+            })
+            .await
+            .map_err(|e| UpdateError::Error(e.into()))?
+            .map_err(|e| UpdateError::Error(e.into()))?,
+            None => 0,
+        };
 }

meilisearch-http/src/index_controller/update_actor/handle_impl.rs

@@ -78,6 +78,20 @@ where
         receiver.await.expect("update actor killed.")
     }
 
+    async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = UpdateMsg::Dump { uuid, path, ret };
+        let _ = self.sender.send(msg).await;
+        receiver.await.expect("update actor killed.")
+    }
+
+    async fn get_size(&self, uuid: Uuid) -> Result<u64> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = UpdateMsg::GetSize { uuid, ret };
+        let _ = self.sender.send(msg).await;
+        receiver.await.expect("update actor killed.")
+    }
+
     async fn update(
         &self,
         meta: UpdateMeta,

meilisearch-http/src/index_controller/update_actor/message.rs

@@ -31,7 +31,16 @@ pub enum UpdateMsg<D> {
         path: PathBuf,
         ret: oneshot::Sender<Result<()>>,
     },
+    Dump {
+        uuid: Uuid,
+        path: PathBuf,
+        ret: oneshot::Sender<Result<()>>,
+    },
     GetInfo {
         ret: oneshot::Sender<Result<UpdateStoreInfo>>,
     },
+    GetSize {
+        uuid: Uuid,
+        ret: oneshot::Sender<Result<u64>>,
+    },
 }

meilisearch-http/src/index_controller/update_actor/mod.rs

@@ -40,8 +40,11 @@ pub trait UpdateActorHandle {
     async fn get_all_updates_status(&self, uuid: Uuid) -> Result<Vec<UpdateStatus>>;
     async fn update_status(&self, uuid: Uuid, id: u64) -> Result<UpdateStatus>;
     async fn delete(&self, uuid: Uuid) -> Result<()>;
-    async fn snapshot(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()>;
+    async fn create(&self, uuid: Uuid) -> Result<()>;
+    async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>;
+    async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()>;
     async fn get_info(&self) -> Result<UpdateStoreInfo>;
+    async fn get_size(&self, uuid: Uuid) -> Result<u64>;
     async fn update(
         &self,
         meta: UpdateMeta,

meilisearch-http/src/index_controller/update_actor/update_store.rs

@@ -499,9 +499,56 @@ impl UpdateStore {
         Ok(())
     }
 
+    pub fn dump(
+        &self,
+        txn: &mut heed::RwTxn,
+        path: impl AsRef<Path>,
+        uuid: Uuid,
+    ) -> anyhow::Result<()> {
+        let update_path = path.as_ref().join("updates");
+        create_dir_all(&update_path)?;
+
+        let mut dump_path = update_path.join(format!("update-{}", uuid));
+        // acquire write lock to prevent further writes during dump
+        create_dir_all(&dump_path)?;
+        dump_path.push("data.mdb");
+
+        // create db dump
+        self.env.copy_to_path(&dump_path, CompactionOption::Enabled)?;
+
+        let update_files_path = update_path.join("update_files");
+        create_dir_all(&update_files_path)?;
+
+        for path in self.pending.iter(&txn)? {
+            let (_, path) = path?;
+            let name = path.file_name().unwrap();
+            let to = update_files_path.join(name);
+            copy(path, to)?;
+        }
+
+        Ok(())
+    }
+
     pub fn get_info(&self) -> anyhow::Result<UpdateStoreInfo> {
         let mut size = self.env.size();
         let txn = self.env.read_txn()?;
+        for entry in self.pending_queue.iter(&txn)? {
+            let (_, pending) = entry?;
+            if let Some(path) = pending.content_path() {
+                size += File::open(path)?.metadata()?.len();
+            }
+        }
+        let processing = match *self.state.read() {
+            State::Processing(uuid, _) => Some(uuid),
+            _ => None,
+        };
+
+        Ok(UpdateStoreInfo { size, processing })
+    }
+
+    pub fn get_size(&self, txn: &heed::RoTxn) -> anyhow::Result<u64> {
+        let mut size = self.env.size();
+        let txn = self.env.read_txn()?;
 
         for entry in self.pending_queue.iter(&txn)? {
             let (_, pending) = entry?;