wip

crates/dump/src/lib.rs

@@ -1,7 +1,8 @@
 #![allow(clippy::type_complexity)]
 #![allow(clippy::wrong_self_convention)]
 
-use meilisearch_types::batches::BatchId;
+use meilisearch_types::batch_view::BatchView;
+use meilisearch_types::batches::{Batch, BatchId};
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::keys::Key;
 use meilisearch_types::milli::update::IndexDocumentsMethod;
@@ -91,6 +92,13 @@ pub struct TaskDump {
     pub finished_at: Option<OffsetDateTime>,
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct BatchDump {
+    pub original: Batch,
+    pub tasks: RoaringBitmap,
+}
+
 // A `Kind` specific version made for the dump. If modified you may break the dump.
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]

crates/dump/src/reader/mod.rs

@@ -101,6 +101,16 @@ impl DumpReader {
         }
     }
 
+    pub fn batches(&mut self) -> Result<Box<dyn Iterator<Item = Result<v6::Batch>> + '_>> {
+        match self {
+            DumpReader::Current(current) => Ok(current.dumps()),
+            // There was no batches in the previous version
+            DumpReader::Compat(_compat) => {
+                Ok(Box::new(std::iter::empty()) as Box<dyn Iterator<Item = Result<v6::Batch>> + '_>)
+            }
+        }
+    }
+
     pub fn keys(&mut self) -> Result<Box<dyn Iterator<Item = Result<v6::Key>> + '_>> {
         match self {
             DumpReader::Current(current) => Ok(current.keys()),

crates/dump/src/reader/v6/mod.rs

@@ -18,6 +18,7 @@ pub type Checked = meilisearch_types::settings::Checked;
 pub type Unchecked = meilisearch_types::settings::Unchecked;
 
 pub type Task = crate::TaskDump;
+pub type Batch = crate::BatchDump;
 pub type Key = meilisearch_types::keys::Key;
 pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures;
 
@@ -48,6 +49,7 @@ pub struct V6Reader {
     instance_uid: Option<Uuid>,
     metadata: Metadata,
     tasks: BufReader<File>,
+    batches: BufReader<File>,
     keys: BufReader<File>,
     features: Option<RuntimeTogglableFeatures>,
 }
@@ -82,6 +84,7 @@ impl V6Reader {
             metadata: serde_json::from_reader(&*meta_file)?,
             instance_uid,
             tasks: BufReader::new(File::open(dump.path().join("tasks").join("queue.jsonl"))?),
+            batches: BufReader::new(File::open(dump.path().join("batches").join("queue.jsonl"))?),
             keys: BufReader::new(File::open(dump.path().join("keys.jsonl"))?),
             features,
             dump,
@@ -124,7 +127,7 @@ impl V6Reader {
         &mut self,
     ) -> Box<dyn Iterator<Item = Result<(Task, Option<Box<super::UpdateFile>>)>> + '_> {
         Box::new((&mut self.tasks).lines().map(|line| -> Result<_> {
-            let task: Task = serde_json::from_str(&line?).unwrap();
+            let task: Task = serde_json::from_str(&line?)?;
 
             let update_file_path = self
                 .dump
@@ -145,6 +148,14 @@ impl V6Reader {
         }))
     }
 
+    pub fn dumps(&mut self) -> Box<dyn Iterator<Item = Result<Batch>> + '_> {
+        Box::new(
+            (&mut self.batches)
+                .lines()
+                .map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }),
+        )
+    }
+
     pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<Key>> + '_> {
         Box::new(
             (&mut self.keys).lines().map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }),

crates/dump/src/writer.rs

@@ -14,7 +14,7 @@ use time::OffsetDateTime;
 use uuid::Uuid;
 
 use crate::reader::Document;
-use crate::{IndexMetadata, Metadata, Result, TaskDump, CURRENT_DUMP_VERSION};
+use crate::{BatchDump, IndexMetadata, Metadata, Result, TaskDump, CURRENT_DUMP_VERSION};
 
 pub struct DumpWriter {
     dir: TempDir,
@@ -174,7 +174,7 @@ impl BatchWriter {
 
     /// Pushes batches in the dump.
     /// The batches doesn't contains any private information thus we don't need a special type like with the tasks.
-    pub fn push_batch(&mut self, batch: &BatchView) -> Result<()> {
+    pub fn push_batch(&mut self, batch: &BatchDump) -> Result<()> {
         self.queue.write_all(&serde_json::to_vec(batch)?)?;
         self.queue.write_all(b"\n")?;
         Ok(())