snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-10-30 23:46:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

dump the content of the dump tasks instead of recreating at import time with wrong API keys

Browse Source
This commit is contained in:
Tamo
2022-10-17 17:38:31 +02:00
committed by Clément Renault
parent 655705eb2b
commit 6bd6321226
6 changed files with 54 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
dump/src/lib.rs
View File

@@ -1,8 +1,10 @@
use meilisearch_types::{ use meilisearch_types::{
error::ResponseError, error::ResponseError,
keys::Key,
milli::update::IndexDocumentsMethod, milli::update::IndexDocumentsMethod,
settings::Unchecked, settings::Unchecked,
tasks::{Details, KindWithContent, Status, Task, TaskId}, tasks::{Details, KindWithContent, Status, Task, TaskId},
InstanceUid,
}; };
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@@ -120,6 +122,8 @@ pub enum KindDump {
}, },
DumpExport { DumpExport {
dump_uid: String, dump_uid: String,
keys: Vec<Key>,
instance_uid: Option<InstanceUid>,
}, },
Snapshot, Snapshot,
} }
@@ -181,7 +185,15 @@ impl From<KindWithContent> for KindDump {
KindWithContent::TaskDeletion { query, tasks } => { KindWithContent::TaskDeletion { query, tasks } => {
KindDump::DeleteTasks { query, tasks } KindDump::DeleteTasks { query, tasks }
} }
KindWithContent::DumpExport { dump_uid, .. } => KindDump::DumpExport { dump_uid }, KindWithContent::DumpExport {
dump_uid,
keys,
instance_uid,
} => KindDump::DumpExport {
dump_uid,
keys,
instance_uid,
},
KindWithContent::Snapshot => KindDump::Snapshot, KindWithContent::Snapshot => KindDump::Snapshot,
} }
} }
@@ -444,7 +456,7 @@ pub(crate) mod test {
drop(indexes); drop(indexes);
// ==== checking the task queue // ==== checking the task queue
for (task, expected) in dump.tasks().zip(create_test_tasks()) { for (task, expected) in dump.tasks().unwrap().zip(create_test_tasks()) {
let (task, content_file) = task.unwrap(); let (task, content_file) = task.unwrap();
assert_eq!(task, expected.0); assert_eq!(task, expected.0);
@@ -463,7 +475,7 @@ pub(crate) mod test {
} }
// ==== checking the keys // ==== checking the keys
for (key, expected) in dump.keys().zip(create_test_api_keys()) { for (key, expected) in dump.keys().unwrap().zip(create_test_api_keys()) {
assert_eq!(key.unwrap(), expected); assert_eq!(key.unwrap(), expected);
} }
} }

34
dump/src/reader/compat/v5_to_v6.rs
View File

@@ -54,13 +54,16 @@ impl CompatV5ToV6 {
pub fn tasks( pub fn tasks(
&mut self, &mut self,
) -> Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_> { ) -> Result<Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_>> {
let instance_uid = self.instance_uid().ok().flatten().map(|uid| uid.clone());
let keys = self.keys()?.collect::<Result<Vec<_>>>()?;
let tasks = match self { let tasks = match self {
CompatV5ToV6::V5(v5) => v5.tasks(), CompatV5ToV6::V5(v5) => v5.tasks(),
CompatV5ToV6::Compat(compat) => compat.tasks(), CompatV5ToV6::Compat(compat) => compat.tasks(),
}; };
Box::new(tasks.map(|task| { Ok(Box::new(tasks.map(move |task| {
task.map(|(task, content_file)| { task.and_then(|(task, content_file)| {
let task_view: v5::tasks::TaskView = task.clone().into(); let task_view: v5::tasks::TaskView = task.clone().into();
let task = v6::Task { let task = v6::Task {
@@ -116,9 +119,11 @@ impl CompatV5ToV6 {
allow_index_creation, allow_index_creation,
settings: settings.into(), settings: settings.into(),
}, },
v5::tasks::TaskContent::Dump { uid } => { v5::tasks::TaskContent::Dump { uid } => v6::Kind::DumpExport {
v6::Kind::DumpExport { dump_uid: uid } dump_uid: uid,
} keys: keys.clone(),
instance_uid: instance_uid.clone(),
},
}, },
details: task_view.details.map(|details| match details { details: task_view.details.map(|details| match details {
v5::Details::DocumentAddition { v5::Details::DocumentAddition {
@@ -152,17 +157,18 @@ impl CompatV5ToV6 {
finished_at: task_view.finished_at, finished_at: task_view.finished_at,
}; };
(task, content_file) Ok((task, content_file))
}) })
})) })))
} }
pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<v6::Key>> + '_> { pub fn keys(&mut self) -> Result<Box<dyn Iterator<Item = Result<v6::Key>> + '_>> {
let keys = match self { let keys = match self {
CompatV5ToV6::V5(v5) => v5.keys(), CompatV5ToV6::V5(v5) => v5.keys()?,
CompatV5ToV6::Compat(compat) => compat.keys(), CompatV5ToV6::Compat(compat) => compat.keys(),
}; };
Box::new(keys.map(|key| {
Ok(Box::new(keys.map(|key| {
key.map(|key| v6::Key { key.map(|key| v6::Key {
description: key.description, description: key.description,
name: key.name, name: key.name,
@@ -186,7 +192,7 @@ impl CompatV5ToV6 {
created_at: key.created_at, created_at: key.created_at,
updated_at: key.updated_at, updated_at: key.updated_at,
}) })
})) })))
} }
} }
@@ -412,7 +418,7 @@ pub(crate) mod test {
insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d"); insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
// tasks // tasks
let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap(); let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip(); let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"8c6cd41457c0b7e4c6727c9c85b7abac"); meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"8c6cd41457c0b7e4c6727c9c85b7abac");
assert_eq!(update_files.len(), 22); assert_eq!(update_files.len(), 22);
@@ -421,7 +427,7 @@ pub(crate) mod test {
assert!(update_files[2..].iter().all(|u| u.is_none())); // everything already processed assert!(update_files[2..].iter().all(|u| u.is_none())); // everything already processed
// keys // keys
let keys = dump.keys().collect::<Result<Vec<_>>>().unwrap(); let keys = dump.keys().unwrap().collect::<Result<Vec<_>>>().unwrap();
meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"c9d2b467fe2fca0b35580d8a999808fb"); meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"c9d2b467fe2fca0b35580d8a999808fb");
// indexes // indexes

8
dump/src/reader/mod.rs
View File

@@ -98,16 +98,16 @@ impl DumpReader {
pub fn tasks( pub fn tasks(
&mut self, &mut self,
) -> Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_> { ) -> Result<Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_>> {
match self { match self {
DumpReader::Current(current) => current.tasks(), DumpReader::Current(current) => Ok(current.tasks()),
DumpReader::Compat(compat) => compat.tasks(), DumpReader::Compat(compat) => compat.tasks(),
} }
} }
pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<v6::Key>> + '_> { pub fn keys(&mut self) -> Result<Box<dyn Iterator<Item = Result<v6::Key>> + '_>> {
match self { match self {
DumpReader::Current(current) => current.keys(), DumpReader::Current(current) => Ok(current.keys()),
DumpReader::Compat(compat) => compat.keys(), DumpReader::Compat(compat) => compat.keys(),
} }
} }

13
dump/src/reader/v5/mod.rs
View File

@@ -34,7 +34,7 @@
use std::{ use std::{
fs::{self, File}, fs::{self, File},
io::{BufRead, BufReader}, io::{BufRead, BufReader, Seek, SeekFrom},
path::Path, path::Path,
}; };
@@ -176,12 +176,11 @@ impl V5Reader {
})) }))
} }
pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<Key>> + '_> { pub fn keys(&mut self) -> Result<Box<dyn Iterator<Item = Result<Key>> + '_>> {
Box::new( self.keys.seek(SeekFrom::Start(0))?;
(&mut self.keys) Ok(Box::new((&mut self.keys).lines().map(
.lines() |line| -> Result<_> { Ok(serde_json::from_str(&line?)?) },
.map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }), )))
)
} }
} }

12
index-scheduler/src/lib.rs
View File

@@ -11,10 +11,8 @@ pub type TaskId = u32;
use dump::{KindDump, TaskDump, UpdateFile}; use dump::{KindDump, TaskDump, UpdateFile};
pub use error::Error; pub use error::Error;
use meilisearch_types::keys::Key;
use meilisearch_types::milli::documents::DocumentsBatchBuilder; use meilisearch_types::milli::documents::DocumentsBatchBuilder;
use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task}; use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task};
use meilisearch_types::InstanceUid;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
@@ -400,8 +398,6 @@ impl IndexScheduler {
&mut self, &mut self,
task: TaskDump, task: TaskDump,
content_file: Option<Box<UpdateFile>>, content_file: Option<Box<UpdateFile>>,
keys: &[Key],
instance_uid: Option<InstanceUid>,
) -> Result<Task> { ) -> Result<Task> {
// Currently we don't need to access the tasks queue while loading a dump thus I can block everything. // Currently we don't need to access the tasks queue while loading a dump thus I can block everything.
let mut wtxn = self.env.write_txn()?; let mut wtxn = self.env.write_txn()?;
@@ -479,9 +475,13 @@ impl IndexScheduler {
KindDump::DeleteTasks { query, tasks } => { KindDump::DeleteTasks { query, tasks } => {
KindWithContent::TaskDeletion { query, tasks } KindWithContent::TaskDeletion { query, tasks }
} }
KindDump::DumpExport { dump_uid } => KindWithContent::DumpExport { KindDump::DumpExport {
dump_uid, dump_uid,
keys: keys.to_vec(), keys,
instance_uid,
} => KindWithContent::DumpExport {
dump_uid,
keys,
instance_uid, instance_uid,
}, },
KindDump::Snapshot => KindWithContent::Snapshot, KindDump::Snapshot => KindWithContent::Snapshot,

6
meilisearch-http/src/lib.rs
View File

@@ -190,7 +190,7 @@ fn import_dump(
// 2. Import the `Key`s. // 2. Import the `Key`s.
let mut keys = Vec::new(); let mut keys = Vec::new();
auth.raw_delete_all_keys()?; auth.raw_delete_all_keys()?;
for key in dump_reader.keys() { for key in dump_reader.keys()? {
let key = key?; let key = key?;
auth.raw_insert_key(key.clone())?; auth.raw_insert_key(key.clone())?;
keys.push(key); keys.push(key);
@@ -259,9 +259,9 @@ fn import_dump(
} }
// 4. Import the tasks. // 4. Import the tasks.
for ret in dump_reader.tasks() { for ret in dump_reader.tasks()? {
let (task, file) = ret?; let (task, file) = ret?;
index_scheduler.register_dumped_task(task, file, &keys, instance_uid)?; index_scheduler.register_dumped_task(task, file)?;
} }
Ok(()) Ok(())
} }