snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-09-12 07:36:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

WIP rebase on main

Browse Source
This commit is contained in:
tamo
2021-05-05 14:11:56 +02:00
parent 0f94ef8abc
commit c3552cecdf
20 changed files with 158 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
meilisearch-http/src/index_controller/update_actor/actor.rs
View File

@ -235,11 +235,11 @@ where
Ok(())
}
async fn handle_dump(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()> {
async fn handle_dump(&self, uuids: HashSet<(String, Uuid)>, path: PathBuf) -> Result<()> {
let index_handle = self.index_handle.clone();
let update_store = self.store.clone();
tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
update_store.dump(&uuids, &path)?;
update_store.dump(&uuids, path.to_path_buf())?;
// Perform the snapshot of each index concurently. Only a third of the capabilities of
// the index actor at a time not to put too much pressure on the index actor
@ -247,7 +247,7 @@ where
let handle = &index_handle;
let mut stream = futures::stream::iter(uuids.iter())
.map(|&uuid| handle.dump(uuid, path.clone()))
.map(|(uid, uuid)| handle.dump(uid.clone(), *uuid, path.clone()))
.buffer_unordered(CONCURRENT_INDEX_MSG / 3);
Handle::current().block_on(async {

8
meilisearch-http/src/index_controller/update_actor/handle_impl.rs
View File

@ -71,16 +71,16 @@ where
receiver.await.expect("update actor killed.")
}
async fn get_info(&self) -> Result<UpdateStoreInfo> {
async fn dump(&self, uuids: HashSet<(String, Uuid)>, path: PathBuf) -> Result<()> {
let (ret, receiver) = oneshot::channel();
let msg = UpdateMsg::GetInfo { ret };
let msg = UpdateMsg::Dump { uuids, path, ret };
let _ = self.sender.send(msg).await;
receiver.await.expect("update actor killed.")
}
async fn dump(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()> {
async fn get_info(&self) -> Result<UpdateStoreInfo> {
let (ret, receiver) = oneshot::channel();
let msg = UpdateMsg::Dump { uuids, path, ret };
let msg = UpdateMsg::GetInfo { ret };
let _ = self.sender.send(msg).await;
receiver.await.expect("update actor killed.")
}

2
meilisearch-http/src/index_controller/update_actor/message.rs
View File

@ -32,7 +32,7 @@ pub enum UpdateMsg<D> {
ret: oneshot::Sender<Result<()>>,
},
Dump {
uuids: HashSet<Uuid>,
uuids: HashSet<(String, Uuid)>,
path: PathBuf,
ret: oneshot::Sender<Result<()>>,
},

2
meilisearch-http/src/index_controller/update_actor/mod.rs
View File

@ -40,7 +40,7 @@ pub trait UpdateActorHandle {
async fn update_status(&self, uuid: Uuid, id: u64) -> Result<UpdateStatus>;
async fn delete(&self, uuid: Uuid) -> Result<()>;
async fn snapshot(&self, uuid: HashSet<Uuid>, path: PathBuf) -> Result<()>;
async fn dump(&self, uuid: HashSet<Uuid>, path: PathBuf) -> Result<()>;
async fn dump(&self, uuid: HashSet<(String, Uuid)>, path: PathBuf) -> Result<()>;
async fn get_info(&self) -> Result<UpdateStoreInfo>;
async fn update(
&self,

30
meilisearch-http/src/index_controller/update_actor/update_store.rs
View File

@ -1,4 +1,4 @@
use std::borrow::Cow;
use std::{borrow::Cow, path::PathBuf};
use std::collections::{BTreeMap, HashSet};
use std::convert::TryInto;
use std::fs::{copy, create_dir_all, remove_file, File};
@ -294,6 +294,7 @@ impl UpdateStore {
result: UpdateStatus,
index_uuid: Uuid,
) -> heed::Result<()> {
// TODO: TAMO: load already processed updates
let mut wtxn = self.env.write_txn()?;
let (_global_id, update_id) = self.next_update_id(&mut wtxn, index_uuid)?;
self.updates.remap_key_type::<UpdateKeyCodec>().put(&mut wtxn, &(index_uuid, update_id), &result)?;
@ -516,31 +517,34 @@ impl UpdateStore {
Ok(())
}
pub fn dump(&self, uuids: &HashSet<Uuid>, path: impl AsRef<Path>) -> anyhow::Result<()> {
pub fn dump(&self, uuids: &HashSet<(String, Uuid)>, path: PathBuf) -> anyhow::Result<()> {
use std::io::prelude::*;
let state_lock = self.state.write();
state_lock.swap(State::Snapshoting); // TODO: rename the state
state_lock.swap(State::Snapshoting); // TODO: TAMO rename the state somehow
let txn = self.env.write_txn()?;
let update_path = path.as_ref().join("updates");
create_dir_all(&update_path)?;
for (uid, uuid) in uuids.iter() {
let file = File::create(path.join(uid).join("updates.jsonl"))?;
let mut file = std::io::BufWriter::new(file);
// acquire write lock to prevent further writes during dump
create_dir_all(&update_path)?;
let db_path = update_path.join("data.mdb");
for update in &self.list(*uuid)? {
serde_json::to_writer(&mut file, update)?;
file.write_all(b"\n")?;
}
}
// TODO: everything
// create db dump
self.env.copy_to_path(&db_path, CompactionOption::Enabled)?;
// TODO: TAMO: the updates
// already processed updates seems to works, but I've not tried with currently running updates
let update_files_path = update_path.join("update_files");
let update_files_path = path.join("update_files");
create_dir_all(&update_files_path)?;
let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data();
for entry in pendings {
let ((_, uuid, _), pending) = entry?;
if uuids.contains(&uuid) {
if uuids.iter().any(|(_, id)| id == &uuid) {
if let Some(path) = pending.decode()?.content_path() {
let name = path.file_name().unwrap();
let to = update_files_path.join(name);