snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-07-28 01:01:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

WIP: refactor IndexController

Browse Source 
change the architecture of the index controller to allow it to own an
index store.
This commit is contained in:
mpostma
2021-01-16 15:09:48 +01:00
parent 686f987180
commit 6a3f625e11
15 changed files with 1197 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
src/data/updates.rs
View File

@ -1,18 +1,20 @@
use std::ops::Deref;
use milli::update::{IndexDocumentsMethod, UpdateFormat};
//use milli::update_store::UpdateStatus;
use async_compression::tokio_02::write::GzipEncoder;
use futures_util::stream::StreamExt;
use tokio::io::AsyncWriteExt;
use milli::update::{IndexDocumentsMethod, UpdateFormat};
use milli::update_store::UpdateStatus;
use super::Data;
use crate::updates::{UpdateMeta, UpdateResult, UpdateStatusResponse, Settings};
use crate::index_controller::IndexController;
use crate::index_controller::{UpdateStatusResponse, Settings};
impl Data {
pub async fn add_documents<B, E, S>(
&self,
_index: S,
index: S,
method: IndexDocumentsMethod,
format: UpdateFormat,
mut stream: impl futures::Stream<Item=Result<B, E>> + Unpin,
@ -20,7 +22,7 @@ impl Data {
where
B: Deref<Target = [u8]>,
E: std::error::Error + Send + Sync + 'static,
S: AsRef<str>,
S: AsRef<str> + Send + Sync + 'static,
{
let file = tokio::task::spawn_blocking(tempfile::tempfile).await?;
let file = tokio::fs::File::from_std(file?);
@ -37,43 +39,39 @@ impl Data {
let file = file.into_std().await;
let mmap = unsafe { memmap::Mmap::map(&file)? };
let meta = UpdateMeta::DocumentsAddition { method, format };
let queue = self.update_queue.clone();
let update = tokio::task::spawn_blocking(move || queue.register_update(meta, &mmap[..])).await??;
let indexes = self.indexes.clone();
let update = tokio::task::spawn_blocking(move ||indexes.add_documents(index, method, format, &mmap[..])).await??;
Ok(update.into())
}
pub async fn update_settings<S: AsRef<str>>(
pub async fn update_settings<S: AsRef<str> + Send + Sync + 'static>(
&self,
_index: S,
index: S,
settings: Settings
) -> anyhow::Result<UpdateStatusResponse> {
let meta = UpdateMeta::Settings(settings);
let queue = self.update_queue.clone();
let update = tokio::task::spawn_blocking(move || queue.register_update(meta, &[])).await??;
let indexes = self.indexes.clone();
let update = tokio::task::spawn_blocking(move || indexes.update_settings(index, settings)).await??;
Ok(update.into())
}
#[inline]
pub fn get_update_status(&self, _index: &str, uid: u64) -> anyhow::Result<Option<UpdateStatus<UpdateMeta, UpdateResult, String>>> {
self.update_queue.get_update_status(uid)
}
//#[inline]
//pub fn get_update_status<S: AsRef<str>>(&self, _index: S, uid: u64) -> anyhow::Result<Option<UpdateStatus<UpdateMeta, UpdateResult, String>>> {
//self.indexes.get_update_status(uid)
//}
pub fn get_updates_status(&self, _index: &str) -> anyhow::Result<Vec<UpdateStatus<UpdateMeta, UpdateResult, String>>> {
let result = self.update_queue.iter_metas(|processing, processed, pending, aborted, failed| {
let mut metas = processing
.map(UpdateStatus::from)
.into_iter()
.chain(processed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from))
.chain(pending.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from))
.chain(aborted.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from))
.chain(failed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from))
.collect::<Vec<_>>();
metas.sort_by(|a, b| a.id().cmp(&b.id()));
Ok(metas)
})?;
Ok(result)
}
//pub fn get_updates_status(&self, _index: &str) -> anyhow::Result<Vec<UpdateStatus<UpdateMeta, UpdateResult, String>>> {
//let result = self.update_queue.iter_metas(|processing, processed, pending, aborted, failed| {
//let mut metas = processing
//.map(UpdateStatus::from)
//.into_iter()
//.chain(processed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from))
//.chain(pending.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from))
//.chain(aborted.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from))
//.chain(failed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from))
//.collect::<Vec<_>>();
//metas.sort_by(|a, b| a.id().cmp(&b.id()));
//Ok(metas)
//})?;
//Ok(result)
//}
}