snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-07-27 00:31:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

WIP: refactor IndexController

Browse Source 
change the architecture of the index controller to allow it to own an
index store.
This commit is contained in:
mpostma
2021-01-16 15:09:48 +01:00
parent 686f987180
commit 6a3f625e11
15 changed files with 1197 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
src/data/mod.rs
View File

@ -8,17 +8,16 @@ use std::sync::Arc;
use sha2::Digest;
use crate::{option::Opt, updates::Settings};
use crate::updates::UpdateQueue;
use crate::index_controller::IndexController;
use crate::{option::Opt, index_controller::Settings};
use crate::index_controller::{IndexStore, UpdateStore};
#[derive(Clone)]
pub struct Data {
inner: Arc<DataInner>,
inner: Arc<DataInner<UpdateStore>>,
}
impl Deref for Data {
type Target = DataInner;
type Target = DataInner<UpdateStore>;
fn deref(&self) -> &Self::Target {
&self.inner
@ -26,8 +25,8 @@ impl Deref for Data {
}
#[derive(Clone)]
pub struct DataInner {
pub indexes: Arc<IndexController<UpdateQueue>>,
pub struct DataInner<I> {
pub indexes: Arc<I>,
api_keys: ApiKeys,
options: Opt,
}
@ -58,11 +57,10 @@ impl ApiKeys {
impl Data {
pub fn new(options: Opt) -> anyhow::Result<Data> {
let db_size = options.max_mdb_size.get_bytes() as usize;
let indexes = IndexController::new(&options.db_path)?;
let indexes = Arc::new(indexes);
let update_queue = Arc::new(UpdateQueue::new(&options, indexes.clone())?);
let path = options.db_path.clone();
let index_store = IndexStore::new(&path)?;
let index_controller = UpdateStore::new(index_store);
let indexes = Arc::new(index_controller);
let mut api_keys = ApiKeys {
master: options.clone().master_key,
@ -72,31 +70,28 @@ impl Data {
api_keys.generate_missing_api_keys();
let inner = DataInner { indexes, options, update_queue, api_keys };
let inner = DataInner { indexes, options, api_keys };
let inner = Arc::new(inner);
Ok(Data { inner })
}
pub fn settings<S: AsRef<str>>(&self, _index: S) -> anyhow::Result<Settings> {
let txn = self.indexes.env.read_txn()?;
let fields_map = self.indexes.fields_ids_map(&txn)?;
println!("fields_map: {:?}", fields_map);
pub fn settings<S: AsRef<str>>(&self, index_uid: S) -> anyhow::Result<Settings> {
let index = self.indexes
.get(&index_uid)?
.ok_or_else(|| anyhow::anyhow!("Index {} does not exist.", index_uid.as_ref()))?;
let displayed_attributes = self.indexes
.displayed_fields(&txn)?
let displayed_attributes = index
.displayed_fields()?
.map(|fields| fields.into_iter().map(String::from).collect())
.unwrap_or_else(|| vec!["*".to_string()]);
let searchable_attributes = self.indexes
.searchable_fields(&txn)?
.map(|fields| fields
.into_iter()
.map(String::from)
.collect())
let searchable_attributes = index
.searchable_fields()?
.map(|fields| fields.into_iter().map(String::from).collect())
.unwrap_or_else(|| vec!["*".to_string()]);
let faceted_attributes = self.indexes.faceted_fields(&txn)?
let faceted_attributes = index.faceted_fields()?
.into_iter()
.map(|(k, v)| (k, v.to_string()))
.collect();

4
src/data/search.rs
View File

@ -107,10 +107,10 @@ impl Data {
pub fn search<S: AsRef<str>>(&self, index: S, search_query: SearchQuery) -> anyhow::Result<SearchResult> {
let start = Instant::now();
let index = self.indexes
.get(index)?
.get(&index)?
.ok_or_else(|| Error::OpenIndex(format!("Index {} doesn't exists.", index.as_ref())))?;
let Results { found_words, documents_ids, nb_hits, limit, .. } = index.search(search_query)?;
let Results { found_words, documents_ids, nb_hits, limit, .. } = index.search(&search_query)?;
let fields_ids_map = index.fields_ids_map()?;

66
src/data/updates.rs
View File

@ -1,18 +1,20 @@
use std::ops::Deref;
use milli::update::{IndexDocumentsMethod, UpdateFormat};
//use milli::update_store::UpdateStatus;
use async_compression::tokio_02::write::GzipEncoder;
use futures_util::stream::StreamExt;
use tokio::io::AsyncWriteExt;
use milli::update::{IndexDocumentsMethod, UpdateFormat};
use milli::update_store::UpdateStatus;
use super::Data;
use crate::updates::{UpdateMeta, UpdateResult, UpdateStatusResponse, Settings};
use crate::index_controller::IndexController;
use crate::index_controller::{UpdateStatusResponse, Settings};
impl Data {
pub async fn add_documents<B, E, S>(
&self,
_index: S,
index: S,
method: IndexDocumentsMethod,
format: UpdateFormat,
mut stream: impl futures::Stream<Item=Result<B, E>> + Unpin,
@ -20,7 +22,7 @@ impl Data {
where
B: Deref<Target = [u8]>,
E: std::error::Error + Send + Sync + 'static,
S: AsRef<str>,
S: AsRef<str> + Send + Sync + 'static,
{
let file = tokio::task::spawn_blocking(tempfile::tempfile).await?;
let file = tokio::fs::File::from_std(file?);
@ -37,43 +39,39 @@ impl Data {
let file = file.into_std().await;
let mmap = unsafe { memmap::Mmap::map(&file)? };
let meta = UpdateMeta::DocumentsAddition { method, format };
let queue = self.update_queue.clone();
let update = tokio::task::spawn_blocking(move || queue.register_update(meta, &mmap[..])).await??;
let indexes = self.indexes.clone();
let update = tokio::task::spawn_blocking(move ||indexes.add_documents(index, method, format, &mmap[..])).await??;
Ok(update.into())
}
pub async fn update_settings<S: AsRef<str>>(
pub async fn update_settings<S: AsRef<str> + Send + Sync + 'static>(
&self,
_index: S,
index: S,
settings: Settings
) -> anyhow::Result<UpdateStatusResponse> {
let meta = UpdateMeta::Settings(settings);
let queue = self.update_queue.clone();
let update = tokio::task::spawn_blocking(move || queue.register_update(meta, &[])).await??;
let indexes = self.indexes.clone();
let update = tokio::task::spawn_blocking(move || indexes.update_settings(index, settings)).await??;
Ok(update.into())
}
#[inline]
pub fn get_update_status(&self, _index: &str, uid: u64) -> anyhow::Result<Option<UpdateStatus<UpdateMeta, UpdateResult, String>>> {
self.update_queue.get_update_status(uid)
}
//#[inline]
//pub fn get_update_status<S: AsRef<str>>(&self, _index: S, uid: u64) -> anyhow::Result<Option<UpdateStatus<UpdateMeta, UpdateResult, String>>> {
//self.indexes.get_update_status(uid)
//}
pub fn get_updates_status(&self, _index: &str) -> anyhow::Result<Vec<UpdateStatus<UpdateMeta, UpdateResult, String>>> {
let result = self.update_queue.iter_metas(|processing, processed, pending, aborted, failed| {
let mut metas = processing
.map(UpdateStatus::from)
.into_iter()
.chain(processed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from))
.chain(pending.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from))
.chain(aborted.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from))
.chain(failed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from))
.collect::<Vec<_>>();
metas.sort_by(|a, b| a.id().cmp(&b.id()));
Ok(metas)
})?;
Ok(result)
}
//pub fn get_updates_status(&self, _index: &str) -> anyhow::Result<Vec<UpdateStatus<UpdateMeta, UpdateResult, String>>> {
//let result = self.update_queue.iter_metas(|processing, processed, pending, aborted, failed| {
//let mut metas = processing
//.map(UpdateStatus::from)
//.into_iter()
//.chain(processed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from))
//.chain(pending.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from))
//.chain(aborted.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from))
//.chain(failed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from))
//.collect::<Vec<_>>();
//metas.sort_by(|a, b| a.id().cmp(&b.id()));
//Ok(metas)
//})?;
//Ok(result)
//}
}