mirror of
https://github.com/meilisearch/meilisearch.git
synced 2025-07-27 00:31:02 +00:00
feat(all): Task store
implements: https://github.com/meilisearch/specifications/blob/develop/text/0060-refashion-updates-apis.md linked PR: - #1889 - #1891 - #1892 - #1902 - #1906 - #1911 - #1914 - #1915 - #1916 - #1918 - #1924 - #1925 - #1926 - #1930 - #1936 - #1937 - #1942 - #1944 - #1945 - #1946 - #1947 - #1950 - #1951 - #1957 - #1959 - #1960 - #1961 - #1962 - #1964 - https://github.com/meilisearch/milli/pull/414 - https://github.com/meilisearch/milli/pull/409 - https://github.com/meilisearch/milli/pull/406 - https://github.com/meilisearch/milli/pull/418 - close #1687 - close #1786 - close #1940 - close #1948 - close #1949 - close #1932 - close #1956
This commit is contained in:
Marin Postma
@ -5,7 +5,7 @@ mod dump;
|
||||
pub mod error;
|
||||
mod search;
|
||||
pub mod update_handler;
|
||||
mod updates;
|
||||
pub mod updates;
|
||||
|
||||
#[allow(clippy::module_inception)]
|
||||
mod index;
|
||||
@ -22,191 +22,40 @@ pub use test::MockIndex as Index;
|
||||
/// code for unit testing, in places where an index would normally be used.
|
||||
#[cfg(test)]
|
||||
pub mod test {
|
||||
use std::any::Any;
|
||||
use std::collections::HashMap;
|
||||
use std::panic::{RefUnwindSafe, UnwindSafe};
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::Arc;
|
||||
|
||||
use milli::update::{DocumentAdditionResult, DocumentDeletionResult, IndexDocumentsMethod};
|
||||
use nelson::Mocker;
|
||||
use serde_json::{Map, Value};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::index_controller::update_file_store::UpdateFileStore;
|
||||
use crate::index_controller::updates::status::{Failed, Processed, Processing};
|
||||
|
||||
use super::error::Result;
|
||||
use super::index::Index;
|
||||
use super::update_handler::UpdateHandler;
|
||||
use super::{Checked, IndexMeta, IndexStats, SearchQuery, SearchResult, Settings};
|
||||
use crate::update_file_store::UpdateFileStore;
|
||||
|
||||
pub struct Stub<A, R> {
|
||||
name: String,
|
||||
times: Mutex<Option<usize>>,
|
||||
stub: Box<dyn Fn(A) -> R + Sync + Send>,
|
||||
invalidated: AtomicBool,
|
||||
}
|
||||
|
||||
impl<A, R> Drop for Stub<A, R> {
|
||||
fn drop(&mut self) {
|
||||
if !self.invalidated.load(Ordering::Relaxed) {
|
||||
let lock = self.times.lock().unwrap();
|
||||
if let Some(n) = *lock {
|
||||
assert_eq!(n, 0, "{} not called enough times", self.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<A, R> Stub<A, R> {
|
||||
fn invalidate(&self) {
|
||||
self.invalidated.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
impl<A: UnwindSafe, R> Stub<A, R> {
|
||||
fn call(&self, args: A) -> R {
|
||||
let mut lock = self.times.lock().unwrap();
|
||||
match *lock {
|
||||
Some(0) => panic!("{} called to many times", self.name),
|
||||
Some(ref mut times) => {
|
||||
*times -= 1;
|
||||
}
|
||||
None => (),
|
||||
}
|
||||
|
||||
// Since we add assertions in the drop implementation for Stub, a panic can occur in a
|
||||
// panic, causing a hard abort of the program. To handle that, we catch the panic, and
|
||||
// set the stub as invalidated so the assertions aren't run during the drop.
|
||||
impl<'a, A, R> RefUnwindSafe for StubHolder<'a, A, R> {}
|
||||
struct StubHolder<'a, A, R>(&'a (dyn Fn(A) -> R + Sync + Send));
|
||||
|
||||
let stub = StubHolder(self.stub.as_ref());
|
||||
|
||||
match std::panic::catch_unwind(|| (stub.0)(args)) {
|
||||
Ok(r) => r,
|
||||
Err(panic) => {
|
||||
self.invalidate();
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct StubStore {
|
||||
inner: Arc<Mutex<HashMap<String, Box<dyn Any + Sync + Send>>>>,
|
||||
}
|
||||
|
||||
impl StubStore {
|
||||
pub fn insert<A: 'static, R: 'static>(&self, name: String, stub: Stub<A, R>) {
|
||||
let mut lock = self.inner.lock().unwrap();
|
||||
lock.insert(name, Box::new(stub));
|
||||
}
|
||||
|
||||
pub fn get<A, B>(&self, name: &str) -> Option<&Stub<A, B>> {
|
||||
let mut lock = self.inner.lock().unwrap();
|
||||
match lock.get_mut(name) {
|
||||
Some(s) => {
|
||||
let s = s.as_mut() as *mut dyn Any as *mut Stub<A, B>;
|
||||
Some(unsafe { &mut *s })
|
||||
}
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct StubBuilder<'a, A, R> {
|
||||
name: String,
|
||||
store: &'a StubStore,
|
||||
times: Option<usize>,
|
||||
_f: std::marker::PhantomData<fn(A) -> R>,
|
||||
}
|
||||
|
||||
impl<'a, A: 'static, R: 'static> StubBuilder<'a, A, R> {
|
||||
/// Asserts the stub has been called exactly `times` times.
|
||||
#[must_use]
|
||||
pub fn times(mut self, times: usize) -> Self {
|
||||
self.times = Some(times);
|
||||
self
|
||||
}
|
||||
|
||||
/// Asserts the stub has been called exactly once.
|
||||
#[must_use]
|
||||
pub fn once(mut self) -> Self {
|
||||
self.times = Some(1);
|
||||
self
|
||||
}
|
||||
|
||||
/// The function that will be called when the stub is called. This needs to be called to
|
||||
/// actually build the stub and register it to the stub store.
|
||||
pub fn then(self, f: impl Fn(A) -> R + Sync + Send + 'static) {
|
||||
let times = Mutex::new(self.times);
|
||||
let stub = Stub {
|
||||
stub: Box::new(f),
|
||||
times,
|
||||
name: self.name.clone(),
|
||||
invalidated: AtomicBool::new(false),
|
||||
};
|
||||
|
||||
self.store.insert(self.name, stub);
|
||||
}
|
||||
}
|
||||
|
||||
/// Mocker allows to stub metod call on any struct. you can register stubs by calling
|
||||
/// `Mocker::when` and retrieve it in the proxy implementation when with `Mocker::get`.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Mocker {
|
||||
store: StubStore,
|
||||
}
|
||||
|
||||
impl Mocker {
|
||||
pub fn when<A, R>(&self, name: &str) -> StubBuilder<A, R> {
|
||||
StubBuilder {
|
||||
name: name.to_string(),
|
||||
store: &self.store,
|
||||
times: None,
|
||||
_f: std::marker::PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get<A, R>(&self, name: &str) -> &Stub<A, R> {
|
||||
match self.store.get(name) {
|
||||
Some(stub) => stub,
|
||||
None => {
|
||||
// panic here causes the stubs to get dropped, and panic in turn. To prevent
|
||||
// that, we forget them, and let them be cleaned by the os later. This is not
|
||||
// optimal, but is still better than nested panicks.
|
||||
let mut stubs = self.store.inner.lock().unwrap();
|
||||
let stubs = std::mem::take(&mut *stubs);
|
||||
std::mem::forget(stubs);
|
||||
panic!("unexpected call to {}", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Clone)]
|
||||
pub enum MockIndex {
|
||||
Vrai(Index),
|
||||
Faux(Arc<Mocker>),
|
||||
Real(Index),
|
||||
Mock(Arc<Mocker>),
|
||||
}
|
||||
|
||||
impl MockIndex {
|
||||
pub fn faux(faux: Mocker) -> Self {
|
||||
Self::Faux(Arc::new(faux))
|
||||
pub fn mock(mocker: Mocker) -> Self {
|
||||
Self::Mock(Arc::new(mocker))
|
||||
}
|
||||
|
||||
pub fn open(
|
||||
path: impl AsRef<Path>,
|
||||
size: usize,
|
||||
update_file_store: Arc<UpdateFileStore>,
|
||||
uuid: Uuid,
|
||||
update_handler: Arc<UpdateHandler>,
|
||||
) -> Result<Self> {
|
||||
let index = Index::open(path, size, update_file_store, uuid, update_handler)?;
|
||||
Ok(Self::Vrai(index))
|
||||
let index = Index::open(path, size, uuid, update_handler)?;
|
||||
Ok(Self::Real(index))
|
||||
}
|
||||
|
||||
pub fn load_dump(
|
||||
@ -215,41 +64,33 @@ pub mod test {
|
||||
size: usize,
|
||||
update_handler: &UpdateHandler,
|
||||
) -> anyhow::Result<()> {
|
||||
Index::load_dump(src, dst, size, update_handler)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn handle_update(&self, update: Processing) -> std::result::Result<Processed, Failed> {
|
||||
match self {
|
||||
MockIndex::Vrai(index) => index.handle_update(update),
|
||||
MockIndex::Faux(faux) => faux.get("handle_update").call(update),
|
||||
}
|
||||
Index::load_dump(src, dst, size, update_handler)
|
||||
}
|
||||
|
||||
pub fn uuid(&self) -> Uuid {
|
||||
match self {
|
||||
MockIndex::Vrai(index) => index.uuid(),
|
||||
MockIndex::Faux(faux) => faux.get("uuid").call(()),
|
||||
MockIndex::Real(index) => index.uuid(),
|
||||
MockIndex::Mock(m) => unsafe { m.get("uuid").call(()) },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stats(&self) -> Result<IndexStats> {
|
||||
match self {
|
||||
MockIndex::Vrai(index) => index.stats(),
|
||||
MockIndex::Faux(_) => todo!(),
|
||||
MockIndex::Real(index) => index.stats(),
|
||||
MockIndex::Mock(m) => unsafe { m.get("stats").call(()) },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn meta(&self) -> Result<IndexMeta> {
|
||||
match self {
|
||||
MockIndex::Vrai(index) => index.meta(),
|
||||
MockIndex::Faux(_) => todo!(),
|
||||
MockIndex::Real(index) => index.meta(),
|
||||
MockIndex::Mock(_) => todo!(),
|
||||
}
|
||||
}
|
||||
pub fn settings(&self) -> Result<Settings<Checked>> {
|
||||
match self {
|
||||
MockIndex::Vrai(index) => index.settings(),
|
||||
MockIndex::Faux(_) => todo!(),
|
||||
MockIndex::Real(index) => index.settings(),
|
||||
MockIndex::Mock(_) => todo!(),
|
||||
}
|
||||
}
|
||||
|
||||
@ -260,10 +101,10 @@ pub mod test {
|
||||
attributes_to_retrieve: Option<Vec<S>>,
|
||||
) -> Result<Vec<Map<String, Value>>> {
|
||||
match self {
|
||||
MockIndex::Vrai(index) => {
|
||||
MockIndex::Real(index) => {
|
||||
index.retrieve_documents(offset, limit, attributes_to_retrieve)
|
||||
}
|
||||
MockIndex::Faux(_) => todo!(),
|
||||
MockIndex::Mock(_) => todo!(),
|
||||
}
|
||||
}
|
||||
|
||||
@ -273,49 +114,93 @@ pub mod test {
|
||||
attributes_to_retrieve: Option<Vec<S>>,
|
||||
) -> Result<Map<String, Value>> {
|
||||
match self {
|
||||
MockIndex::Vrai(index) => index.retrieve_document(doc_id, attributes_to_retrieve),
|
||||
MockIndex::Faux(_) => todo!(),
|
||||
MockIndex::Real(index) => index.retrieve_document(doc_id, attributes_to_retrieve),
|
||||
MockIndex::Mock(_) => todo!(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn size(&self) -> u64 {
|
||||
match self {
|
||||
MockIndex::Vrai(index) => index.size(),
|
||||
MockIndex::Faux(_) => todo!(),
|
||||
MockIndex::Real(index) => index.size(),
|
||||
MockIndex::Mock(_) => todo!(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn snapshot(&self, path: impl AsRef<Path>) -> Result<()> {
|
||||
match self {
|
||||
MockIndex::Vrai(index) => index.snapshot(path),
|
||||
MockIndex::Faux(faux) => faux.get("snapshot").call(path.as_ref()),
|
||||
MockIndex::Real(index) => index.snapshot(path),
|
||||
MockIndex::Mock(m) => unsafe { m.get("snapshot").call(path.as_ref()) },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn inner(&self) -> &milli::Index {
|
||||
pub fn close(self) {
|
||||
match self {
|
||||
MockIndex::Vrai(index) => index.inner(),
|
||||
MockIndex::Faux(_) => todo!(),
|
||||
MockIndex::Real(index) => index.close(),
|
||||
MockIndex::Mock(m) => unsafe { m.get("close").call(()) },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update_primary_key(&self, primary_key: Option<String>) -> Result<IndexMeta> {
|
||||
match self {
|
||||
MockIndex::Vrai(index) => index.update_primary_key(primary_key),
|
||||
MockIndex::Faux(_) => todo!(),
|
||||
}
|
||||
}
|
||||
pub fn perform_search(&self, query: SearchQuery) -> Result<SearchResult> {
|
||||
match self {
|
||||
MockIndex::Vrai(index) => index.perform_search(query),
|
||||
MockIndex::Faux(faux) => faux.get("perform_search").call(query),
|
||||
MockIndex::Real(index) => index.perform_search(query),
|
||||
MockIndex::Mock(m) => unsafe { m.get("perform_search").call(query) },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn dump(&self, path: impl AsRef<Path>) -> Result<()> {
|
||||
match self {
|
||||
MockIndex::Vrai(index) => index.dump(path),
|
||||
MockIndex::Faux(faux) => faux.get("dump").call(path.as_ref()),
|
||||
MockIndex::Real(index) => index.dump(path),
|
||||
MockIndex::Mock(m) => unsafe { m.get("dump").call(path.as_ref()) },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update_documents(
|
||||
&self,
|
||||
method: IndexDocumentsMethod,
|
||||
content_uuid: Uuid,
|
||||
primary_key: Option<String>,
|
||||
file_store: UpdateFileStore,
|
||||
) -> Result<DocumentAdditionResult> {
|
||||
match self {
|
||||
MockIndex::Real(index) => {
|
||||
index.update_documents(method, content_uuid, primary_key, file_store)
|
||||
}
|
||||
MockIndex::Mock(mocker) => unsafe {
|
||||
mocker.get("update_documents").call((
|
||||
method,
|
||||
content_uuid,
|
||||
primary_key,
|
||||
file_store,
|
||||
))
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update_settings(&self, settings: &Settings<Checked>) -> Result<()> {
|
||||
match self {
|
||||
MockIndex::Real(index) => index.update_settings(settings),
|
||||
MockIndex::Mock(m) => unsafe { m.get("update_settings").call(settings) },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update_primary_key(&self, primary_key: String) -> Result<IndexMeta> {
|
||||
match self {
|
||||
MockIndex::Real(index) => index.update_primary_key(primary_key),
|
||||
MockIndex::Mock(m) => unsafe { m.get("update_primary_key").call(primary_key) },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn delete_documents(&self, ids: &[String]) -> Result<DocumentDeletionResult> {
|
||||
match self {
|
||||
MockIndex::Real(index) => index.delete_documents(ids),
|
||||
MockIndex::Mock(m) => unsafe { m.get("delete_documents").call(ids) },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn clear_documents(&self) -> Result<()> {
|
||||
match self {
|
||||
MockIndex::Real(index) => index.clear_documents(),
|
||||
MockIndex::Mock(m) => unsafe { m.get("clear_documents").call(()) },
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -327,7 +212,7 @@ pub mod test {
|
||||
.times(2)
|
||||
.then(|_: &Path| -> Result<()> { Ok(()) });
|
||||
|
||||
let index = MockIndex::faux(faux);
|
||||
let index = MockIndex::mock(faux);
|
||||
|
||||
let path = PathBuf::from("hello");
|
||||
index.snapshot(&path).unwrap();
|
||||
@ -339,7 +224,7 @@ pub mod test {
|
||||
fn test_faux_unexisting_method_stub() {
|
||||
let faux = Mocker::default();
|
||||
|
||||
let index = MockIndex::faux(faux);
|
||||
let index = MockIndex::mock(faux);
|
||||
|
||||
let path = PathBuf::from("hello");
|
||||
index.snapshot(&path).unwrap();
|
||||
@ -356,7 +241,7 @@ pub mod test {
|
||||
panic!();
|
||||
});
|
||||
|
||||
let index = MockIndex::faux(faux);
|
||||
let index = MockIndex::mock(faux);
|
||||
|
||||
let path = PathBuf::from("hello");
|
||||
index.snapshot(&path).unwrap();
|
||||
|
||||
Reference in New Issue
Block a user