From 750d3360187af6696483a3f9b12c310a8bd22790 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 19 Nov 2019 16:02:03 +0100 Subject: [PATCH 01/28] Bump Cargo.lock meili versions --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d637974f1..c6db089bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -839,7 +839,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "meilidb-core" -version = "0.7.0" +version = "0.8.0" dependencies = [ "arc-swap 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -856,9 +856,9 @@ dependencies = [ "indexmap 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "levenshtein_automata 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "meilidb-schema 0.6.0", - "meilidb-tokenizer 0.6.1", - "meilidb-types 0.1.0", + "meilidb-schema 0.8.0", + "meilidb-tokenizer 0.8.0", + "meilidb-types 0.8.0", "once_cell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "ordered-float 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "rustyline 5.0.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -876,7 +876,7 @@ dependencies = [ [[package]] name = "meilidb-http" -version = "0.3.0" +version = "0.8.0" dependencies = [ "async-compression 0.1.0-alpha.7 (registry+https://github.com/rust-lang/crates.io-index)", "bincode 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -889,8 +889,8 @@ dependencies = [ "jemallocator 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "main_error 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "meilidb-core 0.7.0", - "meilidb-schema 0.6.0", + "meilidb-core 0.8.0", + "meilidb-schema 0.8.0", "pretty-bytes 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -908,7 +908,7 @@ dependencies = [ [[package]] name = "meilidb-schema" -version = "0.6.0" +version = "0.8.0" dependencies = [ "bincode 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -919,7 +919,7 @@ dependencies = [ [[package]] name = "meilidb-tokenizer" -version = "0.6.1" +version = "0.8.0" dependencies = [ "deunicode 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "slice-group-by 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -927,7 +927,7 @@ dependencies = [ [[package]] name = "meilidb-types" -version = "0.1.0" +version = "0.8.0" dependencies = [ "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", "zerocopy 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", From 076e7818102297e3d50095202c8054546d235e41 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 19 Nov 2019 16:04:16 +0100 Subject: [PATCH 02/28] Add name, created_at and updated_at informations into main index --- meilidb-core/src/store/main.rs | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/meilidb-core/src/store/main.rs b/meilidb-core/src/store/main.rs index eeb04d21f..d35189ea9 100644 --- a/meilidb-core/src/store/main.rs +++ b/meilidb-core/src/store/main.rs @@ -1,17 +1,22 @@ +use chrono::{DateTime, Utc}; use crate::RankedMap; -use heed::types::{ByteSlice, OwnedType, SerdeBincode, Str}; use heed::Result as ZResult; +use heed::types::{ByteSlice, OwnedType, SerdeBincode, Str}; use meilidb_schema::Schema; use std::sync::Arc; +const CREATED_AT: &str = "created-at"; const CUSTOMS_KEY: &str = "customs-key"; +const NAME: &str = "name"; const NUMBER_OF_DOCUMENTS_KEY: &str = "number-of-documents"; const RANKED_MAP_KEY: &str = "ranked-map"; const SCHEMA_KEY: &str = "schema"; -const SYNONYMS_KEY: &str = "synonyms"; const STOP_WORDS_KEY: &str = "stop-words"; +const SYNONYMS_KEY: &str = "synonyms"; const WORDS_KEY: &str = "words"; +type SerdeDatetime = SerdeBincode>; + #[derive(Copy, Clone)] pub struct Main { pub(crate) main: heed::PolyDatabase, @@ -22,6 +27,22 @@ impl Main { self.main.clear(writer) } + pub fn name(self, reader: &heed::RoTxn) -> ZResult> { + Ok(self.main.get::(reader, NAME)?.map(|name| name.to_owned())) + } + + pub fn put_name(self, writer: &mut heed::RwTxn, name: &str) -> ZResult<()> { + self.main.put::(writer, NAME, name) + } + + pub fn created_at(self, reader: &heed::RoTxn) -> ZResult>> { + self.main.get::(reader, CREATED_AT) + } + + pub fn put_created_at(self, writer: &mut heed::RwTxn) -> ZResult<()> { + self.main.put::(writer, CREATED_AT, &Utc::now()) + } + pub fn put_words_fst(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> ZResult<()> { let bytes = fst.as_fst().as_bytes(); self.main.put::(writer, WORDS_KEY, bytes) From 5527457655180db23d6c225fc4a81d0448021908 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 19 Nov 2019 16:07:24 +0100 Subject: [PATCH 03/28] Rewrite create_index route new path, body request and response --- meilidb-http/src/routes/index.rs | 81 +++++++++++++++++++++++--------- meilidb-http/src/routes/mod.rs | 5 +- 2 files changed, 62 insertions(+), 24 deletions(-) diff --git a/meilidb-http/src/routes/index.rs b/meilidb-http/src/routes/index.rs index 0522d584f..e915a3573 100644 --- a/meilidb-http/src/routes/index.rs +++ b/meilidb-http/src/routes/index.rs @@ -1,9 +1,12 @@ +use meilidb_schema::Schema; use http::StatusCode; use meilidb_core::ProcessedUpdateResult; -use meilidb_schema::Schema; +use rand::seq::SliceRandom; +use serde::{Deserialize, Serialize}; use serde_json::json; use tide::response::IntoResponse; use tide::{Context, Response}; +use chrono::{DateTime, Utc}; use crate::error::{ResponseError, SResult}; use crate::helpers::tide::ContextExt; @@ -12,6 +15,15 @@ use crate::models::token::ACL::*; use crate::routes::document::IndexUpdateResponse; use crate::Data; +fn generate_uid() -> String { + let mut rng = rand::thread_rng(); + let sample = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + sample + .choose_multiple(&mut rng, 8) + .map(|c| *c as char) + .collect() +} + pub async fn list_indexes(ctx: Context) -> SResult { ctx.is_allowed(IndexesRead)?; let list = ctx @@ -48,23 +60,34 @@ pub async fn get_index_schema(ctx: Context) -> SResult { } } +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct IndexCreateRequest { + name: String, + schema: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct IndexCreateResponse { + name: String, + uid: String, + schema: Option, + update_id: Option, + created_at: DateTime, + updated_at: DateTime, +} + pub async fn create_index(mut ctx: Context) -> SResult { ctx.is_allowed(IndexesWrite)?; - let index_name = ctx.url_param("index")?; + let body = ctx.body_json::().await.map_err(ResponseError::bad_request)?; - let body = ctx.body_bytes().await.map_err(ResponseError::bad_request)?; - let schema: Option = if body.is_empty() { - None - } else { - serde_json::from_slice::(&body) - .map_err(ResponseError::bad_request) - .map(|s| Some(s.into()))? - }; + let generated_uid = generate_uid(); let db = &ctx.state().db; - let created_index = match db.create_index(&index_name) { + let created_index = match db.create_index(&generated_uid) { Ok(index) => index, Err(e) => return Err(ResponseError::create_index(e)), }; @@ -72,23 +95,37 @@ pub async fn create_index(mut ctx: Context) -> SResult { let env = &db.env; let mut writer = env.write_txn().map_err(ResponseError::internal)?; - match schema { - Some(schema) => { - let update_id = created_index + created_index.main + .put_name(&mut writer, &body.name) + .map_err(ResponseError::internal)?; + created_index.main + .put_created_at(&mut writer) + .map_err(ResponseError::internal)?; + created_index.main + + let schema: Option = body.schema.clone().map(|s| s.into()); + let mut response_update_id = None; + if let Some(schema) = schema { + let update_id = created_index .schema_update(&mut writer, schema.clone()) .map_err(ResponseError::internal)?; + response_update_id = Some(update_id) + } - writer.commit().map_err(ResponseError::internal)?; + writer.commit().map_err(ResponseError::internal)?; - let response_body = IndexUpdateResponse { update_id }; - Ok(tide::response::json(response_body) + let response_body = IndexCreateResponse { + name: body.name, + uid: generated_uid, + schema: body.schema, + update_id: response_update_id, + created_at: Utc::now(), + updated_at: Utc::now(), + }; + + Ok(tide::response::json(response_body) .with_status(StatusCode::CREATED) .into_response()) - } - None => Ok(Response::new(tide::Body::empty()) - .with_status(StatusCode::NO_CONTENT) - .into_response()), - } } pub async fn update_schema(mut ctx: Context) -> SResult { diff --git a/meilidb-http/src/routes/mod.rs b/meilidb-http/src/routes/mod.rs index efcf5d4f4..c44134cc8 100644 --- a/meilidb-http/src/routes/mod.rs +++ b/meilidb-http/src/routes/mod.rs @@ -13,7 +13,9 @@ pub mod synonym; pub fn load_routes(app: &mut tide::App) { app.at("").nest(|router| { router.at("/indexes").nest(|router| { - router.at("/").get(index::list_indexes); + router.at("/") + .get(index::list_indexes) + .post(index::create_index); router.at("/search").post(search::search_multi_index); @@ -29,7 +31,6 @@ pub fn load_routes(app: &mut tide::App) { router .at("/") .get(index::get_index_schema) - .post(index::create_index) .put(index::update_schema) .delete(index::delete_index); From a90facaa41e0e22590f6353ec399b4c14df8214a Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 19 Nov 2019 16:15:49 +0100 Subject: [PATCH 04/28] Rename index_name by index_uid --- meilidb-core/examples/from_file.rs | 14 +++++++------- meilidb-core/src/database.rs | 18 +++++++++--------- meilidb-http/src/data.rs | 24 ++++++++++++------------ meilidb-http/src/helpers/tide.rs | 6 +++--- meilidb-http/src/routes/index.rs | 16 ++++++++-------- meilidb-http/src/routes/search.rs | 12 ++++++------ meilidb-http/src/routes/stats.rs | 20 ++++++++++---------- 7 files changed, 55 insertions(+), 55 deletions(-) diff --git a/meilidb-core/examples/from_file.rs b/meilidb-core/examples/from_file.rs index f7a135dd8..963254dd6 100644 --- a/meilidb-core/examples/from_file.rs +++ b/meilidb-core/examples/from_file.rs @@ -22,7 +22,7 @@ struct IndexCommand { database_path: PathBuf, #[structopt(long, default_value = "default")] - index_name: String, + index_uid: String, /// The csv file to index. #[structopt(parse(from_os_str))] @@ -46,7 +46,7 @@ struct SearchCommand { database_path: PathBuf, #[structopt(long, default_value = "default")] - index_name: String, + index_uid: String, /// Timeout after which the search will return results. #[structopt(long)] @@ -76,7 +76,7 @@ struct ShowUpdatesCommand { database_path: PathBuf, #[structopt(long, default_value = "default")] - index_name: String, + index_uid: String, } #[derive(Debug, StructOpt)] @@ -106,9 +106,9 @@ fn index_command(command: IndexCommand, database: Database) -> Result<(), Box index, - None => database.create_index(&command.index_name).unwrap(), + None => database.create_index(&command.index_uid).unwrap(), }; database.set_update_callback(Box::new(update_fn)); @@ -318,7 +318,7 @@ fn crop_text( fn search_command(command: SearchCommand, database: Database) -> Result<(), Box> { let env = &database.env; let index = database - .open_index(&command.index_name) + .open_index(&command.index_uid) .expect("Could not find index"); let reader = env.read_txn().unwrap(); @@ -446,7 +446,7 @@ fn show_updates_command( ) -> Result<(), Box> { let env = &database.env; let index = database - .open_index(&command.index_name) + .open_index(&command.index_uid) .expect("Could not find index"); let reader = env.read_txn().unwrap(); diff --git a/meilidb-core/src/database.rs b/meilidb-core/src/database.rs index fc290a0f6..8cd3cd2a0 100644 --- a/meilidb-core/src/database.rs +++ b/meilidb-core/src/database.rs @@ -45,7 +45,7 @@ pub type UpdateEventsEmitter = Sender; fn update_awaiter( receiver: UpdateEvents, env: heed::Env, - index_name: &str, + index_uid: &str, update_fn: Arc, index: Index, ) { @@ -91,7 +91,7 @@ fn update_awaiter( // call the user callback when the update and the result are written consistently if let Some(ref callback) = *update_fn.load() { - (callback)(index_name, status); + (callback)(index_uid, status); } } } @@ -116,22 +116,22 @@ impl Database { let mut must_open = Vec::new(); let reader = env.read_txn()?; for result in indexes_store.iter(&reader)? { - let (index_name, _) = result?; - must_open.push(index_name.to_owned()); + let (index_uid, _) = result?; + must_open.push(index_uid.to_owned()); } reader.abort(); // open the previously aggregated indexes let mut indexes = HashMap::new(); - for index_name in must_open { + for index_uid in must_open { let (sender, receiver) = crossbeam_channel::bounded(100); - let index = match store::open(&env, &index_name, sender.clone())? { + let index = match store::open(&env, &index_uid, sender.clone())? { Some(index) => index, None => { log::warn!( "the index {} doesn't exist or has not all the databases", - index_name + index_uid ); continue; } @@ -139,7 +139,7 @@ impl Database { let env_clone = env.clone(); let index_clone = index.clone(); - let name_clone = index_name.clone(); + let name_clone = index_uid.clone(); let update_fn_clone = update_fn.clone(); let handle = thread::spawn(move || { @@ -156,7 +156,7 @@ impl Database { // possible pre-boot updates are consumed sender.send(UpdateEvent::NewUpdate).unwrap(); - let result = indexes.insert(index_name, (index, handle)); + let result = indexes.insert(index_uid, (index, handle)); assert!( result.is_none(), "The index should not have been already open" diff --git a/meilidb-http/src/data.rs b/meilidb-http/src/data.rs index c4fba4af9..9a2617191 100644 --- a/meilidb-http/src/data.rs +++ b/meilidb-http/src/data.rs @@ -47,9 +47,9 @@ impl DataInner { pub fn last_update( &self, reader: &heed::RoTxn, - index_name: &str, + index_uid: &str, ) -> MResult>> { - let key = format!("last-update-{}", index_name); + let key = format!("last-update-{}", index_uid); match self .db .common_store() @@ -60,8 +60,8 @@ impl DataInner { } } - pub fn set_last_update(&self, writer: &mut heed::RwTxn, index_name: &str) -> MResult<()> { - let key = format!("last-update-{}", index_name); + pub fn set_last_update(&self, writer: &mut heed::RwTxn, index_uid: &str) -> MResult<()> { + let key = format!("last-update-{}", index_uid); self.db .common_store() .put::(writer, &key, &Utc::now()) @@ -71,9 +71,9 @@ impl DataInner { pub fn fields_frequency( &self, reader: &heed::RoTxn, - index_name: &str, + index_uid: &str, ) -> MResult> { - let key = format!("fields-frequency-{}", index_name); + let key = format!("fields-frequency-{}", index_uid); match self .db .common_store() @@ -84,11 +84,11 @@ impl DataInner { } } - pub fn compute_stats(&self, writer: &mut heed::RwTxn, index_name: &str) -> MResult<()> { - let index = match self.db.open_index(&index_name) { + pub fn compute_stats(&self, writer: &mut heed::RwTxn, index_uid: &str) -> MResult<()> { + let index = match self.db.open_index(&index_uid) { Some(index) => index, None => { - error!("Impossible to retrieve index {}", index_name); + error!("Impossible to retrieve index {}", index_uid); return Ok(()); } }; @@ -115,7 +115,7 @@ impl DataInner { .map(|(a, c)| (schema.attribute_name(a).to_owned(), c)) .collect(); - let key = format!("fields-frequency-{}", index_name); + let key = format!("fields-frequency-{}", index_uid); self.db .common_store() .put::(writer, &key, &frequency)?; @@ -144,8 +144,8 @@ impl Data { }; let callback_context = data.clone(); - db.set_update_callback(Box::new(move |index_name, status| { - index_update_callback(&index_name, &callback_context, status); + db.set_update_callback(Box::new(move |index_uid, status| { + index_update_callback(&index_uid, &callback_context, status); })); data diff --git a/meilidb-http/src/helpers/tide.rs b/meilidb-http/src/helpers/tide.rs index 3ede9d9f9..8890f6340 100644 --- a/meilidb-http/src/helpers/tide.rs +++ b/meilidb-http/src/helpers/tide.rs @@ -93,12 +93,12 @@ impl ContextExt for Context { } fn index(&self) -> Result { - let index_name = self.url_param("index")?; + let index_uid = self.url_param("index")?; let index = self .state() .db - .open_index(&index_name) - .ok_or(ResponseError::index_not_found(index_name))?; + .open_index(&index_uid) + .ok_or(ResponseError::index_not_found(index_uid))?; Ok(index) } diff --git a/meilidb-http/src/routes/index.rs b/meilidb-http/src/routes/index.rs index e915a3573..53b9f36e5 100644 --- a/meilidb-http/src/routes/index.rs +++ b/meilidb-http/src/routes/index.rs @@ -131,7 +131,7 @@ pub async fn create_index(mut ctx: Context) -> SResult { pub async fn update_schema(mut ctx: Context) -> SResult { ctx.is_allowed(IndexesWrite)?; - let index_name = ctx.url_param("index")?; + let index_uid = ctx.url_param("index")?; let schema = ctx .body_json::() @@ -143,8 +143,8 @@ pub async fn update_schema(mut ctx: Context) -> SResult { let mut writer = env.write_txn().map_err(ResponseError::internal)?; let index = db - .open_index(&index_name) - .ok_or(ResponseError::index_not_found(index_name))?; + .open_index(&index_uid) + .ok_or(ResponseError::index_not_found(index_uid))?; let schema: meilidb_schema::Schema = schema.into(); let update_id = index @@ -206,12 +206,12 @@ pub async fn get_all_updates_status(ctx: Context) -> SResult { pub async fn delete_index(ctx: Context) -> SResult { ctx.is_allowed(IndexesWrite)?; - let index_name = ctx.url_param("index")?; + let index_uid = ctx.url_param("index")?; let found = ctx .state() .db - .delete_index(&index_name) + .delete_index(&index_uid) .map_err(ResponseError::internal)?; if found { @@ -221,12 +221,12 @@ pub async fn delete_index(ctx: Context) -> SResult { } } -pub fn index_update_callback(index_name: &str, data: &Data, _status: ProcessedUpdateResult) { +pub fn index_update_callback(index_uid: &str, data: &Data, _status: ProcessedUpdateResult) { let env = &data.db.env; let mut writer = env.write_txn().unwrap(); - data.compute_stats(&mut writer, &index_name).unwrap(); - data.set_last_update(&mut writer, &index_name).unwrap(); + data.compute_stats(&mut writer, &index_uid).unwrap(); + data.set_last_update(&mut writer, &index_uid).unwrap(); writer.commit().unwrap(); } diff --git a/meilidb-http/src/routes/search.rs b/meilidb-http/src/routes/search.rs index 116f76e03..f144e3f32 100644 --- a/meilidb-http/src/routes/search.rs +++ b/meilidb-http/src/routes/search.rs @@ -181,10 +181,10 @@ pub async fn search_multi_index(mut ctx: Context) -> SResult { let par_body = body.clone(); let responses_per_index: Vec> = index_list .into_par_iter() - .map(move |index_name| { + .map(move |index_uid| { let index: Index = db - .open_index(&index_name) - .ok_or(ResponseError::index_not_found(&index_name))?; + .open_index(&index_uid) + .ok_or(ResponseError::index_not_found(&index_uid))?; let mut search_builder = index.new_search(par_body.query.clone()); @@ -221,7 +221,7 @@ pub async fn search_multi_index(mut ctx: Context) -> SResult { let response = search_builder .search(&reader) .map_err(ResponseError::internal)?; - Ok((index_name, response)) + Ok((index_uid, response)) }) .collect(); @@ -230,11 +230,11 @@ pub async fn search_multi_index(mut ctx: Context) -> SResult { let mut max_query_time = 0; for response in responses_per_index { - if let Ok((index_name, response)) = response { + if let Ok((index_uid, response)) = response { if response.processing_time_ms > max_query_time { max_query_time = response.processing_time_ms; } - hits_map.insert(index_name, response.hits); + hits_map.insert(index_uid, response.hits); } } diff --git a/meilidb-http/src/routes/stats.rs b/meilidb-http/src/routes/stats.rs index dd4d24653..afc8ceaaf 100644 --- a/meilidb-http/src/routes/stats.rs +++ b/meilidb-http/src/routes/stats.rs @@ -23,7 +23,7 @@ struct IndexStatsResponse { pub async fn index_stat(ctx: Context) -> SResult { ctx.is_allowed(Admin)?; - let index_name = ctx.url_param("index")?; + let index_uid = ctx.url_param("index")?; let index = ctx.index()?; let env = &ctx.state().db.env; @@ -36,19 +36,19 @@ pub async fn index_stat(ctx: Context) -> SResult { let fields_frequency = ctx .state() - .fields_frequency(&reader, &index_name) + .fields_frequency(&reader, &index_uid) .map_err(ResponseError::internal)? .unwrap_or_default(); let is_indexing = ctx .state() - .is_indexing(&reader, &index_name) + .is_indexing(&reader, &index_uid) .map_err(ResponseError::internal)? .ok_or(ResponseError::not_found("Index not found"))?; let last_update = ctx .state() - .last_update(&reader, &index_name) + .last_update(&reader, &index_uid) .map_err(ResponseError::internal)?; let response = IndexStatsResponse { @@ -73,11 +73,11 @@ pub async fn get_stats(ctx: Context) -> SResult { let mut index_list = HashMap::new(); if let Ok(indexes_set) = ctx.state().db.indexes_names() { - for index_name in indexes_set { + for index_uid in indexes_set { let db = &ctx.state().db; let env = &db.env; - let index = db.open_index(&index_name).unwrap(); + let index = db.open_index(&index_uid).unwrap(); let reader = env.read_txn().map_err(ResponseError::internal)?; let number_of_documents = index @@ -87,19 +87,19 @@ pub async fn get_stats(ctx: Context) -> SResult { let fields_frequency = ctx .state() - .fields_frequency(&reader, &index_name) + .fields_frequency(&reader, &index_uid) .map_err(ResponseError::internal)? .unwrap_or_default(); let is_indexing = ctx .state() - .is_indexing(&reader, &index_name) + .is_indexing(&reader, &index_uid) .map_err(ResponseError::internal)? .ok_or(ResponseError::not_found("Index not found"))?; let last_update = ctx .state() - .last_update(&reader, &index_name) + .last_update(&reader, &index_uid) .map_err(ResponseError::internal)?; let response = IndexStatsResponse { @@ -108,7 +108,7 @@ pub async fn get_stats(ctx: Context) -> SResult { last_update, fields_frequency, }; - index_list.insert(index_name, response); + index_list.insert(index_uid, response); } } From 39e2b73718e5596abead3f1794871d5bdf0585de Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 19 Nov 2019 16:18:01 +0100 Subject: [PATCH 05/28] Add updatedAt on main index store --- meilidb-core/src/store/main.rs | 9 +++++++++ meilidb-http/src/routes/index.rs | 2 ++ 2 files changed, 11 insertions(+) diff --git a/meilidb-core/src/store/main.rs b/meilidb-core/src/store/main.rs index d35189ea9..55d5f10b0 100644 --- a/meilidb-core/src/store/main.rs +++ b/meilidb-core/src/store/main.rs @@ -13,6 +13,7 @@ const RANKED_MAP_KEY: &str = "ranked-map"; const SCHEMA_KEY: &str = "schema"; const STOP_WORDS_KEY: &str = "stop-words"; const SYNONYMS_KEY: &str = "synonyms"; +const UPDATED_AT: &str = "updated-at"; const WORDS_KEY: &str = "words"; type SerdeDatetime = SerdeBincode>; @@ -43,6 +44,14 @@ impl Main { self.main.put::(writer, CREATED_AT, &Utc::now()) } + pub fn updated_at(self, reader: &heed::RoTxn) -> ZResult>> { + self.main.get::(reader, UPDATED_AT) + } + + pub fn put_updated_at(self, writer: &mut heed::RwTxn) -> ZResult<()> { + self.main.put::(writer, UPDATED_AT, &Utc::now()) + } + pub fn put_words_fst(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> ZResult<()> { let bytes = fst.as_fst().as_bytes(); self.main.put::(writer, WORDS_KEY, bytes) diff --git a/meilidb-http/src/routes/index.rs b/meilidb-http/src/routes/index.rs index 53b9f36e5..af0e15a70 100644 --- a/meilidb-http/src/routes/index.rs +++ b/meilidb-http/src/routes/index.rs @@ -102,6 +102,8 @@ pub async fn create_index(mut ctx: Context) -> SResult { .put_created_at(&mut writer) .map_err(ResponseError::internal)?; created_index.main + .put_updated_at(&mut writer) + .map_err(ResponseError::internal)?; let schema: Option = body.schema.clone().map(|s| s.into()); let mut response_update_id = None; From e97e13ce9f31cbd9ae9cc0a4faa8e78c8b8a4a01 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 19 Nov 2019 17:38:02 +0100 Subject: [PATCH 06/28] Rename index_name to index_uids --- meilidb-core/src/database.rs | 2 +- meilidb-http/src/routes/index.rs | 45 ++++++++++++++++++++++++++++++- meilidb-http/src/routes/search.rs | 2 +- meilidb-http/src/routes/stats.rs | 2 +- 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/meilidb-core/src/database.rs b/meilidb-core/src/database.rs index 8cd3cd2a0..0bdde55cb 100644 --- a/meilidb-core/src/database.rs +++ b/meilidb-core/src/database.rs @@ -251,7 +251,7 @@ impl Database { self.env.copy_to_path(path, CompactionOption::Enabled) } - pub fn indexes_names(&self) -> MResult> { + pub fn indexes_uids(&self) -> MResult> { let indexes = self.indexes.read().unwrap(); Ok(indexes.keys().cloned().collect()) } diff --git a/meilidb-http/src/routes/index.rs b/meilidb-http/src/routes/index.rs index af0e15a70..b31a9b301 100644 --- a/meilidb-http/src/routes/index.rs +++ b/meilidb-http/src/routes/index.rs @@ -29,11 +29,54 @@ pub async fn list_indexes(ctx: Context) -> SResult { let list = ctx .state() .db - .indexes_names() + .indexes_uids() .map_err(ResponseError::internal)?; Ok(tide::response::json(list)) } +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct GetIndexResponse { + name: String, + uid: String, + schema: Option, + created_at: DateTime, + updated_at: DateTime, +} + +pub async fn get_index(ctx: Context) -> SResult { + ctx.is_allowed(IndexesRead)?; + + let index = ctx.index()?; + + let env = &ctx.state().db.env; + let mut reader = env.read_txn().map_err(ResponseError::internal)?; + + let uid = ctx.url_param("index")?.to_string(); + let name = index.main.name(&mut reader) + .map_err(ResponseError::internal)? + .ok_or(ResponseError::internal("Name not found"))?; + let schema = index.main.schema(&mut reader) + .map_err(ResponseError::internal)? + .map(|schema| SchemaBody::from(schema)); + let created_at = index.main.created_at(&mut reader) + .map_err(ResponseError::internal)? + .ok_or(ResponseError::internal("Created date not found"))?; + let updated_at = index.main.updated_at(&mut reader) + .map_err(ResponseError::internal)? + .ok_or(ResponseError::internal("Updated date not found"))?; + + let response_body = GetIndexResponse { + name, + uid, + schema, + created_at, + updated_at, + }; + + Ok(tide::response::json(response_body)) +} + pub async fn get_index_schema(ctx: Context) -> SResult { ctx.is_allowed(IndexesRead)?; diff --git a/meilidb-http/src/routes/search.rs b/meilidb-http/src/routes/search.rs index f144e3f32..e4500bc92 100644 --- a/meilidb-http/src/routes/search.rs +++ b/meilidb-http/src/routes/search.rs @@ -158,7 +158,7 @@ pub async fn search_multi_index(mut ctx: Context) -> SResult { index_list = ctx .state() .db - .indexes_names() + .indexes_uids() .map_err(ResponseError::internal)? .into_iter() .collect(); diff --git a/meilidb-http/src/routes/stats.rs b/meilidb-http/src/routes/stats.rs index afc8ceaaf..296f6f578 100644 --- a/meilidb-http/src/routes/stats.rs +++ b/meilidb-http/src/routes/stats.rs @@ -72,7 +72,7 @@ pub async fn get_stats(ctx: Context) -> SResult { let mut index_list = HashMap::new(); - if let Ok(indexes_set) = ctx.state().db.indexes_names() { + if let Ok(indexes_set) = ctx.state().db.indexes_uids() { for index_uid in indexes_set { let db = &ctx.state().db; let env = &db.env; From 2b3c91aabd0fb2a016a71fcb1bc4074f14266e10 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 19 Nov 2019 17:39:30 +0100 Subject: [PATCH 07/28] Update get_index_schema to allow raw response --- meilidb-http/src/routes/index.rs | 27 ++++++++++++++++++--------- meilidb-schema/src/lib.rs | 2 +- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/meilidb-http/src/routes/index.rs b/meilidb-http/src/routes/index.rs index b31a9b301..540900768 100644 --- a/meilidb-http/src/routes/index.rs +++ b/meilidb-http/src/routes/index.rs @@ -1,9 +1,11 @@ -use meilidb_schema::Schema; +use chrono::{DateTime, Utc}; use http::StatusCode; use meilidb_core::ProcessedUpdateResult; +use meilidb_schema::Schema; use rand::seq::SliceRandom; use serde::{Deserialize, Serialize}; use serde_json::json; +use tide::querystring::ContextExt as QSContextExt; use tide::response::IntoResponse; use tide::{Context, Response}; use chrono::{DateTime, Utc}; @@ -39,7 +41,6 @@ pub async fn list_indexes(ctx: Context) -> SResult { struct GetIndexResponse { name: String, uid: String, - schema: Option, created_at: DateTime, updated_at: DateTime, } @@ -56,9 +57,6 @@ pub async fn get_index(ctx: Context) -> SResult { let name = index.main.name(&mut reader) .map_err(ResponseError::internal)? .ok_or(ResponseError::internal("Name not found"))?; - let schema = index.main.schema(&mut reader) - .map_err(ResponseError::internal)? - .map(|schema| SchemaBody::from(schema)); let created_at = index.main.created_at(&mut reader) .map_err(ResponseError::internal)? .ok_or(ResponseError::internal("Created date not found"))?; @@ -69,7 +67,6 @@ pub async fn get_index(ctx: Context) -> SResult { let response_body = GetIndexResponse { name, uid, - schema, created_at, updated_at, }; @@ -77,23 +74,35 @@ pub async fn get_index(ctx: Context) -> SResult { Ok(tide::response::json(response_body)) } +#[derive(Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct GetSchemaParams { + raw: bool, +} + pub async fn get_index_schema(ctx: Context) -> SResult { ctx.is_allowed(IndexesRead)?; let index = ctx.index()?; + // Tide doesn't support "no query param" + let params: GetSchemaParams = ctx.url_query().unwrap_or_default(); + let env = &ctx.state().db.env; let reader = env.read_txn().map_err(ResponseError::internal)?; let schema = index .main .schema(&reader) - .map_err(ResponseError::create_index)?; + .map_err(ResponseError::open_index)?; match schema { Some(schema) => { - let schema = SchemaBody::from(schema); - Ok(tide::response::json(schema)) + if params.raw { + Ok(tide::response::json(schema.to_builder())) + } else { + Ok(tide::response::json(SchemaBody::from(schema))) + } } None => Ok( tide::response::json(json!({ "message": "missing index schema" })) diff --git a/meilidb-schema/src/lib.rs b/meilidb-schema/src/lib.rs index 929316d41..a7125e434 100644 --- a/meilidb-schema/src/lib.rs +++ b/meilidb-schema/src/lib.rs @@ -145,7 +145,7 @@ struct InnerSchema { } impl Schema { - fn to_builder(&self) -> SchemaBuilder { + pub fn to_builder(&self) -> SchemaBuilder { let identifier = self.inner.identifier.clone(); let attributes = self.attributes_ordered(); SchemaBuilder { From e6465f4ea1a1ea32d3dce8003698ad2e15eabf3d Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 19 Nov 2019 17:40:11 +0100 Subject: [PATCH 08/28] Create a new specific route for schema --- meilidb-http/src/routes/mod.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/meilidb-http/src/routes/mod.rs b/meilidb-http/src/routes/mod.rs index c44134cc8..1ba53f0fc 100644 --- a/meilidb-http/src/routes/mod.rs +++ b/meilidb-http/src/routes/mod.rs @@ -30,10 +30,14 @@ pub fn load_routes(app: &mut tide::App) { router .at("/") - .get(index::get_index_schema) - .put(index::update_schema) + .get(index::get_index) .delete(index::delete_index); + router + .at("/schema") + .get(index::get_index_schema) + .put(index::update_schema); + router.at("/documents").nest(|router| { router .at("/") From c94f4dff71522e8eb3158423f655a5dfc2124e02 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 19 Nov 2019 17:41:10 +0100 Subject: [PATCH 09/28] Do not return update_id on IndexCreateRespnse if it's none --- meilidb-http/src/routes/index.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/meilidb-http/src/routes/index.rs b/meilidb-http/src/routes/index.rs index 540900768..af29318e4 100644 --- a/meilidb-http/src/routes/index.rs +++ b/meilidb-http/src/routes/index.rs @@ -125,6 +125,7 @@ struct IndexCreateResponse { name: String, uid: String, schema: Option, + #[serde(skip_serializing_if = "Option::is_none")] update_id: Option, created_at: DateTime, updated_at: DateTime, From b95acbece00a325291d73ac6b82f20ccc4b592f8 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 19 Nov 2019 17:42:47 +0100 Subject: [PATCH 10/28] Function generate_uid return now lowercased uid --- meilidb-http/src/routes/index.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilidb-http/src/routes/index.rs b/meilidb-http/src/routes/index.rs index af29318e4..e44bdf7cc 100644 --- a/meilidb-http/src/routes/index.rs +++ b/meilidb-http/src/routes/index.rs @@ -19,7 +19,7 @@ use crate::Data; fn generate_uid() -> String { let mut rng = rand::thread_rng(); - let sample = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + let sample = b"abcdefghijklmnopqrstuvwxyz0123456789"; sample .choose_multiple(&mut rng, 8) .map(|c| *c as char) From 394976d3302f22937c16dd13450b3139100e920c Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 Nov 2019 09:57:27 +0100 Subject: [PATCH 11/28] Update list_index route to return all index information, not only list of uid --- meilidb-http/src/routes/index.rs | 41 ++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/meilidb-http/src/routes/index.rs b/meilidb-http/src/routes/index.rs index e44bdf7cc..552f8d867 100644 --- a/meilidb-http/src/routes/index.rs +++ b/meilidb-http/src/routes/index.rs @@ -8,7 +8,6 @@ use serde_json::json; use tide::querystring::ContextExt as QSContextExt; use tide::response::IntoResponse; use tide::{Context, Response}; -use chrono::{DateTime, Utc}; use crate::error::{ResponseError, SResult}; use crate::helpers::tide::ContextExt; @@ -28,17 +27,49 @@ fn generate_uid() -> String { pub async fn list_indexes(ctx: Context) -> SResult { ctx.is_allowed(IndexesRead)?; - let list = ctx + + let indexes_uids = ctx .state() .db .indexes_uids() .map_err(ResponseError::internal)?; - Ok(tide::response::json(list)) + + let env = &ctx.state().db.env; + let mut reader = env.read_txn().map_err(ResponseError::internal)?; + + let mut response_body = Vec::new(); + + for index_uid in indexes_uids { + let index = ctx + .state() + .db + .open_index(&index_uid) + .ok_or(ResponseError::internal(&index_uid))?; + let name = index.main.name(&mut reader) + .map_err(ResponseError::internal)? + .ok_or(ResponseError::internal("Name not found"))?; + let created_at = index.main.created_at(&mut reader) + .map_err(ResponseError::internal)? + .ok_or(ResponseError::internal("Created date not found"))?; + let updated_at = index.main.updated_at(&mut reader) + .map_err(ResponseError::internal)? + .ok_or(ResponseError::internal("Updated date not found"))?; + + let index_reponse = IndexResponse { + name, + uid: index_uid, + created_at, + updated_at, + }; + response_body.push(index_reponse); + } + + Ok(tide::response::json(response_body)) } #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -struct GetIndexResponse { +struct IndexResponse { name: String, uid: String, created_at: DateTime, @@ -64,7 +95,7 @@ pub async fn get_index(ctx: Context) -> SResult { .map_err(ResponseError::internal)? .ok_or(ResponseError::internal("Updated date not found"))?; - let response_body = GetIndexResponse { + let response_body = IndexResponse { name, uid, created_at, From 3286a5213c39e6837e3f90acd29f254c52163dca Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 Nov 2019 10:47:34 +0100 Subject: [PATCH 12/28] Move fields frequency from common store to index main store --- meilidb-core/src/store/main.rs | 19 +++++++++++++++ meilidb-http/src/data.rs | 43 ++++++---------------------------- 2 files changed, 26 insertions(+), 36 deletions(-) diff --git a/meilidb-core/src/store/main.rs b/meilidb-core/src/store/main.rs index 55d5f10b0..c89cd23a1 100644 --- a/meilidb-core/src/store/main.rs +++ b/meilidb-core/src/store/main.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use chrono::{DateTime, Utc}; use crate::RankedMap; use heed::Result as ZResult; @@ -7,6 +8,7 @@ use std::sync::Arc; const CREATED_AT: &str = "created-at"; const CUSTOMS_KEY: &str = "customs-key"; +const FIELDS_FREQUENCY: &str = "fields-frequency"; const NAME: &str = "name"; const NUMBER_OF_DOCUMENTS_KEY: &str = "number-of-documents"; const RANKED_MAP_KEY: &str = "ranked-map"; @@ -16,6 +18,8 @@ const SYNONYMS_KEY: &str = "synonyms"; const UPDATED_AT: &str = "updated-at"; const WORDS_KEY: &str = "words"; +pub type FreqsMap = HashMap; +type SerdeFreqsMap = SerdeBincode; type SerdeDatetime = SerdeBincode>; #[derive(Copy, Clone)] @@ -144,6 +148,21 @@ impl Main { } } + pub fn put_fields_frequency(self, writer: &mut heed::RwTxn, fields_frequency: &FreqsMap) -> ZResult<()> { + self.main + .put::(writer, FIELDS_FREQUENCY, fields_frequency) + } + + pub fn fields_frequency(&self, reader: &heed::RoTxn) -> ZResult> { + match self + .main + .get::(&reader, FIELDS_FREQUENCY)? + { + Some(freqs) => Ok(Some(freqs)), + None => Ok(None), + } + } + pub fn put_customs(self, writer: &mut heed::RwTxn, customs: &[u8]) -> ZResult<()> { self.main .put::(writer, CUSTOMS_KEY, customs) diff --git a/meilidb-http/src/data.rs b/meilidb-http/src/data.rs index 9a2617191..06016b9e7 100644 --- a/meilidb-http/src/data.rs +++ b/meilidb-http/src/data.rs @@ -5,14 +5,12 @@ use std::sync::Arc; use chrono::{DateTime, Utc}; use heed::types::{SerdeBincode, Str}; use log::*; -use meilidb_core::{Database, MResult}; +use meilidb_core::{Database, MResult, Error as MError}; use sysinfo::Pid; use crate::option::Opt; use crate::routes::index::index_update_callback; -pub type FreqsMap = HashMap; -type SerdeFreqsMap = SerdeBincode; type SerdeDatetime = SerdeBincode>; #[derive(Clone)] @@ -44,47 +42,25 @@ impl DataInner { } } - pub fn last_update( - &self, - reader: &heed::RoTxn, - index_uid: &str, - ) -> MResult>> { - let key = format!("last-update-{}", index_uid); + pub fn last_update(&self, reader: &heed::RoTxn) -> MResult>> { match self .db .common_store() - .get::(&reader, &key)? + .get::(&reader, "last-update")? { Some(datetime) => Ok(Some(datetime)), None => Ok(None), } } - pub fn set_last_update(&self, writer: &mut heed::RwTxn, index_uid: &str) -> MResult<()> { - let key = format!("last-update-{}", index_uid); + pub fn set_last_update(&self, writer: &mut heed::RwTxn) -> MResult<()> { self.db .common_store() - .put::(writer, &key, &Utc::now()) + .put::(writer, "last-update", &Utc::now()) .map_err(Into::into) } - pub fn fields_frequency( - &self, - reader: &heed::RoTxn, - index_uid: &str, - ) -> MResult> { - let key = format!("fields-frequency-{}", index_uid); - match self - .db - .common_store() - .get::(&reader, &key)? - { - Some(freqs) => Ok(Some(freqs)), - None => Ok(None), - } - } - - pub fn compute_stats(&self, writer: &mut heed::RwTxn, index_uid: &str) -> MResult<()> { + pub fn compute_stats(&self, mut writer: &mut heed::RwTxn, index_uid: &str) -> MResult<()> { let index = match self.db.open_index(&index_uid) { Some(index) => index, None => { @@ -115,12 +91,7 @@ impl DataInner { .map(|(a, c)| (schema.attribute_name(a).to_owned(), c)) .collect(); - let key = format!("fields-frequency-{}", index_uid); - self.db - .common_store() - .put::(writer, &key, &frequency)?; - - Ok(()) + index.main.put_fields_frequency(&mut writer, &frequency).map_err(MError::Zlmdb) } } From 1b8df0ed8bf3843703fe7aed8b62b61b1e1e0c9a Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 Nov 2019 10:47:57 +0100 Subject: [PATCH 13/28] Remove last_update from stats --- meilidb-http/src/routes/index.rs | 2 +- meilidb-http/src/routes/stats.rs | 27 ++++++--------------------- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/meilidb-http/src/routes/index.rs b/meilidb-http/src/routes/index.rs index 552f8d867..ff9a2998a 100644 --- a/meilidb-http/src/routes/index.rs +++ b/meilidb-http/src/routes/index.rs @@ -312,7 +312,7 @@ pub fn index_update_callback(index_uid: &str, data: &Data, _status: ProcessedUpd let mut writer = env.write_txn().unwrap(); data.compute_stats(&mut writer, &index_uid).unwrap(); - data.set_last_update(&mut writer, &index_uid).unwrap(); + data.set_last_update(&mut writer).unwrap(); writer.commit().unwrap(); } diff --git a/meilidb-http/src/routes/stats.rs b/meilidb-http/src/routes/stats.rs index 296f6f578..5d08f2073 100644 --- a/meilidb-http/src/routes/stats.rs +++ b/meilidb-http/src/routes/stats.rs @@ -1,6 +1,4 @@ use std::collections::HashMap; - -use chrono::{DateTime, Utc}; use pretty_bytes::converter::convert; use serde::Serialize; use sysinfo::{NetworkExt, Pid, ProcessExt, ProcessorExt, System, SystemExt}; @@ -17,7 +15,6 @@ use crate::Data; struct IndexStatsResponse { number_of_documents: u64, is_indexing: bool, - last_update: Option>, fields_frequency: HashMap, } @@ -34,9 +31,9 @@ pub async fn index_stat(ctx: Context) -> SResult { .number_of_documents(&reader) .map_err(ResponseError::internal)?; - let fields_frequency = ctx - .state() - .fields_frequency(&reader, &index_uid) + let fields_frequency = index + .main + .fields_frequency(&reader) .map_err(ResponseError::internal)? .unwrap_or_default(); @@ -46,15 +43,9 @@ pub async fn index_stat(ctx: Context) -> SResult { .map_err(ResponseError::internal)? .ok_or(ResponseError::not_found("Index not found"))?; - let last_update = ctx - .state() - .last_update(&reader, &index_uid) - .map_err(ResponseError::internal)?; - let response = IndexStatsResponse { number_of_documents, is_indexing, - last_update, fields_frequency, }; Ok(tide::response::json(response)) @@ -85,9 +76,9 @@ pub async fn get_stats(ctx: Context) -> SResult { .number_of_documents(&reader) .map_err(ResponseError::internal)?; - let fields_frequency = ctx - .state() - .fields_frequency(&reader, &index_uid) + let fields_frequency = index + .main + .fields_frequency(&reader) .map_err(ResponseError::internal)? .unwrap_or_default(); @@ -97,15 +88,9 @@ pub async fn get_stats(ctx: Context) -> SResult { .map_err(ResponseError::internal)? .ok_or(ResponseError::not_found("Index not found"))?; - let last_update = ctx - .state() - .last_update(&reader, &index_uid) - .map_err(ResponseError::internal)?; - let response = IndexStatsResponse { number_of_documents, is_indexing, - last_update, fields_frequency, }; index_list.insert(index_uid, response); From e22debb994cbc603e71fdd2443cf1705bfdb17b5 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 Nov 2019 11:19:17 +0100 Subject: [PATCH 14/28] Update index updated_at information at each update callback --- meilidb-http/src/routes/index.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/meilidb-http/src/routes/index.rs b/meilidb-http/src/routes/index.rs index ff9a2998a..4eb228f92 100644 --- a/meilidb-http/src/routes/index.rs +++ b/meilidb-http/src/routes/index.rs @@ -1,5 +1,6 @@ use chrono::{DateTime, Utc}; use http::StatusCode; +use log::*; use meilidb_core::ProcessedUpdateResult; use meilidb_schema::Schema; use rand::seq::SliceRandom; @@ -314,5 +315,11 @@ pub fn index_update_callback(index_uid: &str, data: &Data, _status: ProcessedUpd data.compute_stats(&mut writer, &index_uid).unwrap(); data.set_last_update(&mut writer).unwrap(); + if let Some(index) = data.db.open_index(&index_uid) { + if let Err(e) = index.main.put_updated_at(&mut writer) { + error!("Impossible to update updated_at; {}", e) + } + } + writer.commit().unwrap(); } From a0caf0d6d77304aa45f716e7ca587e00194b4128 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 Nov 2019 11:21:21 +0100 Subject: [PATCH 15/28] Remove unused result response on indexes_uids function --- meilidb-core/src/database.rs | 4 ++-- meilidb-http/src/routes/index.rs | 3 +-- meilidb-http/src/routes/search.rs | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/meilidb-core/src/database.rs b/meilidb-core/src/database.rs index 0bdde55cb..72e5a1d0f 100644 --- a/meilidb-core/src/database.rs +++ b/meilidb-core/src/database.rs @@ -251,9 +251,9 @@ impl Database { self.env.copy_to_path(path, CompactionOption::Enabled) } - pub fn indexes_uids(&self) -> MResult> { + pub fn indexes_uids(&self) -> Vec { let indexes = self.indexes.read().unwrap(); - Ok(indexes.keys().cloned().collect()) + indexes.keys().cloned().collect() } pub fn common_store(&self) -> heed::PolyDatabase { diff --git a/meilidb-http/src/routes/index.rs b/meilidb-http/src/routes/index.rs index 4eb228f92..f6fe999f2 100644 --- a/meilidb-http/src/routes/index.rs +++ b/meilidb-http/src/routes/index.rs @@ -32,8 +32,7 @@ pub async fn list_indexes(ctx: Context) -> SResult { let indexes_uids = ctx .state() .db - .indexes_uids() - .map_err(ResponseError::internal)?; + .indexes_uids(); let env = &ctx.state().db.env; let mut reader = env.read_txn().map_err(ResponseError::internal)?; diff --git a/meilidb-http/src/routes/search.rs b/meilidb-http/src/routes/search.rs index e4500bc92..0551a278e 100644 --- a/meilidb-http/src/routes/search.rs +++ b/meilidb-http/src/routes/search.rs @@ -159,7 +159,6 @@ pub async fn search_multi_index(mut ctx: Context) -> SResult { .state() .db .indexes_uids() - .map_err(ResponseError::internal)? .into_iter() .collect(); } From d01a3944c155a1d9e832dc76740150aa6116b9ae Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 Nov 2019 11:21:55 +0100 Subject: [PATCH 16/28] Add last_update information on global /stats route --- meilidb-http/src/routes/stats.rs | 66 ++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/meilidb-http/src/routes/stats.rs b/meilidb-http/src/routes/stats.rs index 5d08f2073..488b10189 100644 --- a/meilidb-http/src/routes/stats.rs +++ b/meilidb-http/src/routes/stats.rs @@ -1,6 +1,7 @@ -use std::collections::HashMap; +use chrono::{DateTime, Utc}; use pretty_bytes::converter::convert; use serde::Serialize; +use std::collections::HashMap; use sysinfo::{NetworkExt, Pid, ProcessExt, ProcessorExt, System, SystemExt}; use tide::{Context, Response}; use walkdir::WalkDir; @@ -55,6 +56,7 @@ pub async fn index_stat(ctx: Context) -> SResult { #[serde(rename_all = "camelCase")] struct StatsResult { database_size: u64, + last_update: Option>, indexes: HashMap, } @@ -63,38 +65,38 @@ pub async fn get_stats(ctx: Context) -> SResult { let mut index_list = HashMap::new(); - if let Ok(indexes_set) = ctx.state().db.indexes_uids() { - for index_uid in indexes_set { - let db = &ctx.state().db; - let env = &db.env; + + let db = &ctx.state().db; + let env = &db.env; + let reader = env.read_txn().map_err(ResponseError::internal)?; - let index = db.open_index(&index_uid).unwrap(); - let reader = env.read_txn().map_err(ResponseError::internal)?; + let indexes_set = ctx.state().db.indexes_uids(); + for index_uid in indexes_set { + let index = db.open_index(&index_uid).unwrap(); + + let number_of_documents = index + .main + .number_of_documents(&reader) + .map_err(ResponseError::internal)?; - let number_of_documents = index - .main - .number_of_documents(&reader) - .map_err(ResponseError::internal)?; + let fields_frequency = index + .main + .fields_frequency(&reader) + .map_err(ResponseError::internal)? + .unwrap_or_default(); - let fields_frequency = index - .main - .fields_frequency(&reader) - .map_err(ResponseError::internal)? - .unwrap_or_default(); + let is_indexing = ctx + .state() + .is_indexing(&reader, &index_uid) + .map_err(ResponseError::internal)? + .ok_or(ResponseError::not_found("Index not found"))?; - let is_indexing = ctx - .state() - .is_indexing(&reader, &index_uid) - .map_err(ResponseError::internal)? - .ok_or(ResponseError::not_found("Index not found"))?; - - let response = IndexStatsResponse { - number_of_documents, - is_indexing, - fields_frequency, - }; - index_list.insert(index_uid, response); - } + let response = IndexStatsResponse { + number_of_documents, + is_indexing, + fields_frequency, + }; + index_list.insert(index_uid, response); } let database_size = WalkDir::new(ctx.state().db_path.clone()) @@ -104,8 +106,14 @@ pub async fn get_stats(ctx: Context) -> SResult { .filter(|metadata| metadata.is_file()) .fold(0, |acc, m| acc + m.len()); + let last_update = ctx + .state() + .last_update(&reader) + .map_err(ResponseError::internal)?; + let response = StatsResult { database_size, + last_update, indexes: index_list, }; From 45ded0498bd09e20e6143a5e7f957bbd8c76dbca Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 Nov 2019 11:24:08 +0100 Subject: [PATCH 17/28] Format code with cargo fmt --- meilidb-core/src/store/main.rs | 39 +++++++++++++--------- meilidb-http/src/data.rs | 7 ++-- meilidb-http/src/routes/index.rs | 55 ++++++++++++++++++++----------- meilidb-http/src/routes/mod.rs | 3 +- meilidb-http/src/routes/search.rs | 7 +--- meilidb-http/src/routes/stats.rs | 5 ++- 6 files changed, 69 insertions(+), 47 deletions(-) diff --git a/meilidb-core/src/store/main.rs b/meilidb-core/src/store/main.rs index c89cd23a1..bc5fe1d10 100644 --- a/meilidb-core/src/store/main.rs +++ b/meilidb-core/src/store/main.rs @@ -1,9 +1,9 @@ -use std::collections::HashMap; -use chrono::{DateTime, Utc}; use crate::RankedMap; -use heed::Result as ZResult; +use chrono::{DateTime, Utc}; use heed::types::{ByteSlice, OwnedType, SerdeBincode, Str}; +use heed::Result as ZResult; use meilidb_schema::Schema; +use std::collections::HashMap; use std::sync::Arc; const CREATED_AT: &str = "created-at"; @@ -32,30 +32,35 @@ impl Main { self.main.clear(writer) } - pub fn name(self, reader: &heed::RoTxn) -> ZResult> { - Ok(self.main.get::(reader, NAME)?.map(|name| name.to_owned())) - } - pub fn put_name(self, writer: &mut heed::RwTxn, name: &str) -> ZResult<()> { self.main.put::(writer, NAME, name) } + pub fn name(self, reader: &heed::RoTxn) -> ZResult> { + Ok(self + .main + .get::(reader, NAME)? + .map(|name| name.to_owned())) + } + + pub fn put_created_at(self, writer: &mut heed::RwTxn) -> ZResult<()> { + self.main + .put::(writer, CREATED_AT, &Utc::now()) + } + pub fn created_at(self, reader: &heed::RoTxn) -> ZResult>> { self.main.get::(reader, CREATED_AT) } - - pub fn put_created_at(self, writer: &mut heed::RwTxn) -> ZResult<()> { - self.main.put::(writer, CREATED_AT, &Utc::now()) + + pub fn put_updated_at(self, writer: &mut heed::RwTxn) -> ZResult<()> { + self.main + .put::(writer, UPDATED_AT, &Utc::now()) } pub fn updated_at(self, reader: &heed::RoTxn) -> ZResult>> { self.main.get::(reader, UPDATED_AT) } - pub fn put_updated_at(self, writer: &mut heed::RwTxn) -> ZResult<()> { - self.main.put::(writer, UPDATED_AT, &Utc::now()) - } - pub fn put_words_fst(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> ZResult<()> { let bytes = fst.as_fst().as_bytes(); self.main.put::(writer, WORDS_KEY, bytes) @@ -148,7 +153,11 @@ impl Main { } } - pub fn put_fields_frequency(self, writer: &mut heed::RwTxn, fields_frequency: &FreqsMap) -> ZResult<()> { + pub fn put_fields_frequency( + self, + writer: &mut heed::RwTxn, + fields_frequency: &FreqsMap, + ) -> ZResult<()> { self.main .put::(writer, FIELDS_FREQUENCY, fields_frequency) } diff --git a/meilidb-http/src/data.rs b/meilidb-http/src/data.rs index 06016b9e7..f8af58187 100644 --- a/meilidb-http/src/data.rs +++ b/meilidb-http/src/data.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use chrono::{DateTime, Utc}; use heed::types::{SerdeBincode, Str}; use log::*; -use meilidb_core::{Database, MResult, Error as MError}; +use meilidb_core::{Database, Error as MError, MResult}; use sysinfo::Pid; use crate::option::Opt; @@ -91,7 +91,10 @@ impl DataInner { .map(|(a, c)| (schema.attribute_name(a).to_owned(), c)) .collect(); - index.main.put_fields_frequency(&mut writer, &frequency).map_err(MError::Zlmdb) + index + .main + .put_fields_frequency(&mut writer, &frequency) + .map_err(MError::Zlmdb) } } diff --git a/meilidb-http/src/routes/index.rs b/meilidb-http/src/routes/index.rs index f6fe999f2..563fb768c 100644 --- a/meilidb-http/src/routes/index.rs +++ b/meilidb-http/src/routes/index.rs @@ -28,11 +28,8 @@ fn generate_uid() -> String { pub async fn list_indexes(ctx: Context) -> SResult { ctx.is_allowed(IndexesRead)?; - - let indexes_uids = ctx - .state() - .db - .indexes_uids(); + + let indexes_uids = ctx.state().db.indexes_uids(); let env = &ctx.state().db.env; let mut reader = env.read_txn().map_err(ResponseError::internal)?; @@ -45,16 +42,22 @@ pub async fn list_indexes(ctx: Context) -> SResult { .db .open_index(&index_uid) .ok_or(ResponseError::internal(&index_uid))?; - let name = index.main.name(&mut reader) + let name = index + .main + .name(&mut reader) .map_err(ResponseError::internal)? .ok_or(ResponseError::internal("Name not found"))?; - let created_at = index.main.created_at(&mut reader) + let created_at = index + .main + .created_at(&mut reader) .map_err(ResponseError::internal)? .ok_or(ResponseError::internal("Created date not found"))?; - let updated_at = index.main.updated_at(&mut reader) + let updated_at = index + .main + .updated_at(&mut reader) .map_err(ResponseError::internal)? .ok_or(ResponseError::internal("Updated date not found"))?; - + let index_reponse = IndexResponse { name, uid: index_uid, @@ -85,13 +88,19 @@ pub async fn get_index(ctx: Context) -> SResult { let mut reader = env.read_txn().map_err(ResponseError::internal)?; let uid = ctx.url_param("index")?.to_string(); - let name = index.main.name(&mut reader) + let name = index + .main + .name(&mut reader) .map_err(ResponseError::internal)? .ok_or(ResponseError::internal("Name not found"))?; - let created_at = index.main.created_at(&mut reader) + let created_at = index + .main + .created_at(&mut reader) .map_err(ResponseError::internal)? .ok_or(ResponseError::internal("Created date not found"))?; - let updated_at = index.main.updated_at(&mut reader) + let updated_at = index + .main + .updated_at(&mut reader) .map_err(ResponseError::internal)? .ok_or(ResponseError::internal("Updated date not found"))?; @@ -165,7 +174,10 @@ struct IndexCreateResponse { pub async fn create_index(mut ctx: Context) -> SResult { ctx.is_allowed(IndexesWrite)?; - let body = ctx.body_json::().await.map_err(ResponseError::bad_request)?; + let body = ctx + .body_json::() + .await + .map_err(ResponseError::bad_request)?; let generated_uid = generate_uid(); @@ -179,13 +191,16 @@ pub async fn create_index(mut ctx: Context) -> SResult { let env = &db.env; let mut writer = env.write_txn().map_err(ResponseError::internal)?; - created_index.main + created_index + .main .put_name(&mut writer, &body.name) .map_err(ResponseError::internal)?; - created_index.main + created_index + .main .put_created_at(&mut writer) .map_err(ResponseError::internal)?; - created_index.main + created_index + .main .put_updated_at(&mut writer) .map_err(ResponseError::internal)?; @@ -193,8 +208,8 @@ pub async fn create_index(mut ctx: Context) -> SResult { let mut response_update_id = None; if let Some(schema) = schema { let update_id = created_index - .schema_update(&mut writer, schema.clone()) - .map_err(ResponseError::internal)?; + .schema_update(&mut writer, schema.clone()) + .map_err(ResponseError::internal)?; response_update_id = Some(update_id) } @@ -210,8 +225,8 @@ pub async fn create_index(mut ctx: Context) -> SResult { }; Ok(tide::response::json(response_body) - .with_status(StatusCode::CREATED) - .into_response()) + .with_status(StatusCode::CREATED) + .into_response()) } pub async fn update_schema(mut ctx: Context) -> SResult { diff --git a/meilidb-http/src/routes/mod.rs b/meilidb-http/src/routes/mod.rs index 1ba53f0fc..02cede3d4 100644 --- a/meilidb-http/src/routes/mod.rs +++ b/meilidb-http/src/routes/mod.rs @@ -13,7 +13,8 @@ pub mod synonym; pub fn load_routes(app: &mut tide::App) { app.at("").nest(|router| { router.at("/indexes").nest(|router| { - router.at("/") + router + .at("/") .get(index::list_indexes) .post(index::create_index); diff --git a/meilidb-http/src/routes/search.rs b/meilidb-http/src/routes/search.rs index 0551a278e..6e1b558eb 100644 --- a/meilidb-http/src/routes/search.rs +++ b/meilidb-http/src/routes/search.rs @@ -155,12 +155,7 @@ pub async fn search_multi_index(mut ctx: Context) -> SResult { for index in index_list.clone() { if index == "*" { - index_list = ctx - .state() - .db - .indexes_uids() - .into_iter() - .collect(); + index_list = ctx.state().db.indexes_uids().into_iter().collect(); } } diff --git a/meilidb-http/src/routes/stats.rs b/meilidb-http/src/routes/stats.rs index 488b10189..af2e3e49d 100644 --- a/meilidb-http/src/routes/stats.rs +++ b/meilidb-http/src/routes/stats.rs @@ -65,15 +65,14 @@ pub async fn get_stats(ctx: Context) -> SResult { let mut index_list = HashMap::new(); - let db = &ctx.state().db; let env = &db.env; - let reader = env.read_txn().map_err(ResponseError::internal)?; + let reader = env.read_txn().map_err(ResponseError::internal)?; let indexes_set = ctx.state().db.indexes_uids(); for index_uid in indexes_set { let index = db.open_index(&index_uid).unwrap(); - + let number_of_documents = index .main .number_of_documents(&reader) From cc97889b3749e33029a97b404d5bef75ce149d30 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 Nov 2019 13:56:43 +0100 Subject: [PATCH 18/28] Add stop-word is now PATCH method --- meilidb-http/src/routes/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilidb-http/src/routes/mod.rs b/meilidb-http/src/routes/mod.rs index 02cede3d4..dcb2b78ea 100644 --- a/meilidb-http/src/routes/mod.rs +++ b/meilidb-http/src/routes/mod.rs @@ -76,7 +76,7 @@ pub fn load_routes(app: &mut tide::App) { router .at("/") .get(stop_words::list) - .put(stop_words::add) + .patch(stop_words::add) .delete(stop_words::delete); }); From ca1390069931dbd5f85ec470dfef2bce86e25f07 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 Nov 2019 14:03:19 +0100 Subject: [PATCH 19/28] Add async routes should return ACCEPTED status code response --- meilidb-http/src/routes/synonym.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilidb-http/src/routes/synonym.rs b/meilidb-http/src/routes/synonym.rs index de45e5b86..b7d52a5a1 100644 --- a/meilidb-http/src/routes/synonym.rs +++ b/meilidb-http/src/routes/synonym.rs @@ -115,7 +115,7 @@ pub async fn create(mut ctx: Context) -> SResult { let response_body = IndexUpdateResponse { update_id }; Ok(tide::response::json(response_body) - .with_status(StatusCode::CREATED) + .with_status(StatusCode::ACCEPTED) .into_response()) } From 5f0f699f37a34599181ebd9c31a83268cc38352a Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 Nov 2019 14:03:55 +0100 Subject: [PATCH 20/28] Move route to clear all synonyms on DELETE /synonyms --- meilidb-http/src/routes/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/meilidb-http/src/routes/mod.rs b/meilidb-http/src/routes/mod.rs index dcb2b78ea..42778f641 100644 --- a/meilidb-http/src/routes/mod.rs +++ b/meilidb-http/src/routes/mod.rs @@ -60,7 +60,7 @@ pub fn load_routes(app: &mut tide::App) { }); router.at("/synonym").nest(|router| { - router.at("/").get(synonym::list).post(synonym::create); + router.at("/").get(synonym::list).post(synonym::create).delete(synonym::clear); router .at("/:synonym") @@ -69,7 +69,6 @@ pub fn load_routes(app: &mut tide::App) { .delete(synonym::delete); router.at("/batch").post(synonym::batch_write); - router.at("/clear").post(synonym::clear); }); router.at("/stop-words").nest(|router| { From 878dd6912ebb4433152acdcce7f5be5aa1276511 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 Nov 2019 14:06:56 +0100 Subject: [PATCH 21/28] Return a HTTP 401 instead of 404 if token is not found --- meilidb-http/src/helpers/tide.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/meilidb-http/src/helpers/tide.rs b/meilidb-http/src/helpers/tide.rs index 8890f6340..9a9c9adfa 100644 --- a/meilidb-http/src/helpers/tide.rs +++ b/meilidb-http/src/helpers/tide.rs @@ -38,9 +38,9 @@ impl ContextExt for Context { .common_store() .get::>(&reader, &token_key) .map_err(ResponseError::internal)? - .ok_or(ResponseError::not_found(format!( - "token key: {}", - token_key + .ok_or(ResponseError::invalid_token(format!( + "token key does not exist: {}", + user_api_key )))?; if token_config.revoked { From 530738cfe9b3b6936bca23b5359e0ba7bccc33f6 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 Nov 2019 14:12:12 +0100 Subject: [PATCH 22/28] Format code --- meilidb-core/src/store/main.rs | 2 +- meilidb-http/src/routes/mod.rs | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/meilidb-core/src/store/main.rs b/meilidb-core/src/store/main.rs index bc5fe1d10..04ea6aee5 100644 --- a/meilidb-core/src/store/main.rs +++ b/meilidb-core/src/store/main.rs @@ -51,7 +51,7 @@ impl Main { pub fn created_at(self, reader: &heed::RoTxn) -> ZResult>> { self.main.get::(reader, CREATED_AT) } - + pub fn put_updated_at(self, writer: &mut heed::RwTxn) -> ZResult<()> { self.main .put::(writer, UPDATED_AT, &Utc::now()) diff --git a/meilidb-http/src/routes/mod.rs b/meilidb-http/src/routes/mod.rs index 42778f641..e48e9dffa 100644 --- a/meilidb-http/src/routes/mod.rs +++ b/meilidb-http/src/routes/mod.rs @@ -60,7 +60,11 @@ pub fn load_routes(app: &mut tide::App) { }); router.at("/synonym").nest(|router| { - router.at("/").get(synonym::list).post(synonym::create).delete(synonym::clear); + router + .at("/") + .get(synonym::list) + .post(synonym::create) + .delete(synonym::clear); router .at("/:synonym") From 1f1cb1f501026038524eb87026a2767bddc047fc Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 Nov 2019 14:18:21 +0100 Subject: [PATCH 23/28] Rename browse_documents into get_all_documents and always respond HTTP Ok --- meilidb-http/src/routes/document.rs | 14 ++++---------- meilidb-http/src/routes/mod.rs | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/meilidb-http/src/routes/document.rs b/meilidb-http/src/routes/document.rs index 19ea90c13..164c129c2 100644 --- a/meilidb-http/src/routes/document.rs +++ b/meilidb-http/src/routes/document.rs @@ -74,7 +74,7 @@ struct BrowseQuery { attributes_to_retrieve: Option, } -pub async fn browse_documents(ctx: Context) -> SResult { +pub async fn get_all_documents(ctx: Context) -> SResult { ctx.is_allowed(DocumentsRead)?; let index = ctx.index()?; @@ -114,15 +114,9 @@ pub async fn browse_documents(ctx: Context) -> SResult { } } - if response_body.is_empty() { - Ok(tide::response::json(response_body) - .with_status(StatusCode::NO_CONTENT) - .into_response()) - } else { - Ok(tide::response::json(response_body) - .with_status(StatusCode::OK) - .into_response()) - } + Ok(tide::response::json(response_body) + .with_status(StatusCode::OK) + .into_response()) } fn infered_schema(document: &IndexMap) -> Option { diff --git a/meilidb-http/src/routes/mod.rs b/meilidb-http/src/routes/mod.rs index e48e9dffa..828b6c678 100644 --- a/meilidb-http/src/routes/mod.rs +++ b/meilidb-http/src/routes/mod.rs @@ -42,7 +42,7 @@ pub fn load_routes(app: &mut tide::App) { router.at("/documents").nest(|router| { router .at("/") - .get(document::browse_documents) + .get(document::get_all_documents) .post(document::add_or_replace_multiple_documents) .put(document::add_or_update_multiple_documents) .delete(document::clear_all_documents); From cd95b243bb9a3cbd86eb708cf4e2b070681a1427 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 Nov 2019 15:00:06 +0100 Subject: [PATCH 24/28] Add the update index route --- meilidb-http/src/routes/index.rs | 67 ++++++++++++++++++++++++++++++++ meilidb-http/src/routes/mod.rs | 1 + 2 files changed, 68 insertions(+) diff --git a/meilidb-http/src/routes/index.rs b/meilidb-http/src/routes/index.rs index 563fb768c..421af9593 100644 --- a/meilidb-http/src/routes/index.rs +++ b/meilidb-http/src/routes/index.rs @@ -229,6 +229,73 @@ pub async fn create_index(mut ctx: Context) -> SResult { .into_response()) } +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct UpdateIndexRequest { + name: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct UpdateIndexResponse { + name: String, + uid: String, + created_at: DateTime, + updated_at: DateTime, +} + +pub async fn update_index(mut ctx: Context) -> SResult { + ctx.is_allowed(IndexesWrite)?; + + let body = ctx + .body_json::() + .await + .map_err(ResponseError::bad_request)?; + + let index_uid = ctx.url_param("index")?; + let index = ctx.index()?; + + let db = &ctx.state().db; + + let env = &db.env; + let mut writer = env.write_txn().map_err(ResponseError::internal)?; + + index + .main + .put_name(&mut writer, &body.name) + .map_err(ResponseError::internal)?; + + index + .main + .put_updated_at(&mut writer) + .map_err(ResponseError::internal)?; + + writer.commit().map_err(ResponseError::internal)?; + let reader = env.read_txn().map_err(ResponseError::internal)?; + + let created_at = index + .main + .created_at(&reader) + .map_err(ResponseError::internal)? + .ok_or(ResponseError::internal("Created date not found"))?; + let updated_at = index + .main + .updated_at(&reader) + .map_err(ResponseError::internal)? + .ok_or(ResponseError::internal("Updated date not found"))?; + + let response_body = UpdateIndexResponse { + name: body.name, + uid: index_uid, + created_at, + updated_at, + }; + + Ok(tide::response::json(response_body) + .with_status(StatusCode::ACCEPTED) + .into_response()) +} + pub async fn update_schema(mut ctx: Context) -> SResult { ctx.is_allowed(IndexesWrite)?; diff --git a/meilidb-http/src/routes/mod.rs b/meilidb-http/src/routes/mod.rs index 828b6c678..f24532565 100644 --- a/meilidb-http/src/routes/mod.rs +++ b/meilidb-http/src/routes/mod.rs @@ -32,6 +32,7 @@ pub fn load_routes(app: &mut tide::App) { router .at("/") .get(index::get_index) + .put(index::update_index) .delete(index::delete_index); router From c69ae8154f516fce61ebfb5f3bf6a813f5070b2f Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 Nov 2019 15:25:34 +0100 Subject: [PATCH 25/28] Allow to receive schema update formated as SchemaBuilder --- meilidb-http/src/routes/index.rs | 96 +++++++++++++++++--------------- 1 file changed, 52 insertions(+), 44 deletions(-) diff --git a/meilidb-http/src/routes/index.rs b/meilidb-http/src/routes/index.rs index 421af9593..3382c4d3a 100644 --- a/meilidb-http/src/routes/index.rs +++ b/meilidb-http/src/routes/index.rs @@ -2,7 +2,7 @@ use chrono::{DateTime, Utc}; use http::StatusCode; use log::*; use meilidb_core::ProcessedUpdateResult; -use meilidb_schema::Schema; +use meilidb_schema::{Schema, SchemaBuilder}; use rand::seq::SliceRandom; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -114,44 +114,6 @@ pub async fn get_index(ctx: Context) -> SResult { Ok(tide::response::json(response_body)) } -#[derive(Default, Deserialize)] -#[serde(rename_all = "camelCase")] -struct GetSchemaParams { - raw: bool, -} - -pub async fn get_index_schema(ctx: Context) -> SResult { - ctx.is_allowed(IndexesRead)?; - - let index = ctx.index()?; - - // Tide doesn't support "no query param" - let params: GetSchemaParams = ctx.url_query().unwrap_or_default(); - - let env = &ctx.state().db.env; - let reader = env.read_txn().map_err(ResponseError::internal)?; - - let schema = index - .main - .schema(&reader) - .map_err(ResponseError::open_index)?; - - match schema { - Some(schema) => { - if params.raw { - Ok(tide::response::json(schema.to_builder())) - } else { - Ok(tide::response::json(SchemaBody::from(schema))) - } - } - None => Ok( - tide::response::json(json!({ "message": "missing index schema" })) - .with_status(StatusCode::NOT_FOUND) - .into_response(), - ), - } -} - #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct IndexCreateRequest { @@ -296,15 +258,62 @@ pub async fn update_index(mut ctx: Context) -> SResult { .into_response()) } +#[derive(Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct SchemaParams { + raw: bool, +} + +pub async fn get_index_schema(ctx: Context) -> SResult { + ctx.is_allowed(IndexesRead)?; + + let index = ctx.index()?; + + // Tide doesn't support "no query param" + let params: SchemaParams = ctx.url_query().unwrap_or_default(); + + let env = &ctx.state().db.env; + let reader = env.read_txn().map_err(ResponseError::internal)?; + + let schema = index + .main + .schema(&reader) + .map_err(ResponseError::open_index)?; + + match schema { + Some(schema) => { + if params.raw { + Ok(tide::response::json(schema.to_builder())) + } else { + Ok(tide::response::json(SchemaBody::from(schema))) + } + } + None => Ok( + tide::response::json(json!({ "message": "missing index schema" })) + .with_status(StatusCode::NOT_FOUND) + .into_response(), + ), + } +} + pub async fn update_schema(mut ctx: Context) -> SResult { ctx.is_allowed(IndexesWrite)?; let index_uid = ctx.url_param("index")?; - let schema = ctx - .body_json::() - .await - .map_err(ResponseError::bad_request)?; + let params: SchemaParams = ctx.url_query().unwrap_or_default(); + + let schema: Schema = if params.raw { + ctx.body_json::() + .await + .map_err(ResponseError::bad_request)? + .build() + } else { + ctx.body_json::() + .await + .map_err(ResponseError::bad_request)? + .into() + }; let db = &ctx.state().db; let env = &db.env; @@ -314,7 +323,6 @@ pub async fn update_schema(mut ctx: Context) -> SResult { .open_index(&index_uid) .ok_or(ResponseError::index_not_found(index_uid))?; - let schema: meilidb_schema::Schema = schema.into(); let update_id = index .schema_update(&mut writer, schema.clone()) .map_err(ResponseError::internal)?; From 7f2e5d091a9548c4b700f7bb833c5b946b05d6cf Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 Nov 2019 15:33:42 +0100 Subject: [PATCH 26/28] Rename routes /synonym to /synonyms --- meilidb-http/src/routes/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilidb-http/src/routes/mod.rs b/meilidb-http/src/routes/mod.rs index f24532565..2065ec8bd 100644 --- a/meilidb-http/src/routes/mod.rs +++ b/meilidb-http/src/routes/mod.rs @@ -60,7 +60,7 @@ pub fn load_routes(app: &mut tide::App) { .post(document::delete_multiple_documents); }); - router.at("/synonym").nest(|router| { + router.at("/synonyms").nest(|router| { router .at("/") .get(synonym::list) From 055368acd8afade96bbd27e920da752ffc922558 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 Nov 2019 17:28:46 +0100 Subject: [PATCH 27/28] Fix for review --- meilidb-core/src/store/main.rs | 24 ++--- meilidb-http/src/data.rs | 12 ++- meilidb-http/src/helpers/meilidb.rs | 2 +- meilidb-http/src/helpers/tide.rs | 2 +- meilidb-http/src/routes/document.rs | 4 +- meilidb-http/src/routes/index.rs | 148 ++++++++++++++++------------ meilidb-http/src/routes/search.rs | 1 + meilidb-http/src/routes/stats.rs | 56 ++++++----- meilidb-schema/src/lib.rs | 2 +- 9 files changed, 140 insertions(+), 111 deletions(-) diff --git a/meilidb-core/src/store/main.rs b/meilidb-core/src/store/main.rs index 04ea6aee5..cd9245a52 100644 --- a/meilidb-core/src/store/main.rs +++ b/meilidb-core/src/store/main.rs @@ -6,16 +6,16 @@ use meilidb_schema::Schema; use std::collections::HashMap; use std::sync::Arc; -const CREATED_AT: &str = "created-at"; +const CREATED_AT_KEY: &str = "created-at"; const CUSTOMS_KEY: &str = "customs-key"; -const FIELDS_FREQUENCY: &str = "fields-frequency"; -const NAME: &str = "name"; +const FIELDS_FREQUENCY_KEY: &str = "fields-frequency"; +const NAME_KEY: &str = "name"; const NUMBER_OF_DOCUMENTS_KEY: &str = "number-of-documents"; const RANKED_MAP_KEY: &str = "ranked-map"; const SCHEMA_KEY: &str = "schema"; const STOP_WORDS_KEY: &str = "stop-words"; const SYNONYMS_KEY: &str = "synonyms"; -const UPDATED_AT: &str = "updated-at"; +const UPDATED_AT_KEY: &str = "updated-at"; const WORDS_KEY: &str = "words"; pub type FreqsMap = HashMap; @@ -33,32 +33,32 @@ impl Main { } pub fn put_name(self, writer: &mut heed::RwTxn, name: &str) -> ZResult<()> { - self.main.put::(writer, NAME, name) + self.main.put::(writer, NAME_KEY, name) } pub fn name(self, reader: &heed::RoTxn) -> ZResult> { Ok(self .main - .get::(reader, NAME)? + .get::(reader, NAME_KEY)? .map(|name| name.to_owned())) } pub fn put_created_at(self, writer: &mut heed::RwTxn) -> ZResult<()> { self.main - .put::(writer, CREATED_AT, &Utc::now()) + .put::(writer, CREATED_AT_KEY, &Utc::now()) } pub fn created_at(self, reader: &heed::RoTxn) -> ZResult>> { - self.main.get::(reader, CREATED_AT) + self.main.get::(reader, CREATED_AT_KEY) } pub fn put_updated_at(self, writer: &mut heed::RwTxn) -> ZResult<()> { self.main - .put::(writer, UPDATED_AT, &Utc::now()) + .put::(writer, UPDATED_AT_KEY, &Utc::now()) } pub fn updated_at(self, reader: &heed::RoTxn) -> ZResult>> { - self.main.get::(reader, UPDATED_AT) + self.main.get::(reader, UPDATED_AT_KEY) } pub fn put_words_fst(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> ZResult<()> { @@ -159,13 +159,13 @@ impl Main { fields_frequency: &FreqsMap, ) -> ZResult<()> { self.main - .put::(writer, FIELDS_FREQUENCY, fields_frequency) + .put::(writer, FIELDS_FREQUENCY_KEY, fields_frequency) } pub fn fields_frequency(&self, reader: &heed::RoTxn) -> ZResult> { match self .main - .get::(&reader, FIELDS_FREQUENCY)? + .get::(reader, FIELDS_FREQUENCY_KEY)? { Some(freqs) => Ok(Some(freqs)), None => Ok(None), diff --git a/meilidb-http/src/data.rs b/meilidb-http/src/data.rs index f8af58187..ef2d4404e 100644 --- a/meilidb-http/src/data.rs +++ b/meilidb-http/src/data.rs @@ -4,13 +4,15 @@ use std::sync::Arc; use chrono::{DateTime, Utc}; use heed::types::{SerdeBincode, Str}; -use log::*; +use log::error; use meilidb_core::{Database, Error as MError, MResult}; use sysinfo::Pid; use crate::option::Opt; use crate::routes::index::index_update_callback; +const LAST_UPDATE_KEY: &str = "last-update"; + type SerdeDatetime = SerdeBincode>; #[derive(Clone)] @@ -46,7 +48,7 @@ impl DataInner { match self .db .common_store() - .get::(&reader, "last-update")? + .get::(reader, LAST_UPDATE_KEY)? { Some(datetime) => Ok(Some(datetime)), None => Ok(None), @@ -56,11 +58,11 @@ impl DataInner { pub fn set_last_update(&self, writer: &mut heed::RwTxn) -> MResult<()> { self.db .common_store() - .put::(writer, "last-update", &Utc::now()) + .put::(writer, LAST_UPDATE_KEY, &Utc::now()) .map_err(Into::into) } - pub fn compute_stats(&self, mut writer: &mut heed::RwTxn, index_uid: &str) -> MResult<()> { + pub fn compute_stats(&self, writer: &mut heed::RwTxn, index_uid: &str) -> MResult<()> { let index = match self.db.open_index(&index_uid) { Some(index) => index, None => { @@ -93,7 +95,7 @@ impl DataInner { index .main - .put_fields_frequency(&mut writer, &frequency) + .put_fields_frequency(writer, &frequency) .map_err(MError::Zlmdb) } } diff --git a/meilidb-http/src/helpers/meilidb.rs b/meilidb-http/src/helpers/meilidb.rs index d00ba29be..2e6a17ae9 100644 --- a/meilidb-http/src/helpers/meilidb.rs +++ b/meilidb-http/src/helpers/meilidb.rs @@ -1,6 +1,6 @@ use crate::routes::setting::{RankingOrdering, SettingBody}; use indexmap::IndexMap; -use log::*; +use log::error; use meilidb_core::criterion::*; use meilidb_core::Highlight; use meilidb_core::{Index, RankedMap}; diff --git a/meilidb-http/src/helpers/tide.rs b/meilidb-http/src/helpers/tide.rs index 9a9c9adfa..be9cddaa9 100644 --- a/meilidb-http/src/helpers/tide.rs +++ b/meilidb-http/src/helpers/tide.rs @@ -39,7 +39,7 @@ impl ContextExt for Context { .get::>(&reader, &token_key) .map_err(ResponseError::internal)? .ok_or(ResponseError::invalid_token(format!( - "token key does not exist: {}", + "Api key does not exist: {}", user_api_key )))?; diff --git a/meilidb-http/src/routes/document.rs b/meilidb-http/src/routes/document.rs index 164c129c2..88c051c66 100644 --- a/meilidb-http/src/routes/document.rs +++ b/meilidb-http/src/routes/document.rs @@ -114,9 +114,7 @@ pub async fn get_all_documents(ctx: Context) -> SResult { } } - Ok(tide::response::json(response_body) - .with_status(StatusCode::OK) - .into_response()) + Ok(tide::response::json(response_body)) } fn infered_schema(document: &IndexMap) -> Option { diff --git a/meilidb-http/src/routes/index.rs b/meilidb-http/src/routes/index.rs index 3382c4d3a..6f0e5f66c 100644 --- a/meilidb-http/src/routes/index.rs +++ b/meilidb-http/src/routes/index.rs @@ -1,6 +1,6 @@ use chrono::{DateTime, Utc}; use http::StatusCode; -use log::*; +use log::error; use meilidb_core::ProcessedUpdateResult; use meilidb_schema::{Schema, SchemaBuilder}; use rand::seq::SliceRandom; @@ -32,46 +32,51 @@ pub async fn list_indexes(ctx: Context) -> SResult { let indexes_uids = ctx.state().db.indexes_uids(); let env = &ctx.state().db.env; - let mut reader = env.read_txn().map_err(ResponseError::internal)?; + let reader = env.read_txn().map_err(ResponseError::internal)?; let mut response_body = Vec::new(); for index_uid in indexes_uids { - let index = ctx - .state() - .db - .open_index(&index_uid) - .ok_or(ResponseError::internal(&index_uid))?; - let name = index - .main - .name(&mut reader) - .map_err(ResponseError::internal)? - .ok_or(ResponseError::internal("Name not found"))?; - let created_at = index - .main - .created_at(&mut reader) - .map_err(ResponseError::internal)? - .ok_or(ResponseError::internal("Created date not found"))?; - let updated_at = index - .main - .updated_at(&mut reader) - .map_err(ResponseError::internal)? - .ok_or(ResponseError::internal("Updated date not found"))?; + let index = ctx.state().db.open_index(&index_uid); - let index_reponse = IndexResponse { - name, - uid: index_uid, - created_at, - updated_at, - }; - response_body.push(index_reponse); + match index { + Some(index) => { + let name = index + .main + .name(&reader) + .map_err(ResponseError::internal)? + .ok_or(ResponseError::internal("'name' not found"))?; + let created_at = index + .main + .created_at(&reader) + .map_err(ResponseError::internal)? + .ok_or(ResponseError::internal("'created_at' date not found"))?; + let updated_at = index + .main + .updated_at(&reader) + .map_err(ResponseError::internal)? + .ok_or(ResponseError::internal("'updated_at' date not found"))?; + + let index_reponse = IndexResponse { + name, + uid: index_uid, + created_at, + updated_at, + }; + response_body.push(index_reponse); + } + None => error!( + "Index {} is referenced in the indexes list but cannot be found", + index_uid + ), + } } Ok(tide::response::json(response_body)) } -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] struct IndexResponse { name: String, uid: String, @@ -85,24 +90,24 @@ pub async fn get_index(ctx: Context) -> SResult { let index = ctx.index()?; let env = &ctx.state().db.env; - let mut reader = env.read_txn().map_err(ResponseError::internal)?; + let reader = env.read_txn().map_err(ResponseError::internal)?; - let uid = ctx.url_param("index")?.to_string(); + let uid = ctx.url_param("index")?; let name = index .main - .name(&mut reader) + .name(&reader) .map_err(ResponseError::internal)? - .ok_or(ResponseError::internal("Name not found"))?; + .ok_or(ResponseError::internal("'name' not found"))?; let created_at = index .main - .created_at(&mut reader) + .created_at(&reader) .map_err(ResponseError::internal)? - .ok_or(ResponseError::internal("Created date not found"))?; + .ok_or(ResponseError::internal("'created_at' date not found"))?; let updated_at = index .main - .updated_at(&mut reader) + .updated_at(&reader) .map_err(ResponseError::internal)? - .ok_or(ResponseError::internal("Updated date not found"))?; + .ok_or(ResponseError::internal("'updated_at' date not found"))?; let response_body = IndexResponse { name, @@ -114,15 +119,15 @@ pub async fn get_index(ctx: Context) -> SResult { Ok(tide::response::json(response_body)) } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct IndexCreateRequest { name: String, schema: Option, } -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] struct IndexCreateResponse { name: String, uid: String, @@ -166,11 +171,11 @@ pub async fn create_index(mut ctx: Context) -> SResult { .put_updated_at(&mut writer) .map_err(ResponseError::internal)?; - let schema: Option = body.schema.clone().map(|s| s.into()); + let schema: Option = body.schema.clone().map(Into::into); let mut response_update_id = None; if let Some(schema) = schema { let update_id = created_index - .schema_update(&mut writer, schema.clone()) + .schema_update(&mut writer, schema) .map_err(ResponseError::internal)?; response_update_id = Some(update_id) } @@ -191,14 +196,14 @@ pub async fn create_index(mut ctx: Context) -> SResult { .into_response()) } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct UpdateIndexRequest { name: String, } -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] struct UpdateIndexResponse { name: String, uid: String, @@ -239,12 +244,12 @@ pub async fn update_index(mut ctx: Context) -> SResult { .main .created_at(&reader) .map_err(ResponseError::internal)? - .ok_or(ResponseError::internal("Created date not found"))?; + .ok_or(ResponseError::internal("'created_at' date not found"))?; let updated_at = index .main .updated_at(&reader) .map_err(ResponseError::internal)? - .ok_or(ResponseError::internal("Updated date not found"))?; + .ok_or(ResponseError::internal("'updated_at' date not found"))?; let response_body = UpdateIndexResponse { name: body.name, @@ -259,7 +264,7 @@ pub async fn update_index(mut ctx: Context) -> SResult { } #[derive(Default, Deserialize)] -#[serde(rename_all = "camelCase")] +#[serde(rename_all = "camelCase", deny_unknown_fields)] struct SchemaParams { raw: bool, } @@ -283,16 +288,12 @@ pub async fn get_index_schema(ctx: Context) -> SResult { match schema { Some(schema) => { if params.raw { - Ok(tide::response::json(schema.to_builder())) + Ok(tide::response::json(schema)) } else { Ok(tide::response::json(SchemaBody::from(schema))) } } - None => Ok( - tide::response::json(json!({ "message": "missing index schema" })) - .with_status(StatusCode::NOT_FOUND) - .into_response(), - ), + None => Err(ResponseError::not_found("missing index schema")), } } @@ -303,7 +304,7 @@ pub async fn update_schema(mut ctx: Context) -> SResult { let params: SchemaParams = ctx.url_query().unwrap_or_default(); - let schema: Schema = if params.raw { + let schema = if params.raw { ctx.body_json::() .await .map_err(ResponseError::bad_request)? @@ -397,18 +398,35 @@ pub async fn delete_index(ctx: Context) -> SResult { } } -pub fn index_update_callback(index_uid: &str, data: &Data, _status: ProcessedUpdateResult) { - let env = &data.db.env; - let mut writer = env.write_txn().unwrap(); - - data.compute_stats(&mut writer, &index_uid).unwrap(); - data.set_last_update(&mut writer).unwrap(); +pub fn index_update_callback(index_uid: &str, data: &Data, status: ProcessedUpdateResult) { + if status.error.is_some() { + return; + } if let Some(index) = data.db.open_index(&index_uid) { + let env = &data.db.env; + let mut writer = match env.write_txn() { + Ok(writer) => writer, + Err(e) => { + error!("Impossible to get write_txn; {}", e); + return; + } + }; + + if let Err(e) = data.compute_stats(&mut writer, &index_uid) { + error!("Impossible to compute stats; {}", e) + } + + if let Err(e) = data.set_last_update(&mut writer) { + error!("Impossible to update last_update; {}", e) + } + if let Err(e) = index.main.put_updated_at(&mut writer) { error!("Impossible to update updated_at; {}", e) } - } - writer.commit().unwrap(); + if let Err(e) = writer.commit() { + error!("Impossible to get write_txn; {}", e); + } + } } diff --git a/meilidb-http/src/routes/search.rs b/meilidb-http/src/routes/search.rs index 6e1b558eb..d94958bc2 100644 --- a/meilidb-http/src/routes/search.rs +++ b/meilidb-http/src/routes/search.rs @@ -156,6 +156,7 @@ pub async fn search_multi_index(mut ctx: Context) -> SResult { for index in index_list.clone() { if index == "*" { index_list = ctx.state().db.indexes_uids().into_iter().collect(); + break; } } diff --git a/meilidb-http/src/routes/stats.rs b/meilidb-http/src/routes/stats.rs index af2e3e49d..0a5604cb0 100644 --- a/meilidb-http/src/routes/stats.rs +++ b/meilidb-http/src/routes/stats.rs @@ -1,7 +1,9 @@ +use std::collections::HashMap; + use chrono::{DateTime, Utc}; +use log::error; use pretty_bytes::converter::convert; use serde::Serialize; -use std::collections::HashMap; use sysinfo::{NetworkExt, Pid, ProcessExt, ProcessorExt, System, SystemExt}; use tide::{Context, Response}; use walkdir::WalkDir; @@ -42,7 +44,7 @@ pub async fn index_stat(ctx: Context) -> SResult { .state() .is_indexing(&reader, &index_uid) .map_err(ResponseError::internal)? - .ok_or(ResponseError::not_found("Index not found"))?; + .ok_or(ResponseError::internal("'is_indexing' date not found"))?; let response = IndexStatsResponse { number_of_documents, @@ -71,31 +73,39 @@ pub async fn get_stats(ctx: Context) -> SResult { let indexes_set = ctx.state().db.indexes_uids(); for index_uid in indexes_set { - let index = db.open_index(&index_uid).unwrap(); + let index = ctx.state().db.open_index(&index_uid); - let number_of_documents = index - .main - .number_of_documents(&reader) - .map_err(ResponseError::internal)?; + match index { + Some(index) => { + let number_of_documents = index + .main + .number_of_documents(&reader) + .map_err(ResponseError::internal)?; - let fields_frequency = index - .main - .fields_frequency(&reader) - .map_err(ResponseError::internal)? - .unwrap_or_default(); + let fields_frequency = index + .main + .fields_frequency(&reader) + .map_err(ResponseError::internal)? + .unwrap_or_default(); - let is_indexing = ctx - .state() - .is_indexing(&reader, &index_uid) - .map_err(ResponseError::internal)? - .ok_or(ResponseError::not_found("Index not found"))?; + let is_indexing = ctx + .state() + .is_indexing(&reader, &index_uid) + .map_err(ResponseError::internal)? + .ok_or(ResponseError::internal("'is_indexing' date not found"))?; - let response = IndexStatsResponse { - number_of_documents, - is_indexing, - fields_frequency, - }; - index_list.insert(index_uid, response); + let response = IndexStatsResponse { + number_of_documents, + is_indexing, + fields_frequency, + }; + index_list.insert(index_uid, response); + } + None => error!( + "Index {:?} is referenced in the indexes list but cannot be found", + index_uid + ), + } } let database_size = WalkDir::new(ctx.state().db_path.clone()) diff --git a/meilidb-schema/src/lib.rs b/meilidb-schema/src/lib.rs index a7125e434..929316d41 100644 --- a/meilidb-schema/src/lib.rs +++ b/meilidb-schema/src/lib.rs @@ -145,7 +145,7 @@ struct InnerSchema { } impl Schema { - pub fn to_builder(&self) -> SchemaBuilder { + fn to_builder(&self) -> SchemaBuilder { let identifier = self.inner.identifier.clone(); let attributes = self.attributes_ordered(); SchemaBuilder { From d60aa722c007a0d10bd02c5ae744926a145f312f Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 21 Nov 2019 11:38:21 +0100 Subject: [PATCH 28/28] Allow to update `expireAt` and `revoked` on token --- meilidb-http/src/routes/key.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/meilidb-http/src/routes/key.rs b/meilidb-http/src/routes/key.rs index cdefc39ba..94b6ba2fa 100644 --- a/meilidb-http/src/routes/key.rs +++ b/meilidb-http/src/routes/key.rs @@ -117,6 +117,8 @@ pub struct UpdatedRequest { description: Option, acl: Option>, indexes: Option>, + expires_at: Option>, + revoked: Option, } pub async fn update(mut ctx: Context) -> SResult { @@ -154,6 +156,14 @@ pub async fn update(mut ctx: Context) -> SResult { token_config.indexes = indexes; } + if let Some(expires_at) = data.expires_at { + token_config.expires_at = expires_at; + } + + if let Some(revoked) = data.revoked { + token_config.revoked = revoked; + } + token_config.updated_at = Utc::now(); common_store @@ -163,7 +173,7 @@ pub async fn update(mut ctx: Context) -> SResult { writer.commit().map_err(ResponseError::internal)?; Ok(tide::response::json(token_config) - .with_status(StatusCode::ACCEPTED) + .with_status(StatusCode::OK) .into_response()) } @@ -185,5 +195,5 @@ pub async fn delete(ctx: Context) -> SResult { writer.commit().map_err(ResponseError::internal)?; - Ok(StatusCode::ACCEPTED) + Ok(StatusCode::NO_CONTENT) }