Merge #914

914: lazily create an index on documents push r=LegendreM a=qdequele

Create an index if it's possible when a user trying to send data to a non-existing index. https://github.com/meilisearch/MeiliSearch/issues/918

Co-authored-by: qdequele <quentin@meilisearch.com>
Co-authored-by: qdequele <quentin@dequelen.me>

meilisearch-http/src/routes/document.rs

@@ -45,7 +45,8 @@ async fn get_document(
 
     let reader = data.db.main_read_txn()?;
 
-    let internal_id = index.main
+    let internal_id = index
+        .main
         .external_to_internal_docid(&reader, &path.document_id)?
         .ok_or(Error::document_not_found(&path.document_id))?;
 
@@ -166,47 +167,41 @@ async fn update_multiple_documents(
     body: web::Json<Vec<Document>>,
     is_partial: bool,
 ) -> Result<HttpResponse, ResponseError> {
-    let index = data
-        .db
-        .open_index(&path.index_uid)
-        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let update_id = data.get_or_create_index(&path.index_uid, |index| {
+        let reader = data.db.main_read_txn()?;
 
-    let reader = data.db.main_read_txn()?;
+        let mut schema = index
+            .main
+            .schema(&reader)?
+            .ok_or(meilisearch_core::Error::SchemaMissing)?;
 
-    let mut schema = index
-        .main
-        .schema(&reader)?
-        .ok_or(meilisearch_core::Error::SchemaMissing)?;
+        if schema.primary_key().is_none() {
+            let id = match &params.primary_key {
+                Some(id) => id.to_string(),
+                None => body
+                    .first()
+                    .and_then(find_primary_key)
+                    .ok_or(meilisearch_core::Error::MissingPrimaryKey)?,
+            };
 
-    if schema.primary_key().is_none() {
-        let id = match &params.primary_key {
-            Some(id) => id.to_string(),
-            None => body
-                .first()
-                .and_then(find_primary_key)
-                .ok_or(meilisearch_core::Error::MissingPrimaryKey)?
+            schema.set_primary_key(&id).map_err(Error::bad_request)?;
+
+            data.db.main_write(|w| index.main.put_schema(w, &schema))?;
+        }
+
+        let mut document_addition = if is_partial {
+            index.documents_partial_addition()
+        } else {
+            index.documents_addition()
         };
 
-        schema
-            .set_primary_key(&id)
-            .map_err(Error::bad_request)?;
+        for document in body.into_inner() {
+            document_addition.update_document(document);
+        }
 
-        data.db.main_write(|w| index.main.put_schema(w, &schema))?;
-    }
-
-    let mut document_addition = if is_partial {
-        index.documents_partial_addition()
-    } else {
-        index.documents_addition()
-    };
-
-    for document in body.into_inner() {
-        document_addition.update_document(document);
-    }
-
-    let update_id = data.db.update_write(|w| document_addition.finalize(w))?;
-
-    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
+        Ok(data.db.update_write(|w| document_addition.finalize(w))?)
+    })?;
+    return Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)));
 }
 
 #[post("/indexes/{index_uid}/documents", wrap = "Authentication::Private")]
@@ -243,7 +238,6 @@ async fn delete_documents(
         .open_index(&path.index_uid)
         .ok_or(Error::index_not_found(&path.index_uid))?;
 
-
     let mut documents_deletion = index.documents_deletion();
 
     for document_id in body.into_inner() {

meilisearch-http/src/routes/setting.rs

@@ -53,13 +53,12 @@ async fn update_all(
     path: web::Path<IndexParam>,
     body: web::Json<Settings>,
 ) -> Result<HttpResponse, ResponseError> {
-    let settings = body
-        .into_inner()
-        .to_update()
-        .map_err(Error::bad_request)?;
-
-    let update_id = data.db.update_write::<_, _, Error>(|writer| {
-        update_all_settings_txn(&data, settings, &path.index_uid, writer)
+    let update_id = data.get_or_create_index(&path.index_uid, |index| {
+        Ok(data.db.update_write::<_, _, ResponseError>(|writer| {
+            let settings = body.into_inner().to_update().map_err(Error::bad_request)?;
+            let update_id = index.settings_update(writer, settings)?;
+            Ok(update_id)
+        })?)
     })?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
@@ -71,11 +70,7 @@ pub fn get_all_sync(data: &web::Data<Data>, reader: &MainReader, index_uid: &str
         .open_index(index_uid)
         .ok_or(Error::index_not_found(index_uid))?;
 
-    let stop_words: BTreeSet<String> = index
-        .main
-        .stop_words(reader)?
-        .into_iter()
-        .collect();
+    let stop_words: BTreeSet<String> = index.main.stop_words(&reader)?.into_iter().collect();
 
     let synonyms_list = index.main.synonyms(reader)?;
 
@@ -94,22 +89,19 @@ pub fn get_all_sync(data: &web::Data<Data>, reader: &MainReader, index_uid: &str
         .map(|r| r.to_string())
         .collect();
 
-
-    let schema = index.main.schema(reader)?;
+    let schema = index.main.schema(&reader)?;
 
     let distinct_attribute = match (index.main.distinct_attribute(reader)?, &schema) {
         (Some(id), Some(schema)) => schema.name(id).map(str::to_string),
         _ => None,
     };
 
-    let attributes_for_faceting = match (&schema, &index.main.attributes_for_faceting(reader)?) {
-        (Some(schema), Some(attrs)) => {
-            attrs
-                .iter()
-                .filter_map(|&id| schema.name(id))
-                .map(str::to_string)
-                .collect()
-        }
+    let attributes_for_faceting = match (&schema, &index.main.attributes_for_faceting(&reader)?) {
+        (Some(schema), Some(attrs)) => attrs
+            .iter()
+            .filter_map(|&id| schema.name(id))
+            .map(str::to_string)
+            .collect(),
         _ => vec![],
     };
 
@@ -159,7 +151,9 @@ async fn delete_all(
         attributes_for_faceting: UpdateState::Clear,
     };
 
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+    let update_id = data
+        .db
+        .update_write(|w| index.settings_update(w, settings))?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -198,18 +192,17 @@ async fn update_rules(
     path: web::Path<IndexParam>,
     body: web::Json<Option<Vec<String>>>,
 ) -> Result<HttpResponse, ResponseError> {
-    let index = data
-        .db
-        .open_index(&path.index_uid)
-        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let update_id = data.get_or_create_index(&path.index_uid, |index| {
+        let settings = Settings {
+            ranking_rules: Some(body.into_inner()),
+            ..Settings::default()
+        };
 
-    let settings = Settings {
-        ranking_rules: Some(body.into_inner()),
-        ..Settings::default()
-    };
-
-    let settings = settings.to_update().map_err(Error::bad_request)?;
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+        let settings = settings.to_update().map_err(Error::bad_request)?;
+        Ok(data
+            .db
+            .update_write(|w| index.settings_update(w, settings))?)
+    })?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -232,7 +225,9 @@ async fn delete_rules(
         ..SettingsUpdate::default()
     };
 
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+    let update_id = data
+        .db
+        .update_write(|w| index.settings_update(w, settings))?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -269,18 +264,17 @@ async fn update_distinct(
     path: web::Path<IndexParam>,
     body: web::Json<Option<String>>,
 ) -> Result<HttpResponse, ResponseError> {
-    let index = data
-        .db
-        .open_index(&path.index_uid)
-        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let update_id = data.get_or_create_index(&path.index_uid, |index| {
+        let settings = Settings {
+            distinct_attribute: Some(body.into_inner()),
+            ..Settings::default()
+        };
 
-    let settings = Settings {
-        distinct_attribute: Some(body.into_inner()),
-        ..Settings::default()
-    };
-
-    let settings = settings.to_update().map_err(Error::bad_request)?;
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+        let settings = settings.to_update().map_err(Error::bad_request)?;
+        Ok(data
+            .db
+            .update_write(|w| index.settings_update(w, settings))?)
+    })?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -303,7 +297,9 @@ async fn delete_distinct(
         ..SettingsUpdate::default()
     };
 
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+    let update_id = data
+        .db
+        .update_write(|w| index.settings_update(w, settings))?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -322,8 +318,7 @@ async fn get_searchable(
         .ok_or(Error::index_not_found(&path.index_uid))?;
     let reader = data.db.main_read_txn()?;
     let schema = index.main.schema(&reader)?;
-    let searchable_attributes: Option<Vec<String>> =
-        schema.as_ref().map(get_indexed_attributes);
+    let searchable_attributes: Option<Vec<String>> = schema.as_ref().map(get_indexed_attributes);
 
     Ok(HttpResponse::Ok().json(searchable_attributes))
 }
@@ -337,19 +332,18 @@ async fn update_searchable(
     path: web::Path<IndexParam>,
     body: web::Json<Option<Vec<String>>>,
 ) -> Result<HttpResponse, ResponseError> {
-    let index = data
-        .db
-        .open_index(&path.index_uid)
-        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let update_id = data.get_or_create_index(&path.index_uid, |index| {
+        let settings = Settings {
+            searchable_attributes: Some(body.into_inner()),
+            ..Settings::default()
+        };
 
-    let settings = Settings {
-        searchable_attributes: Some(body.into_inner()),
-        ..Settings::default()
-    };
+        let settings = settings.to_update().map_err(Error::bad_request)?;
 
-    let settings = settings.to_update().map_err(Error::bad_request)?;
-
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+        Ok(data
+            .db
+            .update_write(|w| index.settings_update(w, settings))?)
+    })?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -372,7 +366,9 @@ async fn delete_searchable(
         ..SettingsUpdate::default()
     };
 
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+    let update_id = data
+        .db
+        .update_write(|w| index.settings_update(w, settings))?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -407,18 +403,17 @@ async fn update_displayed(
     path: web::Path<IndexParam>,
     body: web::Json<Option<BTreeSet<String>>>,
 ) -> Result<HttpResponse, ResponseError> {
-    let index = data
-        .db
-        .open_index(&path.index_uid)
-        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let update_id = data.get_or_create_index(&path.index_uid, |index| {
+        let settings = Settings {
+            displayed_attributes: Some(body.into_inner()),
+            ..Settings::default()
+        };
 
-    let settings = Settings {
-        displayed_attributes: Some(body.into_inner()),
-        ..Settings::default()
-    };
-
-    let settings = settings.to_update().map_err(Error::bad_request)?;
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+        let settings = settings.to_update().map_err(Error::bad_request)?;
+        Ok(data
+            .db
+            .update_write(|w| index.settings_update(w, settings))?)
+    })?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -441,7 +436,9 @@ async fn delete_displayed(
         ..SettingsUpdate::default()
     };
 
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+    let update_id = data
+        .db
+        .update_write(|w| index.settings_update(w, settings))?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -459,20 +456,16 @@ async fn get_attributes_for_faceting(
         .open_index(&path.index_uid)
         .ok_or(Error::index_not_found(&path.index_uid))?;
 
-    let attributes_for_faceting = data
-        .db
-        .main_read::<_, _, ResponseError>(|reader| {
+    let attributes_for_faceting = data.db.main_read::<_, _, ResponseError>(|reader| {
         let schema = index.main.schema(reader)?;
         let attrs = index.main.attributes_for_faceting(reader)?;
         let attr_names = match (&schema, &attrs) {
-            (Some(schema), Some(attrs)) => {
-                attrs
-                    .iter()
-                    .filter_map(|&id| schema.name(id))
-                    .map(str::to_string)
-                    .collect()
-            }
-            _ => vec![]
+            (Some(schema), Some(attrs)) => attrs
+                .iter()
+                .filter_map(|&id| schema.name(id))
+                .map(str::to_string)
+                .collect(),
+            _ => vec![],
         };
         Ok(attr_names)
     })?;
@@ -489,18 +482,17 @@ async fn update_attributes_for_faceting(
     path: web::Path<IndexParam>,
     body: web::Json<Option<Vec<String>>>,
 ) -> Result<HttpResponse, ResponseError> {
-    let index = data
-        .db
-        .open_index(&path.index_uid)
-        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let update_id = data.get_or_create_index(&path.index_uid, |index| {
+        let settings = Settings {
+            attributes_for_faceting: Some(body.into_inner()),
+            ..Settings::default()
+        };
 
-    let settings = Settings {
-        attributes_for_faceting: Some(body.into_inner()),
-        ..Settings::default()
-    };
-
-    let settings = settings.to_update().map_err(Error::bad_request)?;
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+        let settings = settings.to_update().map_err(Error::bad_request)?;
+        Ok(data
+            .db
+            .update_write(|w| index.settings_update(w, settings))?)
+    })?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -523,7 +515,9 @@ async fn delete_attributes_for_faceting(
         ..SettingsUpdate::default()
     };
 
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+    let update_id = data
+        .db
+        .update_write(|w| index.settings_update(w, settings))?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -532,7 +526,8 @@ fn get_indexed_attributes(schema: &Schema) -> Vec<String> {
     if schema.is_indexed_all() {
         ["*"].iter().map(|s| s.to_string()).collect()
     } else {
-        schema.indexed_name()
+        schema
+            .indexed_name()
             .iter()
             .map(|s| s.to_string())
             .collect()
@@ -543,7 +538,8 @@ fn get_displayed_attributes(schema: &Schema) -> BTreeSet<String> {
     if schema.is_displayed_all() {
         ["*"].iter().map(|s| s.to_string()).collect()
     } else {
-        schema.displayed_name()
+        schema
+            .displayed_name()
             .iter()
             .map(|s| s.to_string())
             .collect()

meilisearch-http/src/routes/stop_words.rs

@@ -39,17 +39,16 @@ async fn update(
     path: web::Path<IndexParam>,
     body: web::Json<BTreeSet<String>>,
 ) -> Result<HttpResponse, ResponseError> {
-    let index = data
-        .db
-        .open_index(&path.index_uid)
-        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let update_id = data.get_or_create_index(&path.index_uid, |index| {
+        let settings = SettingsUpdate {
+            stop_words: UpdateState::Update(body.into_inner()),
+            ..SettingsUpdate::default()
+        };
 
-    let settings = SettingsUpdate {
-        stop_words: UpdateState::Update(body.into_inner()),
-        ..SettingsUpdate::default()
-    };
-
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+        Ok(data
+            .db
+            .update_write(|w| index.settings_update(w, settings))?)
+    })?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -72,7 +71,9 @@ async fn delete(
         ..SettingsUpdate::default()
     };
 
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+    let update_id = data
+        .db
+        .update_write(|w| index.settings_update(w, settings))?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }

meilisearch-http/src/routes/synonym.rs

@@ -50,17 +50,16 @@ async fn update(
     path: web::Path<IndexParam>,
     body: web::Json<BTreeMap<String, Vec<String>>>,
 ) -> Result<HttpResponse, ResponseError> {
-    let index = data
-        .db
-        .open_index(&path.index_uid)
-        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let update_id = data.get_or_create_index(&path.index_uid, |index| {
+        let settings = SettingsUpdate {
+            synonyms: UpdateState::Update(body.into_inner()),
+            ..SettingsUpdate::default()
+        };
 
-    let settings = SettingsUpdate {
-        synonyms: UpdateState::Update(body.into_inner()),
-        ..SettingsUpdate::default()
-    };
-
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+        Ok(data
+            .db
+            .update_write(|w| index.settings_update(w, settings))?)
+    })?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -83,7 +82,9 @@ async fn delete(
         ..SettingsUpdate::default()
     };
 
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+    let update_id = data
+        .db
+        .update_write(|w| index.settings_update(w, settings))?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }