Compare commits

...

47 Commits

Author SHA1 Message Date
Mubelotix
99c56630d1 Merge branch 'main' into render-route 2025-08-18 12:31:04 +02:00
Mubelotix
fd54722bce Discard clippy warning 2025-08-05 16:21:50 +02:00
Mubelotix
b26d05aed4 Fix tests 2025-08-05 16:15:09 +02:00
Mubelotix
2d43c50723 Merge branch 'fragment-filters' into render-route 2025-08-05 16:03:28 +02:00
Mubelotix
6113831eb5 Disable clippy warning 2025-08-01 14:56:44 +02:00
Mubelotix
a20f353054 Add columns info in errors 2025-08-01 14:53:41 +02:00
Mubelotix
8fe4d33b5a Refactor build_doc 2025-08-01 12:34:33 +02:00
Mubelotix
1507403596 Add documentation
Co-Authored-By: Louis Dureuil <louis.dureuil@xinra.net>
2025-08-01 12:11:32 +02:00
Mubelotix
233e4a1020 Force docs to be objects 2025-08-01 12:09:35 +02:00
Mubelotix
6e9bdbe2da Rename variable
Co-Authored-By: Louis Dureuil <louis.dureuil@xinra.net>
2025-08-01 11:15:35 +02:00
Mubelotix
926dce707e Remove JsonDocument
Co-Authored-By: Louis Dureuil <louis.dureuil@xinra.net>
2025-08-01 11:14:37 +02:00
Mubelotix
f105a2224a Clarify fields_unused
Co-Authored-By: Louis Dureuil <louis.dureuil@xinra.net>
2025-08-01 11:04:48 +02:00
Mubelotix
1bb95d2bef Simplify parser 2025-08-01 11:00:58 +02:00
Mubelotix
d6da6c27d8 Add test for ugly names 2025-08-01 10:54:48 +02:00
Mubelotix
7f394d59cd Add support for ugly names 2025-08-01 10:22:24 +02:00
Mubelotix
dae4fa874c Merge branch 'fragment-filters' into render-route 2025-08-01 09:13:17 +02:00
Mubelotix
99b4dce8ae Merge branch 'release-v1.16.0' into render-route 2025-08-01 08:57:37 +02:00
Mubelotix
b3c98afe65 Merge branch 'release-v1.16.0' into render-route 2025-07-29 12:05:29 +02:00
Mubelotix
20a0b1c639 Fix error message 2025-07-21 17:01:18 +02:00
Mubelotix
84990dc198 Remove insertFields parameter 2025-07-21 16:29:38 +02:00
Mubelotix
381eddd1f8 Fix tests 2025-07-21 16:09:37 +02:00
Mubelotix
b1d3a8c58f Improve code readability 2025-07-21 16:06:07 +02:00
Mubelotix
6825d917f4 Remove panic 2025-07-21 15:34:07 +02:00
Mubelotix
a4eb83e6a1 Clean code 2025-07-21 15:27:33 +02:00
Mubelotix
3180ebea56 Format code 2025-07-21 15:12:59 +02:00
Mubelotix
1f0d319c67 Add analytics 2025-07-21 15:12:46 +02:00
Mubelotix
69dfd2c76e Clippy warnings 2025-07-21 14:41:42 +02:00
Mubelotix
8a40be1840 Add tests 2025-07-21 14:39:57 +02:00
Mubelotix
ab80bfc4ee Add fields test 2025-07-21 14:26:00 +02:00
Mubelotix
ac13a54a67 Add tests 2025-07-21 14:15:32 +02:00
Mubelotix
07e426f658 Add chat completions test 2025-07-21 14:01:16 +02:00
Mubelotix
be8807c64f Fix flaky test 2025-07-21 13:42:18 +02:00
Mubelotix
338daaef1d Update tests 2025-07-21 13:26:57 +02:00
Mubelotix
289a7f391b Add fields support 2025-07-18 10:35:02 +02:00
Mubelotix
00d9f576ed Add tests 2025-07-17 11:51:12 +02:00
Mubelotix
70fa94146a Add document template test 2025-07-17 11:29:55 +02:00
Mubelotix
a5186863ca Add tests 2025-07-17 11:15:45 +02:00
Mubelotix
3191316cf3 Format 2025-07-17 11:11:33 +02:00
Mubelotix
cc9fd82f79 Add tests and fix issues 2025-07-17 11:08:07 +02:00
Mubelotix
7495233025 Update authorization tests 2025-07-17 10:20:20 +02:00
Mubelotix
2d2de778a7 Template rendering 2025-07-17 08:44:04 +02:00
Mubelotix
f349ba53a0 Add a double action policy 2025-07-17 08:10:35 +02:00
Mubelotix
fc8b6e0f9f Add doc loading 2025-07-17 07:51:15 +02:00
Mubelotix
e124c161ec Template retrieval logic 2025-07-16 16:52:28 +02:00
Mubelotix
f50a5b17b6 Bind route 2025-07-16 15:07:40 +02:00
Mubelotix
e83a49821a Add error handling 2025-07-16 15:07:10 +02:00
Mubelotix
313d804b62 Create structure 2025-07-16 14:30:58 +02:00
28 changed files with 1842 additions and 152 deletions

1
Cargo.lock generated
View File

@@ -3771,6 +3771,7 @@ dependencies = [
"itertools 0.14.0",
"jsonwebtoken",
"lazy_static",
"liquid",
"manifest-dir-macros",
"maplit",
"meili-snap",

View File

@@ -18,8 +18,7 @@ use nom::sequence::{terminated, tuple};
use Condition::*;
use crate::error::IResultExt;
use crate::value::parse_vector_value;
use crate::value::parse_vector_value_cut;
use crate::value::{parse_dotted_value_cut, parse_dotted_value_part};
use crate::Error;
use crate::ErrorKind;
use crate::VectorFilter;
@@ -142,13 +141,13 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option<Token>, VectorFilter<'_>
}
let (input, embedder_name) =
parse_vector_value_cut(input, ErrorKind::VectorFilterInvalidEmbedder)?;
parse_dotted_value_cut(input, ErrorKind::VectorFilterInvalidEmbedder)?;
let (input, filter) = alt((
map(
preceded(tag(".fragments"), |input| {
let (input, _) = tag(".")(input).map_cut(ErrorKind::VectorFilterMissingFragment)?;
parse_vector_value_cut(input, ErrorKind::VectorFilterInvalidFragment)
parse_dotted_value_cut(input, ErrorKind::VectorFilterInvalidFragment)
}),
VectorFilter::Fragment,
),
@@ -159,7 +158,7 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option<Token>, VectorFilter<'_>
))(input)?;
if let Ok((input, point)) = tag::<_, _, ()>(".")(input) {
let opt_value = parse_vector_value(input).ok().map(|(_, v)| v);
let opt_value = parse_dotted_value_part(input).ok().map(|(_, v)| v);
let value =
opt_value.as_ref().map(|v| v.value().to_owned()).unwrap_or_else(|| point.to_string());
let context = opt_value.map(|v| v.original_span()).unwrap_or(point);

View File

@@ -61,7 +61,9 @@ use nom::multi::{many0, separated_list1};
use nom::number::complete::recognize_float;
use nom::sequence::{delimited, preceded, terminated, tuple};
use nom::Finish;
pub use nom::Slice;
use nom_locate::LocatedSpan;
pub use value::parse_dotted_value_part;
pub(crate) use value::parse_value;
use value::word_exact;

View File

@@ -80,8 +80,8 @@ pub fn word_exact<'a, 'b: 'a>(tag: &'b str) -> impl Fn(Span<'a>) -> IResult<'a,
}
}
/// vector_value = ( non_dot_word | singleQuoted | doubleQuoted)
pub fn parse_vector_value(input: Span) -> IResult<Token> {
/// dotted_value_part = ( non_dot_word | singleQuoted | doubleQuoted)
pub fn parse_dotted_value_part(input: Span) -> IResult<Token> {
pub fn non_dot_word(input: Span) -> IResult<Token> {
let (input, word) = take_while1(|c| is_value_component(c) && c != '.')(input)?;
Ok((input, word.into()))
@@ -113,8 +113,8 @@ pub fn parse_vector_value(input: Span) -> IResult<Token> {
}
}
pub fn parse_vector_value_cut<'a>(input: Span<'a>, kind: ErrorKind<'a>) -> IResult<'a, Token<'a>> {
parse_vector_value(input).map_err(|e| match e {
pub fn parse_dotted_value_cut<'a>(input: Span<'a>, kind: ErrorKind<'a>) -> IResult<'a, Token<'a>> {
parse_dotted_value_part(input).map_err(|e| match e {
nom::Err::Failure(e) => match e.kind() {
ErrorKind::Char(c) if *c == '"' || *c == '\'' => {
crate::Error::failure_from_kind(input, ErrorKind::VectorFilterInvalidQuotes)

View File

@@ -936,7 +936,7 @@ impl IndexScheduler {
.into_inner()
.into_iter()
.map(|fragment| {
let value = embedder_options.fragment(&fragment.name).unwrap();
let value = embedder_options.indexing_fragment(&fragment.name).unwrap();
let template = JsonTemplate::new(value.clone()).unwrap();
RuntimeFragment { name: fragment.name, id: fragment.id, template }
})

View File

@@ -419,6 +419,16 @@ InvalidChatCompletionSearchQueryParamPrompt , InvalidRequest , BAD_REQU
InvalidChatCompletionSearchFilterParamPrompt , InvalidRequest , BAD_REQUEST ;
InvalidChatCompletionSearchIndexUidParamPrompt , InvalidRequest , BAD_REQUEST ;
InvalidChatCompletionPreQueryPrompt , InvalidRequest , BAD_REQUEST ;
// Render
InvalidRenderTemplate , InvalidRequest , BAD_REQUEST ;
InvalidRenderTemplateId , InvalidRequest , BAD_REQUEST ;
InvalidRenderTemplateInline , InvalidRequest , BAD_REQUEST ;
InvalidRenderInput , InvalidRequest , BAD_REQUEST ;
InvalidRenderInputDocumentId , InvalidRequest , BAD_REQUEST ;
InvalidRenderInputInline , InvalidRequest , BAD_REQUEST ;
RenderDocumentNotFound , InvalidRequest , NOT_FOUND ;
TemplateParsingError , InvalidRequest , BAD_REQUEST ;
TemplateRenderingError , InvalidRequest , BAD_REQUEST ;
// Webhooks
InvalidWebhooks , InvalidRequest , BAD_REQUEST ;
InvalidWebhookUrl , InvalidRequest , BAD_REQUEST ;

View File

@@ -48,6 +48,7 @@ is-terminal = "0.4.16"
itertools = "0.14.0"
jsonwebtoken = "9.3.1"
lazy_static = "1.5.0"
liquid = "0.26.11"
meilisearch-auth = { path = "../meilisearch-auth" }
meilisearch-types = { path = "../meilisearch-types" }
memmap2 = "0.9.7"

View File

@@ -133,7 +133,7 @@ pub fn extract_token_from_request(
}
}
pub trait Policy {
pub trait Policy: Sized {
fn authenticate(
auth: Data<AuthController>,
token: &str,
@@ -340,6 +340,22 @@ pub mod policies {
}
}
pub struct DoubleActionPolicy<const A: u8, const B: u8>;
impl<const A: u8, const B: u8> Policy for DoubleActionPolicy<A, B> {
fn authenticate(
auth: Data<AuthController>,
token: &str,
index: Option<&str>,
) -> Result<AuthFilter, AuthError> {
let filter_a = ActionPolicy::<A>::authenticate(auth.clone(), token, index)?;
let _filter_b = ActionPolicy::<B>::authenticate(auth, token, index)?;
// There is no point merging the filters here.
// Since they originate from the same API key, they will hold the same information.
Ok(filter_a)
}
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
struct Claims {

View File

@@ -30,6 +30,8 @@ use crate::Opt;
pub mod documents;
pub mod facet_search;
pub mod render;
mod render_analytics;
pub mod search;
mod search_analytics;
#[cfg(test)]
@@ -76,6 +78,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
.service(web::scope("/documents").configure(documents::configure))
.service(web::scope("/search").configure(search::configure))
.service(web::scope("/facet-search").configure(facet_search::configure))
.service(web::scope("/render").configure(render::configure))
.service(web::scope("/similar").configure(similar::configure))
.service(web::scope("/settings").configure(settings::configure)),
);

View File

@@ -0,0 +1,612 @@
use std::collections::BTreeMap;
use actix_web::web::{self, Data};
use actix_web::{HttpRequest, HttpResponse};
use deserr::actix_web::AwebJson;
use deserr::Deserr;
use index_scheduler::IndexScheduler;
use liquid::ValueView;
use meilisearch_types::deserr::DeserrJsonError;
use meilisearch_types::error::deserr_codes::{
InvalidRenderInput, InvalidRenderInputDocumentId, InvalidRenderInputInline,
InvalidRenderTemplate, InvalidRenderTemplateId, InvalidRenderTemplateInline,
};
use meilisearch_types::error::Code;
use meilisearch_types::error::ResponseError;
use meilisearch_types::heed::RoTxn;
use meilisearch_types::index_uid::IndexUid;
use meilisearch_types::keys::actions;
use meilisearch_types::milli::prompt::{build_doc, build_doc_fields, OwnedFields};
use meilisearch_types::milli::vector::db::IndexEmbeddingConfig;
use meilisearch_types::milli::vector::json_template::{self, JsonTemplate};
use meilisearch_types::milli::vector::EmbedderOptions;
use meilisearch_types::milli::{Span, Token};
use meilisearch_types::{heed, milli, Index};
use serde::Serialize;
use serde_json::Value;
use tracing::debug;
use utoipa::{OpenApi, ToSchema};
use crate::analytics::Analytics;
use crate::extractors::authentication::policies::DoubleActionPolicy;
use crate::extractors::authentication::GuardedData;
use crate::extractors::sequential_extractor::SeqHandler;
use crate::routes::indexes::render_analytics::RenderAggregator;
#[derive(OpenApi)]
#[openapi(
paths(render_post),
tags((
name = "Render documents",
description = "The /render route allows rendering templates used by Meilisearch.",
external_docs(url = "https://www.meilisearch.com/docs/reference/api/render"),
)),
)]
pub struct RenderApi;
pub fn configure(cfg: &mut web::ServiceConfig) {
cfg.service(web::resource("").route(web::post().to(SeqHandler(render_post))));
}
/// Render documents with POST
#[utoipa::path(
post,
path = "{indexUid}/render",
tag = "Render documents",
security(("Bearer" = ["settings.get,documents.get", "*.get", "*"])),
params(("indexUid" = String, Path, example = "movies", description = "Index Unique Identifier", nullable = false)),
request_body = RenderQuery,
responses(
(status = 200, description = "The rendered result is returned along with the template", body = RenderResult, content_type = "application/json", example = json!(
{
"template": "{{ doc.breed }} called {{ doc.name }}",
"rendered": "A Jack Russell called Iko"
}
)),
(status = 404, description = "Template or document not found", body = ResponseError, content_type = "application/json", example = json!(
{
"message": "Document with ID `9999` not found.",
"code": "render_document_not_found",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#render_document_not_found"
}
)),
(status = 400, description = "Parameters are incorrect", body = ResponseError, content_type = "application/json", example = json!(
{
"message": "Indexing fragment `mistake` does not exist for embedder `rest`.\n Hint: Available indexing fragments are `basic`, `withBreed`.",
"code": "invalid_render_template_id",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template_id"
}
)),
)
)]
pub async fn render_post(
index_scheduler: GuardedData<
DoubleActionPolicy<{ actions::SETTINGS_GET }, { actions::DOCUMENTS_GET }>,
Data<IndexScheduler>,
>,
index_uid: web::Path<String>,
params: AwebJson<RenderQuery, DeserrJsonError>,
req: HttpRequest,
analytics: web::Data<Analytics>,
) -> Result<HttpResponse, ResponseError> {
let index_uid = IndexUid::try_from(index_uid.into_inner())?;
let index = index_scheduler.index(&index_uid)?;
let query = params.into_inner();
debug!(parameters = ?query, "Render document");
let mut aggregate = RenderAggregator::from_query(&query);
let result = render(index, query).await;
if result.is_ok() {
aggregate.succeed();
}
analytics.publish(aggregate, &req);
let result = result?;
debug!(returns = ?result, "Render document");
Ok(HttpResponse::Ok().json(result))
}
#[derive(Clone, Copy)]
enum FragmentKind {
Indexing,
Search,
}
impl FragmentKind {
fn as_str(&self) -> &'static str {
match self {
FragmentKind::Indexing => "indexing",
FragmentKind::Search => "search",
}
}
fn capitalized(&self) -> &'static str {
match self {
FragmentKind::Indexing => "Indexing",
FragmentKind::Search => "Search",
}
}
}
enum RenderError<'a> {
MultipleTemplates,
MissingTemplate,
EmptyTemplateId,
UnknownTemplateRoot(Token<'a>),
MissingEmbedderName {
available: Vec<String>,
},
EmbedderDoesNotExist {
embedder: Token<'a>,
available: Vec<String>,
},
EmbedderUsesFragments {
embedder: Token<'a>,
},
MissingTemplateAfterEmbedder {
embedder: Token<'a>,
indexing: Vec<String>,
search: Vec<String>,
},
UnknownTemplatePrefix {
embedder: Token<'a>,
found: Token<'a>,
indexing: Vec<String>,
search: Vec<String>,
},
ReponseError(ResponseError),
MissingFragment {
embedder: Token<'a>,
kind: FragmentKind,
available: Vec<String>,
},
FragmentDoesNotExist {
embedder: Token<'a>,
fragment: Token<'a>,
kind: FragmentKind,
available: Vec<String>,
},
LeftOverToken(Token<'a>),
MissingChatCompletionTemplate,
UnknownChatCompletionTemplate(Token<'a>),
ExpectedDotAfterValue(milli::Span<'a>),
ExpectedValue(milli::Span<'a>),
DocumentNotFound(String),
DocumentMustBeMap,
BothInlineDocAndDocId,
TemplateParsing(json_template::Error),
TemplateRendering(json_template::Error),
InputConversion(liquid::Error),
}
impl From<heed::Error> for RenderError<'_> {
fn from(error: heed::Error) -> Self {
RenderError::ReponseError(error.into())
}
}
impl From<milli::Error> for RenderError<'_> {
fn from(error: milli::Error) -> Self {
RenderError::ReponseError(error.into())
}
}
use RenderError::*;
impl From<RenderError<'_>> for ResponseError {
fn from(error: RenderError) -> Self {
fn format_span(span: &Span<'_>) -> String {
let base_column = span.get_utf8_column();
let size = span.fragment().chars().count();
format!("`{}` (cols {}:{})", span.fragment(), base_column, base_column + size)
}
fn format_token(token: &Token<'_>) -> String {
let base_column = token.original_span().get_utf8_column();
let size = token.original_span().fragment().chars().count();
format!("`{}` (cols {}:{})", token.value(), base_column, base_column + size)
}
match error {
MultipleTemplates => ResponseError::from_msg(
String::from("Cannot provide both an inline template and a template ID."),
Code::InvalidRenderTemplate,
),
MissingTemplate => ResponseError::from_msg(
String::from("No template provided. Please provide either an inline template or a template ID."),
Code::InvalidRenderTemplate,
),
EmptyTemplateId => ResponseError::from_msg(
String::from("The template ID is empty.\n Hint: Valid prefixes are `embedders` or `chatCompletions`."),
Code::InvalidRenderTemplateId,
),
UnknownTemplateRoot(root) => ResponseError::from_msg(
format!("Template ID must start with `embedders` or `chatCompletions`, but found {}.", format_token(&root)),
Code::InvalidRenderTemplateId,
),
MissingEmbedderName { mut available } => {
available.sort_unstable();
ResponseError::from_msg(
format!("Template ID configured with `embedders` but no embedder name provided.\n Hint: Available embedders are {}.",
available.iter().map(|s| format!("`{s}`")).collect::<Vec<_>>().join(", ")),
Code::InvalidRenderTemplateId,
)
},
EmbedderDoesNotExist { embedder, mut available } => {
available.sort_unstable();
ResponseError::from_msg(
format!("Embedder {} does not exist.\n Hint: Available embedders are {}.",
format_token(&embedder),
available.iter().map(|s| format!("`{s}`")).collect::<Vec<_>>().join(", ")),
Code::InvalidRenderTemplateId,
)
},
EmbedderUsesFragments { embedder } => ResponseError::from_msg(
format!("Requested document template for embedder {} but it uses fragments.\n Hint: Use `indexingFragments` or `searchFragments` instead.", format_token(&embedder)),
Code::InvalidRenderTemplateId,
),
MissingTemplateAfterEmbedder { embedder, mut indexing, mut search } => {
if indexing.is_empty() && search.is_empty() {
ResponseError::from_msg(
format!("Missing template id after embedder {}.\n Hint: Available template: `documentTemplate`.",
format_token(&embedder)),
Code::InvalidRenderTemplateId,
)
} else {
indexing.sort_unstable();
search.sort_unstable();
ResponseError::from_msg(
format!("Template ID configured with embedder {} but no template kind provided.\n Hint: Available fragments are {}.",
format_token(&embedder),
indexing.iter().map(|s| format!("`indexingFragments.{s}`")).chain(
search.iter().map(|s| format!("`searchFragments.{s}`"))).collect::<Vec<_>>().join(", ")),
Code::InvalidRenderTemplateId,
)
}
},
UnknownTemplatePrefix { embedder, found, mut indexing, mut search } => {
if indexing.is_empty() && search.is_empty() {
ResponseError::from_msg(
format!("Wrong template {} after embedder {}.\n Hint: Available template: `documentTemplate`.", format_token(&found), format_token(&embedder)),
Code::InvalidRenderTemplateId,
)
} else {
indexing.sort_unstable();
search.sort_unstable();
ResponseError::from_msg(
format!("Wrong template {} after embedder {}.\n Hint: Available fragments are {}.",
format_token(&found),
format_token(&embedder),
indexing.iter().map(|s| format!("`indexingFragments.{s}`")).chain(
search.iter().map(|s| format!("`searchFragments.{s}`"))).collect::<Vec<_>>().join(", ")),
Code::InvalidRenderTemplateId,
)
}
},
ReponseError(response_error) => response_error,
MissingFragment { embedder, kind, mut available } => {
available.sort_unstable();
ResponseError::from_msg(
format!("{} fragment name was not provided.\n Hint: Available {} fragments for embedder {} are {}.",
kind.capitalized(),
kind.as_str(),
format_token(&embedder),
available.iter().map(|s| format!("`{s}`")).collect::<Vec<_>>().join(", ")),
Code::InvalidRenderTemplateId,
)
},
FragmentDoesNotExist { embedder, fragment, kind, mut available } => {
available.sort_unstable();
ResponseError::from_msg(
format!("{} fragment {} does not exist for embedder {}.\n Hint: Available {} fragments are {}.",
kind.capitalized(),
format_token(&fragment),
format_token(&embedder),
kind.as_str(),
available.iter().map(|s| format!("`{s}`")).collect::<Vec<_>>().join(", ")),
Code::InvalidRenderTemplateId,
)
},
LeftOverToken(token) => ResponseError::from_msg(
format!("Leftover token {} after parsing template ID", format_token(&token)),
Code::InvalidRenderTemplateId,
),
MissingChatCompletionTemplate => ResponseError::from_msg(
String::from("Missing chat completion template ID. The only available template is `documentTemplate`."),
Code::InvalidRenderTemplateId,
),
UnknownChatCompletionTemplate(id) => ResponseError::from_msg(
format!("Unknown chat completion template ID {}. The only available template is `documentTemplate`.", format_token(&id)),
Code::InvalidRenderTemplateId,
),
DocumentNotFound(doc_id) => ResponseError::from_msg(
format!("Document with ID `{doc_id}` not found."),
Code::RenderDocumentNotFound,
),
DocumentMustBeMap => ResponseError::from_msg(
String::from("The `doc` field must be a map."),
Code::InvalidRenderInput,
),
BothInlineDocAndDocId => ResponseError::from_msg(
String::from("A document id was provided but adding it to the input would overwrite the `doc` field that you already defined inline."),
Code::InvalidRenderInput,
),
TemplateParsing(err) => ResponseError::from_msg(
format!("Error parsing template: {}", err.parsing_error("input")),
Code::TemplateParsingError,
),
TemplateRendering(err) => ResponseError::from_msg(
format!("Error rendering template: {}", err.rendering_error("input")),
Code::TemplateRenderingError,
),
InputConversion(err) => ResponseError::from_msg(
format!("Error converting input to a liquid object: {err}"),
Code::InvalidRenderInput,
),
ExpectedDotAfterValue(span) => ResponseError::from_msg(
format!("Expected a dot after value, but found {}.", format_span(&span)),
Code::InvalidRenderTemplateId,
),
ExpectedValue(span) => ResponseError::from_msg(
format!("Expected a value, but found {}.", format_span(&span)),
Code::InvalidRenderTemplateId,
),
}
}
}
#[allow(clippy::result_large_err)]
fn parse_template_id_fragment<'a>(
name: Option<Token<'a>>,
kind: FragmentKind,
embedding_config: &IndexEmbeddingConfig,
embedder: Token<'a>,
) -> Result<serde_json::Value, RenderError<'a>> {
let get_available =
[EmbedderOptions::indexing_fragments, EmbedderOptions::search_fragments][kind as usize];
let get_specific =
[EmbedderOptions::indexing_fragment, EmbedderOptions::search_fragment][kind as usize];
let fragment = name.ok_or_else(|| MissingFragment {
embedder: embedder.clone(),
kind,
available: get_available(&embedding_config.config.embedder_options),
})?;
let fragment = get_specific(&embedding_config.config.embedder_options, fragment.value())
.ok_or_else(|| FragmentDoesNotExist {
embedder,
fragment,
kind,
available: get_available(&embedding_config.config.embedder_options),
})?;
Ok(fragment.clone())
}
#[allow(clippy::result_large_err)]
fn parse_template_id<'a>(
index: &Index,
rtxn: &RoTxn,
id: &'a str,
) -> Result<(serde_json::Value, bool), RenderError<'a>> {
let mut input: Span = id.into();
let mut next_part = || -> Result<Option<Token<'_>>, RenderError<'a>> {
if input.is_empty() {
return Ok(None);
}
let (mut remaining, value) = milli::filter_parser::parse_dotted_value_part(input)
.map_err(|_| ExpectedValue(input))?;
if !remaining.is_empty() {
if !remaining.starts_with('.') {
return Err(ExpectedDotAfterValue(remaining));
}
remaining = milli::filter_parser::Slice::slice(&remaining, 1..);
}
input = remaining;
Ok(Some(value))
};
let root = next_part()?.ok_or(EmptyTemplateId)?;
let template = match root.value() {
"embedders" => {
let index_embedding_configs = index.embedding_configs();
let embedding_configs = index_embedding_configs.embedding_configs(rtxn)?;
let get_embedders = || embedding_configs.iter().map(|c| c.name.clone()).collect();
let embedder =
next_part()?.ok_or_else(|| MissingEmbedderName { available: get_embedders() })?;
let embedding_config = embedding_configs
.iter()
.find(|config| config.name == embedder.value())
.ok_or_else(|| EmbedderDoesNotExist {
embedder: embedder.clone(),
available: get_embedders(),
})?;
let get_indexing = || embedding_config.config.embedder_options.indexing_fragments();
let get_search = || embedding_config.config.embedder_options.search_fragments();
let template_kind = next_part()?.ok_or_else(|| MissingTemplateAfterEmbedder {
embedder: embedder.clone(),
indexing: get_indexing(),
search: get_search(),
})?;
match template_kind.value() {
"documentTemplate" if !embedding_config.fragments.as_slice().is_empty() => {
return Err(EmbedderUsesFragments { embedder });
}
"documentTemplate" => (
serde_json::Value::String(embedding_config.config.prompt.template.clone()),
true,
),
"indexingFragments" => (
parse_template_id_fragment(
next_part()?,
FragmentKind::Indexing,
embedding_config,
embedder,
)?,
false,
),
"searchFragments" => (
parse_template_id_fragment(
next_part()?,
FragmentKind::Search,
embedding_config,
embedder,
)?,
false,
),
_ => {
return Err(UnknownTemplatePrefix {
embedder,
found: template_kind,
indexing: get_indexing(),
search: get_search(),
})
}
}
}
"chatCompletions" => {
let template_name = next_part()?.ok_or(MissingChatCompletionTemplate)?;
if template_name.value() != "documentTemplate" {
return Err(UnknownChatCompletionTemplate(template_name));
}
let chat_config = index.chat_config(rtxn)?;
(serde_json::Value::String(chat_config.prompt.template.clone()), true)
}
"" => return Err(EmptyTemplateId),
_ => {
return Err(UnknownTemplateRoot(root));
}
};
if let Some(next) = next_part()? {
return Err(LeftOverToken(next));
}
Ok(template)
}
#[allow(clippy::result_large_err)]
async fn render(index: Index, query: RenderQuery) -> Result<RenderResult, ResponseError> {
let RenderQuery { template, input } = query;
let rtxn = index.read_txn()?;
let (template, fields_available) = match (template.inline, template.id) {
(Some(inline), None) => (inline, true),
(None, Some(id)) => parse_template_id(&index, &rtxn, &id)?,
(Some(_), Some(_)) => return Err(MultipleTemplates.into()),
(None, None) => return Err(MissingTemplate.into()),
};
let fields_already_present = input
.as_ref()
.is_some_and(|i| i.inline.as_ref().is_some_and(|i| i.get("fields").is_some()));
let fields_unused = match template.as_str() {
Some(template) => {
// might be a false positive if fields appear as a non-variable
// it is OK to be over-eager here, it will just translate to more work
!template.contains("fields")
}
None => true, // non-text templates cannot use `fields`
};
let has_inline_doc =
input.as_ref().is_some_and(|i| i.inline.as_ref().is_some_and(|i| i.get("doc").is_some()));
let has_document_id = input.as_ref().is_some_and(|i| i.document_id.is_some());
let has_doc = has_inline_doc || has_document_id;
let insert_fields = fields_available && has_doc && !fields_unused && !fields_already_present;
if has_inline_doc && has_document_id {
return Err(BothInlineDocAndDocId.into());
}
let mut rendered = Value::Null;
if let Some(input) = input {
let inline = input.inline.unwrap_or_default();
let mut object = liquid::to_object(&inline).map_err(InputConversion)?;
let doc = match object.get_mut("doc") {
Some(liquid::model::Value::Object(doc)) => Some(doc),
Some(liquid::model::Value::Nil) => None,
None => None,
_ => return Err(DocumentMustBeMap.into()),
};
if insert_fields {
if let Some(doc) = doc {
let doc = doc.clone();
let fid_map_with_meta = index.fields_ids_map_with_metadata(&rtxn)?;
let fields = OwnedFields::new(&doc, &fid_map_with_meta);
object.insert("fields".into(), fields.to_value());
}
}
if let Some(document_id) = input.document_id {
if insert_fields {
let fid_map_with_meta = index.fields_ids_map_with_metadata(&rtxn)?;
let (document, fields) =
build_doc_fields(&index, &rtxn, &document_id, &fid_map_with_meta)?
.ok_or_else(|| DocumentNotFound(document_id))?;
object.insert("doc".into(), document);
object.insert("fields".into(), fields);
} else {
let fid_map = index.fields_ids_map(&rtxn)?;
let document = build_doc(&index, &rtxn, &document_id, &fid_map)?
.ok_or_else(|| DocumentNotFound(document_id))?;
object.insert("doc".into(), document);
}
}
let json_template = JsonTemplate::new(template.clone()).map_err(TemplateParsing)?;
rendered = json_template.render(&object).map_err(TemplateRendering)?;
}
Ok(RenderResult { template, rendered })
}
#[derive(Debug, Clone, PartialEq, Deserr, ToSchema)]
#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
pub struct RenderQuery {
#[deserr(error = DeserrJsonError<InvalidRenderTemplate>)]
pub template: RenderQueryTemplate,
#[deserr(default, error = DeserrJsonError<InvalidRenderInput>)]
pub input: Option<RenderQueryInput>,
}
#[derive(Debug, Clone, PartialEq, Deserr, ToSchema)]
#[deserr(error = DeserrJsonError<InvalidRenderTemplate>, rename_all = camelCase, deny_unknown_fields)]
pub struct RenderQueryTemplate {
#[deserr(default, error = DeserrJsonError<InvalidRenderTemplateId>)]
pub id: Option<String>,
#[deserr(default, error = DeserrJsonError<InvalidRenderTemplateInline>)]
pub inline: Option<Value>,
}
#[derive(Debug, Clone, Default, PartialEq, Deserr, ToSchema)]
#[deserr(error = DeserrJsonError<InvalidRenderInput>, rename_all = camelCase, deny_unknown_fields)]
pub struct RenderQueryInput {
#[deserr(default, error = DeserrJsonError<InvalidRenderInputDocumentId>)]
pub document_id: Option<String>,
#[deserr(default, error = DeserrJsonError<InvalidRenderInputInline>)]
pub inline: Option<BTreeMap<String, Value>>,
}
#[derive(Debug, Clone, Serialize, PartialEq, ToSchema)]
pub struct RenderResult {
template: Value,
rendered: Value,
}

View File

@@ -0,0 +1,89 @@
use serde_json::json;
use crate::analytics::Aggregate;
use crate::routes::indexes::render::RenderQuery;
#[derive(Default)]
pub struct RenderAggregator {
// requests
total_received: usize,
total_succeeded: usize,
// parameters
template_inline: bool,
template_id: bool,
input_inline: bool,
input_id: bool,
input_omitted: bool,
}
impl RenderAggregator {
#[allow(clippy::field_reassign_with_default)]
pub fn from_query(query: &RenderQuery) -> Self {
let RenderQuery { template, input } = query;
let mut ret = Self::default();
ret.total_received = 1;
ret.template_inline = template.inline.is_some();
ret.template_id = template.id.is_some();
ret.input_inline = input.as_ref().is_some_and(|i| i.inline.is_some());
ret.input_id = input.as_ref().is_some_and(|i| i.document_id.is_some());
ret.input_omitted = input.as_ref().is_none();
ret
}
pub fn succeed(&mut self) {
self.total_succeeded += 1;
}
}
impl Aggregate for RenderAggregator {
fn event_name(&self) -> &'static str {
"Documents Rendered"
}
fn aggregate(mut self: Box<Self>, new: Box<Self>) -> Box<Self> {
self.total_received += new.total_received;
self.total_succeeded += new.total_succeeded;
self.template_inline |= new.template_inline;
self.template_id |= new.template_id;
self.input_inline |= new.input_inline;
self.input_id |= new.input_id;
self.input_omitted |= new.input_omitted;
self
}
fn into_event(self: Box<Self>) -> serde_json::Value {
let Self {
total_received,
total_succeeded,
template_inline,
template_id,
input_inline,
input_id,
input_omitted,
} = *self;
json!({
"requests": {
"total_received": total_received,
"total_succeeded": total_succeeded,
"total_failed": total_received.saturating_sub(total_succeeded) // just to be sure we never panics
},
"template": {
"inline": template_inline,
"id": template_id,
},
"input": {
"inline": input_inline,
"id": input_id,
"omitted": input_omitted
},
})
}
}

View File

@@ -1,7 +1,7 @@
use std::collections::{HashMap, HashSet};
use ::time::format_description::well_known::Rfc3339;
use maplit::{hashmap, hashset};
use maplit::hashmap;
use meilisearch::Opt;
use once_cell::sync::Lazy;
use tempfile::TempDir;
@@ -10,73 +10,103 @@ use time::{Duration, OffsetDateTime};
use crate::common::{default_settings, Server, Value};
use crate::json;
pub static AUTHORIZATIONS: Lazy<HashMap<(&'static str, &'static str), HashSet<&'static str>>> =
Lazy::new(|| {
let authorizations = hashmap! {
("POST", "/multi-search") => hashset!{"search", "*"},
("POST", "/indexes/products/search") => hashset!{"search", "*"},
("GET", "/indexes/products/search") => hashset!{"search", "*"},
("POST", "/indexes/products/documents") => hashset!{"documents.add", "documents.*", "*"},
("GET", "/indexes/products/documents") => hashset!{"documents.get", "documents.*", "*"},
("POST", "/indexes/products/documents/fetch") => hashset!{"documents.get", "documents.*", "*"},
("GET", "/indexes/products/documents/0") => hashset!{"documents.get", "documents.*", "*"},
("DELETE", "/indexes/products/documents/0") => hashset!{"documents.delete", "documents.*", "*"},
("POST", "/indexes/products/documents/delete-batch") => hashset!{"documents.delete", "documents.*", "*"},
("POST", "/indexes/products/documents/delete") => hashset!{"documents.delete", "documents.*", "*"},
("GET", "/tasks") => hashset!{"tasks.get", "tasks.*", "*"},
("DELETE", "/tasks") => hashset!{"tasks.delete", "tasks.*", "*"},
("GET", "/tasks?indexUid=products") => hashset!{"tasks.get", "tasks.*", "*"},
("GET", "/tasks/0") => hashset!{"tasks.get", "tasks.*", "*"},
("PATCH", "/indexes/products/") => hashset!{"indexes.update", "indexes.*", "*"},
("GET", "/indexes/products/") => hashset!{"indexes.get", "indexes.*", "*"},
("DELETE", "/indexes/products/") => hashset!{"indexes.delete", "indexes.*", "*"},
("POST", "/indexes") => hashset!{"indexes.create", "indexes.*", "*"},
("GET", "/indexes") => hashset!{"indexes.get", "indexes.*", "*"},
("POST", "/swap-indexes") => hashset!{"indexes.swap", "indexes.*", "*"},
("GET", "/indexes/products/settings") => hashset!{"settings.get", "settings.*", "*"},
("GET", "/indexes/products/settings/displayed-attributes") => hashset!{"settings.get", "settings.*", "*"},
("GET", "/indexes/products/settings/distinct-attribute") => hashset!{"settings.get", "settings.*", "*"},
("GET", "/indexes/products/settings/filterable-attributes") => hashset!{"settings.get", "settings.*", "*"},
("GET", "/indexes/products/settings/ranking-rules") => hashset!{"settings.get", "settings.*", "*"},
("GET", "/indexes/products/settings/searchable-attributes") => hashset!{"settings.get", "settings.*", "*"},
("GET", "/indexes/products/settings/sortable-attributes") => hashset!{"settings.get", "settings.*", "*"},
("GET", "/indexes/products/settings/stop-words") => hashset!{"settings.get", "settings.*", "*"},
("GET", "/indexes/products/settings/synonyms") => hashset!{"settings.get", "settings.*", "*"},
("DELETE", "/indexes/products/settings") => hashset!{"settings.update", "settings.*", "*"},
("PATCH", "/indexes/products/settings") => hashset!{"settings.update", "settings.*", "*"},
("PATCH", "/indexes/products/settings/typo-tolerance") => hashset!{"settings.update", "settings.*", "*"},
("PUT", "/indexes/products/settings/displayed-attributes") => hashset!{"settings.update", "settings.*", "*"},
("PUT", "/indexes/products/settings/distinct-attribute") => hashset!{"settings.update", "settings.*", "*"},
("PUT", "/indexes/products/settings/filterable-attributes") => hashset!{"settings.update", "settings.*", "*"},
("PUT", "/indexes/products/settings/ranking-rules") => hashset!{"settings.update", "settings.*", "*"},
("PUT", "/indexes/products/settings/searchable-attributes") => hashset!{"settings.update", "settings.*", "*"},
("PUT", "/indexes/products/settings/sortable-attributes") => hashset!{"settings.update", "settings.*", "*"},
("PUT", "/indexes/products/settings/stop-words") => hashset!{"settings.update", "settings.*", "*"},
("PUT", "/indexes/products/settings/synonyms") => hashset!{"settings.update", "settings.*", "*"},
("GET", "/indexes/products/stats") => hashset!{"stats.get", "stats.*", "*"},
("GET", "/stats") => hashset!{"stats.get", "stats.*", "*"},
("POST", "/dumps") => hashset!{"dumps.create", "dumps.*", "*"},
("POST", "/snapshots") => hashset!{"snapshots.create", "snapshots.*", "*"},
("GET", "/version") => hashset!{"version", "*"},
("GET", "/metrics") => hashset!{"metrics.get", "metrics.*", "*"},
("POST", "/logs/stream") => hashset!{"metrics.get", "metrics.*", "*"},
("DELETE", "/logs/stream") => hashset!{"metrics.get", "metrics.*", "*"},
("PATCH", "/keys/mykey/") => hashset!{"keys.update", "*"},
("GET", "/keys/mykey/") => hashset!{"keys.get", "*"},
("DELETE", "/keys/mykey/") => hashset!{"keys.delete", "*"},
("POST", "/keys") => hashset!{"keys.create", "*"},
("GET", "/keys") => hashset!{"keys.get", "*"},
("GET", "/experimental-features") => hashset!{"experimental.get", "*"},
("PATCH", "/experimental-features") => hashset!{"experimental.update", "*"},
("GET", "/network") => hashset!{"network.get", "*"},
("PATCH", "/network") => hashset!{"network.update", "*"},
};
macro_rules! hashset {
( $( $val:tt ),* $(,)? ) => {{
let mut set: HashSet<&'static [&'static str]> = HashSet::new();
$(
hashset!(@insert set, $val);
)*
set
}};
authorizations
});
// Match array-like input: ["a", "b"]
(@insert $set:ident, [ $($elem:literal),* ]) => {{
const ITEM: &[&str] = &[$($elem),*];
$set.insert(ITEM);
}};
// Match single literal: "a"
(@insert $set:ident, $val:literal) => {{
const ITEM: &[&str] = &[$val];
$set.insert(ITEM);
}};
}
#[allow(clippy::type_complexity)]
pub static AUTHORIZATIONS: Lazy<
HashMap<(&'static str, &'static str), HashSet<&'static [&'static str]>>,
> = Lazy::new(|| {
let authorizations = hashmap! {
("POST", "/multi-search") => hashset!{"search", "*"},
("POST", "/indexes/products/search") => hashset!{"search", "*"},
("GET", "/indexes/products/search") => hashset!{"search", "*"},
("POST", "/indexes/products/documents") => hashset!{"documents.add", "documents.*", "*"},
("GET", "/indexes/products/documents") => hashset!{"documents.get", "documents.*", "*"},
("POST", "/indexes/products/documents/fetch") => hashset!{"documents.get", "documents.*", "*"},
("GET", "/indexes/products/documents/0") => hashset!{"documents.get", "documents.*", "*"},
("DELETE", "/indexes/products/documents/0") => hashset!{"documents.delete", "documents.*", "*"},
("POST", "/indexes/products/documents/delete-batch") => hashset!{"documents.delete", "documents.*", "*"},
("POST", "/indexes/products/documents/delete") => hashset!{"documents.delete", "documents.*", "*"},
("POST", "/indexes/products/render") => hashset!{["settings.get", "documents.get"], ["documents.*", "settings.get"], ["settings.*", "documents.get"], "*"},
("GET", "/tasks") => hashset!{"tasks.get", "tasks.*", "*"},
("DELETE", "/tasks") => hashset!{"tasks.delete", "tasks.*", "*"},
("GET", "/tasks?indexUid=products") => hashset!{"tasks.get", "tasks.*", "*"},
("GET", "/tasks/0") => hashset!{"tasks.get", "tasks.*", "*"},
("PATCH", "/indexes/products/") => hashset!{"indexes.update", "indexes.*", "*"},
("GET", "/indexes/products/") => hashset!{"indexes.get", "indexes.*", "*"},
("DELETE", "/indexes/products/") => hashset!{"indexes.delete", "indexes.*", "*"},
("POST", "/indexes") => hashset!{"indexes.create", "indexes.*", "*"},
("GET", "/indexes") => hashset!{"indexes.get", "indexes.*", "*"},
("POST", "/swap-indexes") => hashset!{"indexes.swap", "indexes.*", "*"},
("GET", "/indexes/products/settings") => hashset!{"settings.get", "settings.*", "*"},
("GET", "/indexes/products/settings/displayed-attributes") => hashset!{"settings.get", "settings.*", "*"},
("GET", "/indexes/products/settings/distinct-attribute") => hashset!{"settings.get", "settings.*", "*"},
("GET", "/indexes/products/settings/filterable-attributes") => hashset!{"settings.get", "settings.*", "*"},
("GET", "/indexes/products/settings/ranking-rules") => hashset!{"settings.get", "settings.*", "*"},
("GET", "/indexes/products/settings/searchable-attributes") => hashset!{"settings.get", "settings.*", "*"},
("GET", "/indexes/products/settings/sortable-attributes") => hashset!{"settings.get", "settings.*", "*"},
("GET", "/indexes/products/settings/stop-words") => hashset!{"settings.get", "settings.*", "*"},
("GET", "/indexes/products/settings/synonyms") => hashset!{"settings.get", "settings.*", "*"},
("DELETE", "/indexes/products/settings") => hashset!{"settings.update", "settings.*", "*"},
("PATCH", "/indexes/products/settings") => hashset!{"settings.update", "settings.*", "*"},
("PATCH", "/indexes/products/settings/typo-tolerance") => hashset!{"settings.update", "settings.*", "*"},
("PUT", "/indexes/products/settings/displayed-attributes") => hashset!{"settings.update", "settings.*", "*"},
("PUT", "/indexes/products/settings/distinct-attribute") => hashset!{"settings.update", "settings.*", "*"},
("PUT", "/indexes/products/settings/filterable-attributes") => hashset!{"settings.update", "settings.*", "*"},
("PUT", "/indexes/products/settings/ranking-rules") => hashset!{"settings.update", "settings.*", "*"},
("PUT", "/indexes/products/settings/searchable-attributes") => hashset!{"settings.update", "settings.*", "*"},
("PUT", "/indexes/products/settings/sortable-attributes") => hashset!{"settings.update", "settings.*", "*"},
("PUT", "/indexes/products/settings/stop-words") => hashset!{"settings.update", "settings.*", "*"},
("PUT", "/indexes/products/settings/synonyms") => hashset!{"settings.update", "settings.*", "*"},
("GET", "/indexes/products/stats") => hashset!{"stats.get", "stats.*", "*"},
("GET", "/stats") => hashset!{"stats.get", "stats.*", "*"},
("POST", "/dumps") => hashset!{"dumps.create", "dumps.*", "*"},
("POST", "/snapshots") => hashset!{"snapshots.create", "snapshots.*", "*"},
("GET", "/version") => hashset!{"version", "*"},
("GET", "/metrics") => hashset!{"metrics.get", "metrics.*", "*"},
("POST", "/logs/stream") => hashset!{"metrics.get", "metrics.*", "*"},
("DELETE", "/logs/stream") => hashset!{"metrics.get", "metrics.*", "*"},
("PATCH", "/keys/mykey/") => hashset!{"keys.update", "*"},
("GET", "/keys/mykey/") => hashset!{"keys.get", "*"},
("DELETE", "/keys/mykey/") => hashset!{"keys.delete", "*"},
("POST", "/keys") => hashset!{"keys.create", "*"},
("GET", "/keys") => hashset!{"keys.get", "*"},
("GET", "/experimental-features") => hashset!{"experimental.get", "*"},
("PATCH", "/experimental-features") => hashset!{"experimental.update", "*"},
("GET", "/network") => hashset!{"network.get", "*"},
("PATCH", "/network") => hashset!{"network.update", "*"},
};
authorizations
});
pub static ALL_ACTIONS: Lazy<HashSet<&'static str>> = Lazy::new(|| {
AUTHORIZATIONS.values().cloned().reduce(|l, r| l.union(&r).cloned().collect()).unwrap()
AUTHORIZATIONS
.values()
.flat_map(|v| v.iter())
.flat_map(|v| v.iter())
.copied()
.collect::<HashSet<_>>()
});
static INVALID_RESPONSE: Lazy<Value> = Lazy::new(|| {
@@ -164,13 +194,14 @@ async fn error_access_unauthorized_index() {
async fn error_access_unauthorized_action() {
let mut server = Server::new_auth().await;
for ((method, route), action) in AUTHORIZATIONS.iter() {
for ((method, route), actions) in AUTHORIZATIONS.iter() {
// create a new API key letting only the needed action.
server.use_api_key(MASTER_KEY);
let actions = actions.iter().flat_map(|s| s.iter()).copied().collect::<HashSet<_>>();
let content = json!({
"indexes": ["products"],
"actions": ALL_ACTIONS.difference(action).collect::<Vec<_>>(),
"actions": ALL_ACTIONS.difference(&actions).collect::<Vec<_>>(),
"expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(),
});
@@ -194,7 +225,7 @@ async fn access_authorized_master_key() {
server.use_api_key(MASTER_KEY);
// master key must have access to all routes.
for ((method, route), _) in AUTHORIZATIONS.iter() {
for (method, route) in AUTHORIZATIONS.keys() {
let (response, code) = server.dummy_request(method, route).await;
assert_ne!(response, INVALID_RESPONSE.clone(), "on route: {:?} - {:?}", method, route);
@@ -208,13 +239,13 @@ async fn access_authorized_restricted_index() {
let enable_metrics = Opt { experimental_enable_metrics: true, ..default_settings(dir.path()) };
let mut server = Server::new_auth_with_options(enable_metrics, dir).await;
for ((method, route), actions) in AUTHORIZATIONS.iter() {
for action in actions {
for actions in actions {
// create a new API key letting only the needed action.
server.use_api_key(MASTER_KEY);
let content = json!({
"indexes": ["products"],
"actions": [action],
"actions": actions,
"expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(),
});
@@ -232,20 +263,20 @@ async fn access_authorized_restricted_index() {
assert_eq!(
response,
INVALID_METRICS_RESPONSE.clone(),
"on route: {:?} - {:?} with action: {:?}",
"on route: {:?} - {:?} with actions: {:?}",
method,
route,
action
actions
);
assert_eq!(code, 403);
} else {
assert_ne!(
response,
INVALID_RESPONSE.clone(),
"on route: {:?} - {:?} with action: {:?}",
"on route: {:?} - {:?} with actions: {:?}",
method,
route,
action
actions
);
assert_ne!(code, 403);
}
@@ -253,18 +284,74 @@ async fn access_authorized_restricted_index() {
}
}
#[actix_rt::test]
async fn unauthorized_partial_actions() {
let mut server = Server::new_auth().await;
server.use_admin_key(MASTER_KEY).await;
// create index `products`
let index = server.index("products");
let (response, code) = index.create(Some("id")).await;
assert_eq!(202, code, "{:?}", &response);
let task_id = response["taskUid"].as_u64().unwrap();
server.wait_task(task_id).await.succeeded();
// When multiple actions are necessary, the server mustn't accept any combination with one action missing.
for ((method, route), actions) in AUTHORIZATIONS.iter() {
for actions in actions {
if 2 <= actions.len() {
for excluded_action in *actions {
// create a new API key letting all actions except one.
server.use_api_key(MASTER_KEY);
let actions = actions
.iter()
.filter(|&a| a != excluded_action)
.copied()
.collect::<HashSet<_>>();
let content = json!({
"indexes": ["products"],
"actions": actions,
"expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(),
});
let (response, code) = server.add_api_key(content).await;
assert_eq!(201, code, "{:?}", &response);
assert!(response["key"].is_string());
let key = response["key"].as_str().unwrap();
server.use_api_key(key);
let (mut response, code) = server.dummy_request(method, route).await;
response["message"] = serde_json::json!(null);
assert_eq!(
response,
INVALID_RESPONSE.clone(),
"on route: {:?} - {:?} with actions: {:?}",
method,
route,
actions
);
assert_eq!(code, 403, "{:?}", &response);
}
}
}
}
}
#[actix_rt::test]
async fn access_authorized_no_index_restriction() {
let mut server = Server::new_auth().await;
for ((method, route), actions) in AUTHORIZATIONS.iter() {
for action in actions {
for actions in actions {
// create a new API key letting only the needed action.
server.use_api_key(MASTER_KEY);
let content = json!({
"indexes": ["*"],
"actions": [action],
"actions": actions,
"expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(),
});
@@ -280,12 +367,16 @@ async fn access_authorized_no_index_restriction() {
assert_ne!(
response,
INVALID_RESPONSE.clone(),
"on route: {:?} - {:?} with action: {:?}",
"on route: {:?} - {:?} with actions: {:?}",
method,
route,
action
actions
);
assert_ne!(
code, 403,
"on route: {:?} - {:?} with action: {:?}",
method, route, actions
);
assert_ne!(code, 403, "on route: {:?} - {:?} with action: {:?}", method, route, action);
}
}
}
@@ -723,10 +814,17 @@ async fn error_creating_index_without_action() {
server.use_api_key(MASTER_KEY);
// create key with access on all indexes.
let create_index_actions = AUTHORIZATIONS
.get(&("POST", "/indexes"))
.unwrap()
.iter()
.flat_map(|s| s.iter())
.cloned()
.collect::<HashSet<_>>();
let content = json!({
"indexes": ["*"],
// Give all action but the ones allowing to create an index.
"actions": ALL_ACTIONS.iter().cloned().filter(|a| !AUTHORIZATIONS.get(&("POST","/indexes")).unwrap().contains(a)).collect::<Vec<_>>(),
"actions": ALL_ACTIONS.iter().cloned().filter(|a| !create_index_actions.contains(a)).collect::<Vec<_>>(),
"expiresAt": "2050-11-13T00:00:00Z"
});
let (response, code) = server.add_api_key(content).await;

View File

@@ -1,4 +1,4 @@
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use ::time::format_description::well_known::Rfc3339;
use maplit::hashmap;
@@ -467,6 +467,7 @@ async fn error_access_forbidden_routes() {
server.use_api_key(&web_token);
for ((method, route), actions) in AUTHORIZATIONS.iter() {
let actions = actions.iter().flat_map(|s| s.iter()).copied().collect::<HashSet<_>>();
if !actions.contains("search") {
let (mut response, code) = server.dummy_request(method, route).await;
response["message"] = serde_json::json!(null);

View File

@@ -474,6 +474,11 @@ impl<State> Index<'_, State> {
self.service.get(url).await
}
pub async fn render(&self, query: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/render", urlencode(self.uid.as_ref()));
self.service.post_encoded(url, query, self.encoder).await
}
pub async fn settings(&self) -> (Value, StatusCode) {
let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
self.service.get(url).await

View File

@@ -2,4 +2,5 @@ mod add_documents;
mod delete_documents;
mod errors;
mod get_documents;
mod render_documents;
mod update_documents;

View File

@@ -0,0 +1,733 @@
use crate::common::{shared_empty_index, shared_index_for_fragments, Server};
use crate::json;
use meili_snap::{json_string, snapshot};
#[actix_rt::test]
async fn empty_id() {
let index = shared_index_for_fragments().await;
let (value, code) = index.render(json! {{ "template": { "id": "" }}}).await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "The template ID is empty.\n Hint: Valid prefixes are `embedders` or `chatCompletions`.",
"code": "invalid_render_template_id",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template_id"
}
"#);
}
#[actix_rt::test]
async fn wrong_id_prefix() {
let index = shared_index_for_fragments().await;
let (value, code) = index.render(json! {{ "template": { "id": "wrong.disregarded" }}}).await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "Template ID must start with `embedders` or `chatCompletions`, but found `wrong` (cols 1:6).",
"code": "invalid_render_template_id",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template_id"
}
"#);
}
#[actix_rt::test]
async fn missing_embedder() {
let index = shared_index_for_fragments().await;
let (value, code) = index.render(json! {{ "template": { "id": "embedders" }}}).await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "Template ID configured with `embedders` but no embedder name provided.\n Hint: Available embedders are `rest`.",
"code": "invalid_render_template_id",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template_id"
}
"#);
}
#[actix_rt::test]
async fn wrong_embedder() {
let index = shared_index_for_fragments().await;
let (value, code) =
index.render(json! {{ "template": { "id": "embedders.wrong.disregarded" }}}).await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "Embedder `wrong` (cols 11:16) does not exist.\n Hint: Available embedders are `rest`.",
"code": "invalid_render_template_id",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template_id"
}
"#);
}
#[actix_rt::test]
async fn missing_template_kind() {
let index = shared_index_for_fragments().await;
let (value, code) = index.render(json! {{ "template": { "id": "embedders.rest" }}}).await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "Template ID configured with embedder `rest` (cols 11:15) but no template kind provided.\n Hint: Available fragments are `indexingFragments.basic`, `indexingFragments.withBreed`, `searchFragments.justBreed`, `searchFragments.justName`, `searchFragments.query`.",
"code": "invalid_render_template_id",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template_id"
}
"#);
}
#[actix_rt::test]
async fn wrong_template_kind() {
let index = shared_index_for_fragments().await;
let (value, code) =
index.render(json! {{ "template": { "id": "embedders.rest.wrong.disregarded" }}}).await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "Wrong template `wrong` (cols 16:21) after embedder `rest` (cols 11:15).\n Hint: Available fragments are `indexingFragments.basic`, `indexingFragments.withBreed`, `searchFragments.justBreed`, `searchFragments.justName`, `searchFragments.query`.",
"code": "invalid_render_template_id",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template_id"
}
"#);
}
#[actix_rt::test]
async fn document_template_on_fragmented_index() {
let index = shared_index_for_fragments().await;
let (value, code) =
index.render(json! {{ "template": { "id": "embedders.rest.documentTemplate" }}}).await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "Requested document template for embedder `rest` (cols 11:15) but it uses fragments.\n Hint: Use `indexingFragments` or `searchFragments` instead.",
"code": "invalid_render_template_id",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template_id"
}
"#);
}
#[actix_rt::test]
async fn missing_fragment_name() {
let index = shared_index_for_fragments().await;
let (value, code) =
index.render(json! {{ "template": { "id": "embedders.rest.indexingFragments" }}}).await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "Indexing fragment name was not provided.\n Hint: Available indexing fragments for embedder `rest` (cols 11:15) are `basic`, `withBreed`.",
"code": "invalid_render_template_id",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template_id"
}
"#);
let (value, code) =
index.render(json! {{ "template": { "id": "embedders.rest.searchFragments" }}}).await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "Search fragment name was not provided.\n Hint: Available search fragments for embedder `rest` (cols 11:15) are `justBreed`, `justName`, `query`.",
"code": "invalid_render_template_id",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template_id"
}
"#);
}
#[actix_rt::test]
async fn wrong_fragment_name() {
let index = shared_index_for_fragments().await;
let (value, code) = index
.render(json! {{ "template": { "id": "embedders.rest.indexingFragments.wrong" }}})
.await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "Indexing fragment `wrong` (cols 34:39) does not exist for embedder `rest` (cols 11:15).\n Hint: Available indexing fragments are `basic`, `withBreed`.",
"code": "invalid_render_template_id",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template_id"
}
"#);
let (value, code) =
index.render(json! {{ "template": { "id": "embedders.rest.searchFragments.wrong" }}}).await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "Search fragment `wrong` (cols 32:37) does not exist for embedder `rest` (cols 11:15).\n Hint: Available search fragments are `justBreed`, `justName`, `query`.",
"code": "invalid_render_template_id",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template_id"
}
"#);
}
#[actix_rt::test]
async fn leftover_tokens() {
let index = shared_index_for_fragments().await;
let (value, code) = index
.render(
json! {{ "template": { "id": "embedders.rest.indexingFragments.withBreed.leftover" }}},
)
.await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "Leftover token `leftover` (cols 44:52) after parsing template ID",
"code": "invalid_render_template_id",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template_id"
}
"#);
let (value, code) = index
.render(json! {{"template": { "id": "embedders.rest.searchFragments.justBreed.leftover" }}})
.await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "Leftover token `leftover` (cols 42:50) after parsing template ID",
"code": "invalid_render_template_id",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template_id"
}
"#);
let (value, code) = index
.render(json! {{"template": { "id": "chatCompletions.documentTemplate.leftover" }}})
.await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "Leftover token `leftover` (cols 34:42) after parsing template ID",
"code": "invalid_render_template_id",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template_id"
}
"#);
}
#[actix_rt::test]
async fn fragment_retrieval() {
let index = shared_index_for_fragments().await;
let (value, code) = index
.render(json! {{ "template": { "id": "embedders.rest.indexingFragments.withBreed" }}})
.await;
snapshot!(code, @"200 OK");
snapshot!(value, @r#"
{
"template": "{{ doc.name }} is a {{ doc.breed }}",
"rendered": null
}
"#);
let (value, code) = index
.render(json! {{ "template": { "id": "embedders.rest.searchFragments.justBreed" }}})
.await;
snapshot!(code, @"200 OK");
snapshot!(value, @r#"
{
"template": "It's a {{ media.breed }}",
"rendered": null
}
"#);
}
#[actix_rt::test]
async fn missing_chat_completions_template() {
let index = shared_index_for_fragments().await;
let (value, code) = index.render(json! {{ "template": { "id": "chatCompletions" }}}).await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "Missing chat completion template ID. The only available template is `documentTemplate`.",
"code": "invalid_render_template_id",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template_id"
}
"#);
}
#[actix_rt::test]
async fn wrong_chat_completions_template() {
let index = shared_index_for_fragments().await;
let (value, code) =
index.render(json! {{ "template": { "id": "chatCompletions.wrong" }}}).await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "Unknown chat completion template ID `wrong` (cols 17:22). The only available template is `documentTemplate`.",
"code": "invalid_render_template_id",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template_id"
}
"#);
}
#[actix_rt::test]
async fn chat_completions_template_retrieval() {
let index = shared_index_for_fragments().await;
let (value, code) =
index.render(json! {{ "template": { "id": "chatCompletions.documentTemplate" }}}).await;
snapshot!(code, @"200 OK");
snapshot!(value, @r#"
{
"template": "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}",
"rendered": null
}
"#);
}
#[actix_rt::test]
async fn retrieve_document_template() {
let server = Server::new_shared();
let index = server.unique_index();
let (response, code) = index
.update_settings(json!(
{
"embedders": {
"doggo_embedder": {
"source": "huggingFace",
"model": "sentence-transformers/all-MiniLM-L6-v2",
"revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e",
"documentTemplate": "This is a document template {{doc.doggo}}",
}
}
}
))
.await;
snapshot!(code, @"202 Accepted");
server.wait_task(response["taskUid"].as_u64().unwrap()).await;
let (value, code) = index
.render(json! {{ "template": { "id": "embedders.doggo_embedder.documentTemplate" }}})
.await;
snapshot!(code, @"200 OK");
snapshot!(value, @r#"
{
"template": "This is a document template {{doc.doggo}}",
"rendered": null
}
"#);
}
#[actix_rt::test]
async fn render_document_kefir() {
let index = shared_index_for_fragments().await;
let (value, code) = index
.render(json! {{
"template": { "id": "embedders.rest.indexingFragments.basic" },
"input": { "documentId": "0" },
}})
.await;
snapshot!(code, @"200 OK");
snapshot!(value, @r#"
{
"template": "{{ doc.name }} is a dog",
"rendered": "kefir is a dog"
}
"#);
let (value, code) = index
.render(json! {{
"template": { "id": "embedders.rest.indexingFragments.withBreed" },
"input": { "documentId": "0" },
}})
.await;
snapshot!(code, @"400 Bad Request");
snapshot!(json_string!(value, { ".message" => "[ignored]" }), @r#"
{
"message": "[ignored]",
"code": "template_rendering_error",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#template_rendering_error"
}
"#);
}
#[actix_rt::test]
async fn render_inline_document_iko() {
let index = shared_index_for_fragments().await;
let (value, code) = index
.render(json! {{
"template": { "id": "embedders.rest.indexingFragments.basic" },
"input": { "inline": { "doc": { "name": "iko", "breed": "jack russell" } } },
}})
.await;
snapshot!(code, @"200 OK");
snapshot!(value, @r#"
{
"template": "{{ doc.name }} is a dog",
"rendered": "iko is a dog"
}
"#);
let (value, code) = index
.render(json! {{
"template": { "id": "embedders.rest.indexingFragments.withBreed" },
"input": { "inline": { "doc": { "name": "iko", "breed": "jack russell" } } },
}})
.await;
snapshot!(code, @"200 OK");
snapshot!(value, @r#"
{
"template": "{{ doc.name }} is a {{ doc.breed }}",
"rendered": "iko is a jack russell"
}
"#);
let (value, code) = index
.render(json! {{
"template": { "id": "embedders.rest.searchFragments.justBreed" },
"input": { "inline": { "media": { "name": "iko", "breed": "jack russell" } } },
}})
.await;
snapshot!(code, @"200 OK");
snapshot!(value, @r#"
{
"template": "It's a {{ media.breed }}",
"rendered": "It's a jack russell"
}
"#);
}
#[actix_rt::test]
async fn render_doc_not_object() {
let index = shared_empty_index().await;
let (value, code) = index
.render(json! {{
"template": { "inline": "{{ doc }}" },
"input": { "inline": { "doc": "that's not an object, that's a string" } },
}})
.await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "The `doc` field must be a map.",
"code": "invalid_render_input",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_input"
}
"#);
let (value, code) = index
.render(json! {{
"template": { "inline": "default" },
"input": { "inline": { "doc": null } },
}})
.await;
snapshot!(code, @"200 OK");
snapshot!(value, @r#"
{
"template": "default",
"rendered": "default"
}
"#);
}
#[actix_rt::test]
async fn chat_completions() {
let index = shared_index_for_fragments().await;
let (value, code) = index
.render(json! {{
"template": { "id": "chatCompletions.documentTemplate" },
"input": { "documentId": "0" },
}})
.await;
snapshot!(code, @"200 OK");
snapshot!(value, @r#"
{
"template": "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}",
"rendered": "id: 0\nname: kefir\n"
}
"#);
let (value, code) = index
.render(json! {{
"template": { "id": "chatCompletions.documentTemplate" },
"input": { "inline": { "doc": { "name": "iko", "breed": "jack russell" } } },
}})
.await;
snapshot!(code, @"200 OK");
snapshot!(value, @r#"
{
"template": "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}",
"rendered": "name: iko\nbreed: jack russell\n"
}
"#);
}
#[actix_rt::test]
async fn both_document_id_and_inline() {
let index = shared_index_for_fragments().await;
let (value, code) = index
.render(json! {{
"template": { "inline": "{{ doc.name }} compared to {{ media.name }}" },
"input": { "documentId": "0", "inline": { "media": { "name": "iko" } } },
}})
.await;
snapshot!(code, @"200 OK");
snapshot!(value, @r#"
{
"template": "{{ doc.name }} compared to {{ media.name }}",
"rendered": "kefir compared to iko"
}
"#);
}
#[actix_rt::test]
async fn multiple_templates_or_docs() {
let index = shared_index_for_fragments().await;
let (value, code) = index
.render(json! {{
"template": { "id": "whatever", "inline": "whatever" }
}})
.await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "Cannot provide both an inline template and a template ID.",
"code": "invalid_render_template",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template"
}
"#);
let (value, code) = index
.render(json! {{
"template": { "inline": "whatever" },
"input": { "documentId": "0", "inline": { "doc": { "name": "iko" } } }
}})
.await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "A document id was provided but adding it to the input would overwrite the `doc` field that you already defined inline.",
"code": "invalid_render_input",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_input"
}
"#);
}
#[actix_rt::test]
async fn document_not_found() {
let index = shared_index_for_fragments().await;
let (value, code) = index
.render(json! {{
"template": { "id": "embedders.rest.indexingFragments.basic" },
"input": { "documentId": "9999" }
}})
.await;
snapshot!(code, @"404 Not Found");
snapshot!(value, @r#"
{
"message": "Document with ID `9999` not found.",
"code": "render_document_not_found",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#render_document_not_found"
}
"#);
}
#[actix_rt::test]
async fn bad_template() {
let index = shared_index_for_fragments().await;
let (value, code) = index
.render(json! {{
"template": { "inline": "{{ doc.name" },
"input": { "documentId": "0" }
}})
.await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "Error parsing template: error while parsing template: liquid: --> 1:4\n |\n1 | {{ doc.name\n | ^---\n |\n = expected Literal\n",
"code": "template_parsing_error",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#template_parsing_error"
}
"#);
}
#[actix_rt::test]
async fn inline_nested() {
let index = shared_index_for_fragments().await;
let (value, code) = index
.render(json! {{
"template": { "inline": "{{ doc.name }} is a {{ doc.breed.name }} ({{ doc.breed.kind }})" },
"input": { "inline": { "doc": { "name": "iko", "breed": { "name": "jack russell", "kind": "terrier" } } } }
}})
.await;
snapshot!(code, @"200 OK");
snapshot!(value, @r#"
{
"template": "{{ doc.name }} is a {{ doc.breed.name }} ({{ doc.breed.kind }})",
"rendered": "iko is a jack russell (terrier)"
}
"#);
}
#[actix_rt::test]
async fn embedder_document_template() {
let (_mock, setting) = crate::vector::rest::create_mock().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index
.update_settings(json!({
"embedders": {
"rest": setting,
},
}))
.await;
snapshot!(code, @"202 Accepted");
server.wait_task(response.uid()).await.succeeded();
let documents = json!([
{"id": 0, "name": "kefir"},
]);
let (value, code) = index.add_documents(documents, None).await;
snapshot!(code, @"202 Accepted");
server.wait_task(value.uid()).await.succeeded();
let (value, code) = index
.render(json! {{
"template": { "id": "embedders.rest.documentTemplate" },
"input": { "documentId": "0" }
}})
.await;
snapshot!(code, @"200 OK");
snapshot!(value, @r#"
{
"template": "{{doc.name}}",
"rendered": "kefir"
}
"#);
let (value, code) =
index.render(json! {{ "template": { "id": "embedders.rest.wrong.disregarded" }}}).await;
snapshot!(code, @"400 Bad Request");
snapshot!(value, @r#"
{
"message": "Wrong template `wrong` (cols 16:21) after embedder `rest` (cols 11:15).\n Hint: Available template: `documentTemplate`.",
"code": "invalid_render_template_id",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_render_template_id"
}
"#);
}
#[actix_rt::test]
async fn ugly_embedder_and_fragment_names() {
let server = Server::new().await;
let index = server.unique_index();
let (_response, code) = server.set_features(json!({"multimodal": true})).await;
snapshot!(code, @"200 OK");
// Set up a mock server for the embedder
let mock_server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.and(wiremock::matchers::path("/"))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_json(json!({
"data": [0.1, 0.2, 0.3]
})))
.mount(&mock_server)
.await;
// Create an embedder with an ugly name containing quotes and special characters
let (response, code) = index
.update_settings(json!({
"embedders": {
"Open AI \"3.1\"": {
"source": "rest",
"url": mock_server.uri(),
"dimensions": 3,
"request": "{{fragment}}",
"response": {
"data": "{{embedding}}"
},
"indexingFragments": {
"ugly fragment \"name\".": {"value": "{{ doc.name }} processed by AI"}
},
"searchFragments": {
"search with [brackets]": {"value": "It's a {{ media.breed }}"}
}
},
},
}))
.await;
snapshot!(code, @"202 Accepted");
server.wait_task(response.uid()).await.succeeded();
// Test retrieving indexing fragment template with ugly name
let (value, code) = index
.render(json! {{
"template": { "id": r#"embedders."Open AI \"3.1\"".indexingFragments."ugly fragment \"name\".""# },
}})
.await;
snapshot!(code, @"200 OK");
snapshot!(value, @r#"
{
"template": "{{ doc.name }} processed by AI",
"rendered": null
}
"#);
// Test retrieving search fragment template with ugly name
let (value, code) = index
.render(json! {{
"template": { "id": r#"embedders."Open AI \"3.1\"".searchFragments."search with [brackets]""# },
}})
.await;
snapshot!(code, @"200 OK");
snapshot!(value, @r#"
{
"template": "It's a {{ media.breed }}",
"rendered": null
}
"#);
// Test quoting normal parts of the template ID
let (value, code) = index
.render(json! {{
"template": { "id": r#""embedders"."Open AI \"3.1\""."indexingFragments"."ugly fragment \"name\".""# }
}})
.await;
snapshot!(code, @"200 OK");
snapshot!(value, @r#"
{
"template": "{{ doc.name }} processed by AI",
"rendered": null
}
"#);
}

View File

@@ -8,6 +8,7 @@ use crate::common::{
shared_index_with_nested_documents, Server, DOCUMENTS, NESTED_DOCUMENTS,
};
use crate::json;
use crate::vector::rest::create_mock;
#[actix_rt::test]
async fn search_with_filter_string_notation() {
@@ -990,8 +991,8 @@ async fn vector_filter_document_template_but_fragments_used() {
#[actix_rt::test]
async fn vector_filter_document_template() {
let (_mock, setting) = crate::vector::create_mock().await;
let server = crate::vector::get_server_vector().await;
let (_mock, setting) = create_mock().await;
let server = Server::new().await;
let index = server.index("doggo");
let (_response, code) = server.set_features(json!({"multimodal": true})).await;

View File

@@ -3,7 +3,7 @@ mod fragments;
#[cfg(feature = "test-ollama")]
mod ollama;
mod openai;
mod rest;
pub mod rest;
mod settings;
use std::str::FromStr;
@@ -14,11 +14,6 @@ use meilisearch::option::MaxThreads;
use crate::common::index::Index;
use crate::common::{default_settings, GetAllDocumentsOptions, Server};
use crate::json;
pub use rest::create_mock;
pub async fn get_server_vector() -> Server {
Server::new().await
}
#[actix_rt::test]
async fn add_remove_user_provided() {

View File

@@ -4,9 +4,8 @@ use std::env::VarError;
use meili_snap::{json_string, snapshot};
use crate::common::{GetAllDocumentsOptions, Value};
use crate::common::{GetAllDocumentsOptions, Server, Value};
use crate::json;
use crate::vector::get_server_vector;
pub enum Endpoint {
/// Deprecated, undocumented endpoint
@@ -92,7 +91,7 @@ async fn test_both_apis() {
return;
};
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");

View File

@@ -7,9 +7,8 @@ use meili_snap::{json_string, snapshot};
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, Request, ResponseTemplate};
use crate::common::{GetAllDocumentsOptions, Value};
use crate::common::{GetAllDocumentsOptions, Server, Value};
use crate::json;
use crate::vector::get_server_vector;
#[derive(serde::Deserialize)]
struct OpenAiResponses(BTreeMap<String, OpenAiResponse>);
@@ -349,7 +348,7 @@ async fn create_slow_mock() -> (&'static MockServer, Value) {
#[actix_rt::test]
async fn it_works() {
let (_mock, setting) = create_mock().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index
@@ -583,7 +582,7 @@ async fn it_works() {
#[actix_rt::test]
async fn tokenize_long_text() {
let (_mock, setting) = create_mock_tokenized().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index
@@ -646,7 +645,7 @@ async fn tokenize_long_text() {
#[actix_rt::test]
async fn bad_api_key() {
let (_mock, mut setting) = create_mock().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let documents = json!([
@@ -794,7 +793,7 @@ async fn bad_api_key() {
#[actix_rt::test]
async fn bad_model() {
let (_mock, mut setting) = create_mock().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let documents = json!([
@@ -872,7 +871,7 @@ async fn bad_model() {
#[actix_rt::test]
async fn bad_dimensions() {
let (_mock, mut setting) = create_mock().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let documents = json!([
@@ -971,7 +970,7 @@ async fn bad_dimensions() {
#[actix_rt::test]
async fn smaller_dimensions() {
let (_mock, setting) = create_mock_dimensions().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index
@@ -1203,7 +1202,7 @@ async fn smaller_dimensions() {
#[actix_rt::test]
async fn small_embedding_model() {
let (_mock, setting) = create_mock_small_embedding_model().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index
@@ -1434,7 +1433,7 @@ async fn small_embedding_model() {
#[actix_rt::test]
async fn legacy_embedding_model() {
let (_mock, setting) = create_mock_legacy_embedding_model().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index
@@ -1666,7 +1665,7 @@ async fn legacy_embedding_model() {
#[actix_rt::test]
async fn it_still_works() {
let (_mock, setting) = create_fallible_mock().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index
@@ -1898,7 +1897,7 @@ async fn it_still_works() {
#[actix_rt::test]
async fn timeout() {
let (_mock, setting) = create_slow_mock().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index

View File

@@ -8,9 +8,9 @@ use tokio::sync::mpsc;
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, Request, ResponseTemplate};
use crate::common::Value;
use crate::common::{Server, Value};
use crate::json;
use crate::vector::{get_server_vector, GetAllDocumentsOptions};
use crate::vector::GetAllDocumentsOptions;
pub async fn create_mock() -> (&'static MockServer, Value) {
let mock_server = Box::leak(Box::new(MockServer::start().await));
@@ -395,7 +395,7 @@ async fn dummy_testing_the_mock() {
async fn bad_request() {
let (mock, _setting) = create_mock().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
// No placeholder string appear in the template
@@ -631,7 +631,7 @@ async fn bad_request() {
async fn bad_response() {
let (mock, _setting) = create_mock().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
// No placeholder string appear in the template
@@ -907,7 +907,7 @@ async fn bad_response() {
async fn bad_settings() {
let (mock, _setting) = create_mock().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index
@@ -1079,7 +1079,7 @@ async fn bad_settings() {
#[actix_rt::test]
async fn add_vector_and_user_provided() {
let (_mock, setting) = create_mock().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index
@@ -1185,7 +1185,7 @@ async fn add_vector_and_user_provided() {
#[actix_rt::test]
async fn server_returns_bad_request() {
let (mock, _setting) = create_mock_multiple().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index
@@ -1301,7 +1301,7 @@ async fn server_returns_bad_request() {
#[actix_rt::test]
async fn server_returns_bad_response() {
let (mock, _setting) = create_mock_multiple().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index
@@ -1596,7 +1596,7 @@ async fn server_returns_bad_response() {
#[actix_rt::test]
async fn server_returns_multiple() {
let (_mock, setting) = create_mock_multiple().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index
@@ -1702,7 +1702,7 @@ async fn server_returns_multiple() {
#[actix_rt::test]
async fn server_single_input_returns_in_array() {
let (_mock, setting) = create_mock_single_response_in_array().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index
@@ -1808,7 +1808,7 @@ async fn server_single_input_returns_in_array() {
#[actix_rt::test]
async fn server_raw() {
let (_mock, setting) = create_mock_raw().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index
@@ -1915,7 +1915,7 @@ async fn server_raw() {
async fn server_custom_header() {
let (mock, setting) = create_mock_raw_with_custom_header().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index
@@ -2044,7 +2044,7 @@ async fn server_custom_header() {
#[actix_rt::test]
async fn searchable_reindex() {
let (_mock, setting) = create_mock_default_template().await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index
@@ -2154,7 +2154,7 @@ async fn searchable_reindex() {
async fn last_error_stats() {
let (sender, mut receiver) = mpsc::channel(10);
let (_mock, setting) = create_faulty_mock_raw(sender).await;
let server = get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index

View File

@@ -251,7 +251,7 @@ async fn reset_embedder_documents() {
#[actix_rt::test]
async fn ollama_url_checks() {
let server = super::get_server_vector().await;
let server = Server::new().await;
let index = server.index("doggo");
let (response, code) = index

View File

@@ -44,6 +44,7 @@ use std::hash::BuildHasherDefault;
use charabia::normalizer::{CharNormalizer, CompatibilityDecompositionNormalizer};
pub use documents::GeoSortStrategy;
pub use filter_parser;
pub use filter_parser::{Condition, FilterCondition, Span, Token};
use fxhash::{FxHasher32, FxHasher64};
pub use grenad::CompressionType;

View File

@@ -12,11 +12,14 @@ use bumpalo::Bump;
pub(crate) use document::{Document, ParseableDocument};
use error::{NewPromptError, RenderPromptError};
pub use fields::{BorrowedFields, OwnedFields};
use heed::RoTxn;
use liquid::{model::Value as LiquidValue, ValueView};
pub use self::context::Context;
use crate::fields_ids_map::metadata::FieldIdMapWithMetadata;
use crate::update::del_add::DelAdd;
use crate::GlobalFieldsIdsMap;
use crate::update::new::document::DocumentFromDb;
use crate::{FieldsIdsMap, GlobalFieldsIdsMap, Index};
pub struct Prompt {
template: liquid::Template,
@@ -164,6 +167,64 @@ fn truncate(s: &mut String, max_bytes: usize) {
}
}
/// Build the liquid objects corresponding to the `doc` object (without `fields`) of a [`Prompt`] from the given external document id.
pub fn build_doc(
index: &Index,
rtxn: &RoTxn<'_>,
external_id: &str,
fid_map: &FieldsIdsMap,
) -> Result<Option<LiquidValue>, crate::Error> {
_build_doc_fields(index, rtxn, external_id, fid_map, None)
.map(|opt| opt.map(|(doc, _fields)| doc))
}
/// Build the liquid objects corresponding to the `doc` and `fields` object of a [`Prompt`] from the given external document id.
pub fn build_doc_fields(
index: &Index,
rtxn: &RoTxn<'_>,
external_id: &str,
fid_map_with_meta: &FieldIdMapWithMetadata,
) -> Result<Option<(LiquidValue, LiquidValue)>, crate::Error> {
_build_doc_fields(
index,
rtxn,
external_id,
fid_map_with_meta.as_fields_ids_map(),
Some(fid_map_with_meta),
)
.map(|opt| {
opt.map(|(doc, fields)| {
(doc, fields.expect("fid_map_with_meta were provided so fields must be Some"))
})
})
}
fn _build_doc_fields(
index: &Index,
rtxn: &RoTxn<'_>,
external_id: &str,
fid_map: &FieldsIdsMap,
fid_map_with_meta: Option<&FieldIdMapWithMetadata>,
) -> Result<Option<(LiquidValue, Option<LiquidValue>)>, crate::Error> {
let Some(internal_id) = index.external_documents_ids().get(rtxn, external_id)? else {
return Ok(None);
};
let Some(document_from_db) = DocumentFromDb::new(internal_id, rtxn, index, &fid_map)? else {
return Ok(None);
};
let doc_alloc = Bump::new();
let parseable_document = ParseableDocument::new(document_from_db, &doc_alloc);
if let Some(fid_map_with_meta) = fid_map_with_meta {
let fields = OwnedFields::new(&parseable_document, fid_map_with_meta);
Ok(Some((parseable_document.to_value(), Some(fields.to_value()))))
} else {
Ok(Some((parseable_document.to_value(), None)))
}
}
#[cfg(test)]
mod test {
use super::Prompt;

View File

@@ -2041,7 +2041,7 @@ fn embedders(embedding_configs: Vec<IndexEmbeddingConfig>) -> Result<RuntimeEmbe
.into_iter()
.map(|fragment| {
let template = JsonTemplate::new(
embedder_options.fragment(&fragment.name).unwrap().clone(),
embedder_options.indexing_fragment(&fragment.name).unwrap().clone(),
)
.unwrap();

View File

@@ -61,20 +61,28 @@ pub struct Error {
impl Error {
/// Produces an error message when the error happened at rendering time.
pub fn rendering_error(&self, root: &str) -> String {
format!(
"in `{}`, error while rendering template: {}",
path_with_root(root, self.path.iter()),
&self.template_error
)
if self.path.is_empty() {
format!("error while rendering template: {}", &self.template_error)
} else {
format!(
"in `{}`, error while rendering template: {}",
path_with_root(root, self.path.iter()),
&self.template_error
)
}
}
/// Produces an error message when the error happened at parsing time.
pub fn parsing(&self, root: &str) -> String {
format!(
"in `{}`, error while parsing template: {}",
path_with_root(root, self.path.iter()),
&self.template_error
)
pub fn parsing_error(&self, root: &str) -> String {
if self.path.is_empty() {
format!("error while parsing template: {}", &self.template_error)
} else {
format!(
"in `{}`, error while parsing template: {}",
path_with_root(root, self.path.iter()),
&self.template_error
)
}
}
}

View File

@@ -817,7 +817,45 @@ pub enum EmbedderOptions {
}
impl EmbedderOptions {
pub fn fragment(&self, name: &str) -> Option<&serde_json::Value> {
pub fn has_fragments(&self) -> bool {
match &self {
EmbedderOptions::HuggingFace(_)
| EmbedderOptions::OpenAi(_)
| EmbedderOptions::Ollama(_)
| EmbedderOptions::UserProvided(_) => false,
EmbedderOptions::Rest(embedder_options) => {
!embedder_options.indexing_fragments.is_empty()
}
EmbedderOptions::Composite(embedder_options) => {
if let SubEmbedderOptions::Rest(embedder_options) = &embedder_options.index {
!embedder_options.indexing_fragments.is_empty()
} else {
false
}
}
}
}
pub fn indexing_fragments(&self) -> Vec<String> {
match &self {
EmbedderOptions::HuggingFace(_)
| EmbedderOptions::OpenAi(_)
| EmbedderOptions::Ollama(_)
| EmbedderOptions::UserProvided(_) => vec![],
EmbedderOptions::Rest(embedder_options) => {
embedder_options.indexing_fragments.keys().cloned().collect()
}
EmbedderOptions::Composite(embedder_options) => {
if let SubEmbedderOptions::Rest(embedder_options) = &embedder_options.index {
embedder_options.indexing_fragments.keys().cloned().collect()
} else {
vec![]
}
}
}
}
pub fn indexing_fragment(&self, name: &str) -> Option<&serde_json::Value> {
match &self {
EmbedderOptions::HuggingFace(_)
| EmbedderOptions::OpenAi(_)
@@ -836,20 +874,37 @@ impl EmbedderOptions {
}
}
pub fn has_fragments(&self) -> bool {
pub fn search_fragments(&self) -> Vec<String> {
match &self {
EmbedderOptions::HuggingFace(_)
| EmbedderOptions::OpenAi(_)
| EmbedderOptions::Ollama(_)
| EmbedderOptions::UserProvided(_) => false,
| EmbedderOptions::UserProvided(_) => vec![],
EmbedderOptions::Rest(embedder_options) => {
!embedder_options.indexing_fragments.is_empty()
embedder_options.search_fragments.keys().cloned().collect()
}
EmbedderOptions::Composite(embedder_options) => {
if let SubEmbedderOptions::Rest(embedder_options) = &embedder_options.index {
!embedder_options.indexing_fragments.is_empty()
if let SubEmbedderOptions::Rest(embedder_options) = &embedder_options.search {
embedder_options.search_fragments.keys().cloned().collect()
} else {
false
vec![]
}
}
}
}
pub fn search_fragment(&self, name: &str) -> Option<&serde_json::Value> {
match &self {
EmbedderOptions::HuggingFace(_)
| EmbedderOptions::OpenAi(_)
| EmbedderOptions::Ollama(_)
| EmbedderOptions::UserProvided(_) => None,
EmbedderOptions::Rest(embedder_options) => embedder_options.search_fragments.get(name),
EmbedderOptions::Composite(embedder_options) => {
if let SubEmbedderOptions::Rest(embedder_options) = &embedder_options.search {
embedder_options.search_fragments.get(name)
} else {
None
}
}
}

View File

@@ -113,7 +113,7 @@ impl RequestData {
for (name, value) in indexing_fragments {
JsonTemplate::new(value).map_err(|error| {
NewEmbedderError::rest_could_not_parse_template(
error.parsing(&format!(".indexingFragments.{name}")),
error.parsing_error(&format!(".indexingFragments.{name}")),
)
})?;
}
@@ -623,7 +623,7 @@ impl RequestFromFragments {
.map(|(name, value)| {
let json_template = JsonTemplate::new(value).map_err(|error| {
NewEmbedderError::rest_could_not_parse_template(
error.parsing(&format!(".searchFragments.{name}")),
error.parsing_error(&format!(".searchFragments.{name}")),
)
})?;
Ok((name, json_template))