mirror of
https://github.com/meilisearch/meilisearch.git
synced 2025-07-27 08:41:00 +00:00
Merge branch 'main' into key-actions-to-bitflags
This commit is contained in:
ManyTheFish
@ -11,39 +11,42 @@ edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
actix-web = { version = "4.8.0", default-features = false }
|
||||
anyhow = "1.0.86"
|
||||
actix-web = { version = "4.9.0", default-features = false }
|
||||
anyhow = "1.0.95"
|
||||
bumpalo = "3.16.0"
|
||||
bumparaw-collections = "0.1.4"
|
||||
convert_case = "0.6.0"
|
||||
csv = "1.3.0"
|
||||
deserr = { version = "0.6.2", features = ["actix-web"] }
|
||||
csv = "1.3.1"
|
||||
deserr = { version = "0.6.3", features = ["actix-web"] }
|
||||
either = { version = "1.13.0", features = ["serde"] }
|
||||
enum-iterator = "2.1.0"
|
||||
file-store = { path = "../file-store" }
|
||||
flate2 = "1.0.30"
|
||||
flate2 = "1.0.35"
|
||||
fst = "0.4.7"
|
||||
memmap2 = "0.9.4"
|
||||
memmap2 = "0.9.5"
|
||||
milli = { path = "../milli" }
|
||||
raw-collections = { git = "https://github.com/meilisearch/raw-collections.git", version = "0.1.0" }
|
||||
roaring = { version = "0.10.6", features = ["serde"] }
|
||||
serde = { version = "1.0.204", features = ["derive"] }
|
||||
roaring = { version = "0.10.10", features = ["serde"] }
|
||||
rustc-hash = "2.1.0"
|
||||
serde = { version = "1.0.217", features = ["derive"] }
|
||||
serde-cs = "0.2.4"
|
||||
serde_json = "1.0.120"
|
||||
tar = "0.4.41"
|
||||
tempfile = "3.10.1"
|
||||
thiserror = "1.0.61"
|
||||
time = { version = "0.3.36", features = [
|
||||
serde_json = { version = "1.0.135", features = ["preserve_order"] }
|
||||
tar = "0.4.43"
|
||||
tempfile = "3.15.0"
|
||||
thiserror = "2.0.9"
|
||||
time = { version = "0.3.37", features = [
|
||||
"serde-well-known",
|
||||
"formatting",
|
||||
"parsing",
|
||||
"macros",
|
||||
] }
|
||||
tokio = "1.38"
|
||||
uuid = { version = "1.10.0", features = ["serde", "v4"] }
|
||||
tokio = "1.42"
|
||||
utoipa = { version = "5.3.1", features = ["macros"] }
|
||||
uuid = { version = "1.11.0", features = ["serde", "v4"] }
|
||||
bitflags = "2.6.0"
|
||||
|
||||
[dev-dependencies]
|
||||
insta = "1.39.0"
|
||||
# fixed version due to format breakages in v1.40
|
||||
insta = "=1.39.0"
|
||||
meili-snap = { path = "../meili-snap" }
|
||||
|
||||
[features]
|
||||
|
||||
@ -1,16 +1,18 @@
|
||||
use milli::progress::ProgressView;
|
||||
use serde::Serialize;
|
||||
use time::{Duration, OffsetDateTime};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use crate::{
|
||||
batches::{Batch, BatchId, BatchStats},
|
||||
task_view::DetailsView,
|
||||
tasks::serialize_duration,
|
||||
};
|
||||
use crate::batches::{Batch, BatchId, BatchStats};
|
||||
use crate::task_view::DetailsView;
|
||||
use crate::tasks::serialize_duration;
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[derive(Debug, Clone, Serialize, ToSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[schema(rename_all = "camelCase")]
|
||||
pub struct BatchView {
|
||||
pub uid: BatchId,
|
||||
pub progress: Option<ProgressView>,
|
||||
pub details: DetailsView,
|
||||
pub stats: BatchStats,
|
||||
#[serde(serialize_with = "serialize_duration", default)]
|
||||
@ -25,6 +27,7 @@ impl BatchView {
|
||||
pub fn from_batch(batch: &Batch) -> Self {
|
||||
Self {
|
||||
uid: batch.uid,
|
||||
progress: batch.progress.clone(),
|
||||
details: batch.details.clone(),
|
||||
stats: batch.stats.clone(),
|
||||
duration: batch.finished_at.map(|finished_at| finished_at - batch.started_at),
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use milli::progress::ProgressView;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use time::OffsetDateTime;
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use crate::{
|
||||
task_view::DetailsView,
|
||||
tasks::{Kind, Status},
|
||||
};
|
||||
use crate::task_view::DetailsView;
|
||||
use crate::tasks::{Kind, Status};
|
||||
|
||||
pub type BatchId = u32;
|
||||
|
||||
@ -15,6 +15,8 @@ pub type BatchId = u32;
|
||||
pub struct Batch {
|
||||
pub uid: BatchId,
|
||||
|
||||
#[serde(skip)]
|
||||
pub progress: Option<ProgressView>,
|
||||
pub details: DetailsView,
|
||||
pub stats: BatchStats,
|
||||
|
||||
@ -22,13 +24,44 @@ pub struct Batch {
|
||||
pub started_at: OffsetDateTime,
|
||||
#[serde(with = "time::serde::rfc3339::option")]
|
||||
pub finished_at: Option<OffsetDateTime>,
|
||||
|
||||
// Enqueued at is never displayed and is only required when removing a batch.
|
||||
// It's always some except when upgrading from a database pre v1.12
|
||||
pub enqueued_at: Option<BatchEnqueuedAt>,
|
||||
}
|
||||
|
||||
#[derive(Default, Debug, Clone, Serialize, Deserialize)]
|
||||
impl PartialEq for Batch {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
let Self { uid, progress, details, stats, started_at, finished_at, enqueued_at } = self;
|
||||
|
||||
*uid == other.uid
|
||||
&& progress.is_none() == other.progress.is_none()
|
||||
&& details == &other.details
|
||||
&& stats == &other.stats
|
||||
&& started_at == &other.started_at
|
||||
&& finished_at == &other.finished_at
|
||||
&& enqueued_at == &other.enqueued_at
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct BatchEnqueuedAt {
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
pub earliest: OffsetDateTime,
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
pub oldest: OffsetDateTime,
|
||||
}
|
||||
|
||||
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[schema(rename_all = "camelCase")]
|
||||
pub struct BatchStats {
|
||||
pub total_nb_tasks: BatchId,
|
||||
pub status: BTreeMap<Status, u32>,
|
||||
pub types: BTreeMap<Kind, u32>,
|
||||
pub index_uids: BTreeMap<String, u32>,
|
||||
#[serde(default, skip_serializing_if = "serde_json::Map::is_empty")]
|
||||
pub progress_trace: serde_json::Map<String, serde_json::Value>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub write_channel_congestion: Option<serde_json::Map<String, serde_json::Value>>,
|
||||
}
|
||||
|
||||
@ -193,6 +193,8 @@ merge_with_error_impl_take_error_message!(ParseTaskKindError);
|
||||
merge_with_error_impl_take_error_message!(ParseTaskStatusError);
|
||||
merge_with_error_impl_take_error_message!(IndexUidFormatError);
|
||||
merge_with_error_impl_take_error_message!(InvalidMultiSearchWeight);
|
||||
merge_with_error_impl_take_error_message!(InvalidNetworkUrl);
|
||||
merge_with_error_impl_take_error_message!(InvalidNetworkSearchApiKey);
|
||||
merge_with_error_impl_take_error_message!(InvalidSearchSemanticRatio);
|
||||
merge_with_error_impl_take_error_message!(InvalidSearchRankingScoreThreshold);
|
||||
merge_with_error_impl_take_error_message!(InvalidSimilarRankingScoreThreshold);
|
||||
|
||||
@ -4,10 +4,11 @@ use std::io::{self, BufWriter};
|
||||
use std::marker::PhantomData;
|
||||
|
||||
use bumpalo::Bump;
|
||||
use bumparaw_collections::RawMap;
|
||||
use memmap2::Mmap;
|
||||
use milli::documents::Error;
|
||||
use milli::Object;
|
||||
use raw_collections::RawMap;
|
||||
use rustc_hash::FxBuildHasher;
|
||||
use serde::de::{SeqAccess, Visitor};
|
||||
use serde::{Deserialize, Deserializer};
|
||||
use serde_json::error::Category;
|
||||
@ -214,13 +215,13 @@ pub fn read_json(input: &File, output: impl io::Write) -> Result<u64> {
|
||||
// We memory map to be able to deserialize into a RawMap that
|
||||
// does not allocate when possible and only materialize the first/top level.
|
||||
let input = unsafe { Mmap::map(input).map_err(DocumentFormatError::Io)? };
|
||||
let mut doc_alloc = Bump::with_capacity(1024 * 1024 * 1024); // 1MiB
|
||||
let mut doc_alloc = Bump::with_capacity(1024 * 1024); // 1MiB
|
||||
|
||||
let mut out = BufWriter::new(output);
|
||||
let mut deserializer = serde_json::Deserializer::from_slice(&input);
|
||||
let res = array_each(&mut deserializer, |obj: &RawValue| {
|
||||
doc_alloc.reset();
|
||||
let map = RawMap::from_raw_value(obj, &doc_alloc)?;
|
||||
let map = RawMap::from_raw_value_and_hasher(obj, FxBuildHasher, &doc_alloc)?;
|
||||
to_writer(&mut out, &map)
|
||||
});
|
||||
let count = match res {
|
||||
@ -250,26 +251,25 @@ pub fn read_json(input: &File, output: impl io::Write) -> Result<u64> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads NDJSON from file and write it in NDJSON in a file checking it along the way.
|
||||
pub fn read_ndjson(input: &File, output: impl io::Write) -> Result<u64> {
|
||||
/// Reads NDJSON from file and checks it.
|
||||
pub fn read_ndjson(input: &File) -> Result<u64> {
|
||||
// We memory map to be able to deserialize into a RawMap that
|
||||
// does not allocate when possible and only materialize the first/top level.
|
||||
let input = unsafe { Mmap::map(input).map_err(DocumentFormatError::Io)? };
|
||||
let mut output = BufWriter::new(output);
|
||||
|
||||
let mut bump = Bump::with_capacity(1024 * 1024);
|
||||
|
||||
let mut count = 0;
|
||||
for result in serde_json::Deserializer::from_slice(&input).into_iter() {
|
||||
bump.reset();
|
||||
count += 1;
|
||||
result
|
||||
.and_then(|raw: &RawValue| {
|
||||
match result {
|
||||
Ok(raw) => {
|
||||
// try to deserialize as a map
|
||||
let map = RawMap::from_raw_value(raw, &bump)?;
|
||||
to_writer(&mut output, &map)
|
||||
})
|
||||
.map_err(|e| DocumentFormatError::from((PayloadType::Ndjson, e)))?;
|
||||
RawMap::from_raw_value_and_hasher(raw, FxBuildHasher, &bump)
|
||||
.map_err(|e| DocumentFormatError::from((PayloadType::Ndjson, e)))?;
|
||||
count += 1;
|
||||
}
|
||||
Err(e) => return Err(DocumentFormatError::from((PayloadType::Ndjson, e))),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(count)
|
||||
|
||||
@ -7,17 +7,25 @@ use aweb::rt::task::JoinError;
|
||||
use convert_case::Casing;
|
||||
use milli::heed::{Error as HeedError, MdbError};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, ToSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[schema(rename_all = "camelCase")]
|
||||
pub struct ResponseError {
|
||||
#[serde(skip)]
|
||||
pub code: StatusCode,
|
||||
/// The error message.
|
||||
pub message: String,
|
||||
/// The error code.
|
||||
#[schema(value_type = Code)]
|
||||
#[serde(rename = "code")]
|
||||
error_code: String,
|
||||
/// The error type.
|
||||
#[schema(value_type = ErrorType)]
|
||||
#[serde(rename = "type")]
|
||||
error_type: String,
|
||||
/// A link to the documentation about this specific error.
|
||||
#[serde(rename = "link")]
|
||||
error_link: String,
|
||||
}
|
||||
@ -97,7 +105,9 @@ pub trait ErrorCode {
|
||||
}
|
||||
|
||||
#[allow(clippy::enum_variant_names)]
|
||||
enum ErrorType {
|
||||
#[derive(ToSchema)]
|
||||
#[schema(rename_all = "snake_case")]
|
||||
pub enum ErrorType {
|
||||
Internal,
|
||||
InvalidRequest,
|
||||
Auth,
|
||||
@ -129,7 +139,8 @@ impl fmt::Display for ErrorType {
|
||||
/// `MyErrorCode::default().error_code()`.
|
||||
macro_rules! make_error_codes {
|
||||
($($code_ident:ident, $err_type:ident, $status:ident);*) => {
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, ToSchema)]
|
||||
#[schema(rename_all = "snake_case")]
|
||||
pub enum Code {
|
||||
$($code_ident),*
|
||||
}
|
||||
@ -230,10 +241,12 @@ InvalidDocumentGeoField , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidVectorDimensions , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidVectorsType , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidDocumentId , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidDocumentIds , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidDocumentLimit , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidDocumentOffset , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidEmbedder , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidHybridQuery , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSearchEmbedder , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSimilarEmbedder , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSearchHybridQuery , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidIndexLimit , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidIndexOffset , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidIndexPrimaryKey , InvalidRequest , BAD_REQUEST ;
|
||||
@ -248,7 +261,13 @@ InvalidMultiSearchMergeFacets , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidMultiSearchQueryFacets , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidMultiSearchQueryPagination , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidMultiSearchQueryRankingRules , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidMultiSearchQueryPosition , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidMultiSearchRemote , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidMultiSearchWeight , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidNetworkRemotes , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidNetworkSelf , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidNetworkSearchApiKey , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidNetworkUrl , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSearchAttributesToSearchOn , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSearchAttributesToCrop , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSearchAttributesToHighlight , InvalidRequest , BAD_REQUEST ;
|
||||
@ -263,6 +282,7 @@ InvalidSearchCropMarker , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSearchFacets , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSearchSemanticRatio , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSearchLocales , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidFacetSearchExhaustiveFacetCount, InvalidRequest , BAD_REQUEST ;
|
||||
InvalidFacetSearchFacetName , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSimilarId , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSearchFilter , InvalidRequest , BAD_REQUEST ;
|
||||
@ -279,6 +299,7 @@ InvalidSearchPage , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSearchQ , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidFacetSearchQuery , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidFacetSearchName , InvalidRequest , BAD_REQUEST ;
|
||||
FacetSearchDisabled , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSearchVector , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSearchShowMatchesPosition , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSearchShowRankingScore , InvalidRequest , BAD_REQUEST ;
|
||||
@ -290,6 +311,8 @@ InvalidSearchDistinct , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSettingsDisplayedAttributes , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSettingsDistinctAttribute , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSettingsProximityPrecision , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSettingsFacetSearch , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSettingsPrefixSearch , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSettingsFaceting , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSettingsFilterableAttributes , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSettingsPagination , InvalidRequest , BAD_REQUEST ;
|
||||
@ -336,14 +359,22 @@ MissingDocumentId , InvalidRequest , BAD_REQUEST ;
|
||||
MissingFacetSearchFacetName , InvalidRequest , BAD_REQUEST ;
|
||||
MissingIndexUid , InvalidRequest , BAD_REQUEST ;
|
||||
MissingMasterKey , Auth , UNAUTHORIZED ;
|
||||
MissingNetworkUrl , InvalidRequest , BAD_REQUEST ;
|
||||
MissingPayload , InvalidRequest , BAD_REQUEST ;
|
||||
MissingSearchHybrid , InvalidRequest , BAD_REQUEST ;
|
||||
MissingSwapIndexes , InvalidRequest , BAD_REQUEST ;
|
||||
MissingTaskFilters , InvalidRequest , BAD_REQUEST ;
|
||||
NoSpaceLeftOnDevice , System , UNPROCESSABLE_ENTITY;
|
||||
PayloadTooLarge , InvalidRequest , PAYLOAD_TOO_LARGE ;
|
||||
RemoteBadResponse , System , BAD_GATEWAY ;
|
||||
RemoteBadRequest , InvalidRequest , BAD_REQUEST ;
|
||||
RemoteCouldNotSendRequest , System , BAD_GATEWAY ;
|
||||
RemoteInvalidApiKey , Auth , FORBIDDEN ;
|
||||
RemoteRemoteError , System , BAD_GATEWAY ;
|
||||
RemoteTimeout , System , BAD_GATEWAY ;
|
||||
TooManySearchRequests , System , SERVICE_UNAVAILABLE ;
|
||||
TaskNotFound , InvalidRequest , NOT_FOUND ;
|
||||
TaskFileNotFound , InvalidRequest , NOT_FOUND ;
|
||||
BatchNotFound , InvalidRequest , NOT_FOUND ;
|
||||
TooManyOpenFiles , System , UNPROCESSABLE_ENTITY ;
|
||||
TooManyVectors , InvalidRequest , BAD_REQUEST ;
|
||||
@ -376,7 +407,7 @@ impl ErrorCode for milli::Error {
|
||||
match error {
|
||||
// TODO: wait for spec for new error codes.
|
||||
UserError::SerdeJson(_)
|
||||
| UserError::InvalidLmdbOpenOptions
|
||||
| UserError::EnvAlreadyOpened
|
||||
| UserError::DocumentLimitReached
|
||||
| UserError::UnknownInternalDocumentId { .. } => Code::Internal,
|
||||
UserError::InvalidStoreFile => Code::InvalidStoreFile,
|
||||
@ -385,6 +416,7 @@ impl ErrorCode for milli::Error {
|
||||
UserError::AttributeLimitReached => Code::MaxFieldsLimitExceeded,
|
||||
UserError::InvalidFilter(_) => Code::InvalidSearchFilter,
|
||||
UserError::InvalidFilterExpression(..) => Code::InvalidSearchFilter,
|
||||
UserError::FilterOperatorNotAllowed { .. } => Code::InvalidSearchFilter,
|
||||
UserError::MissingDocumentId { .. } => Code::MissingDocumentId,
|
||||
UserError::InvalidDocumentId { .. } | UserError::TooManyDocumentIds { .. } => {
|
||||
Code::InvalidDocumentId
|
||||
@ -399,9 +431,10 @@ impl ErrorCode for milli::Error {
|
||||
| UserError::InvalidUrl { .. }
|
||||
| UserError::InvalidSettingsDocumentTemplateMaxBytes { .. }
|
||||
| UserError::InvalidPrompt(_)
|
||||
| UserError::InvalidDisableBinaryQuantization { .. } => {
|
||||
Code::InvalidSettingsEmbedders
|
||||
}
|
||||
| UserError::InvalidDisableBinaryQuantization { .. }
|
||||
| UserError::InvalidSourceForNested { .. }
|
||||
| UserError::MissingSourceForNested { .. }
|
||||
| UserError::InvalidSettingsEmbedder { .. } => Code::InvalidSettingsEmbedders,
|
||||
UserError::TooManyEmbedders(_) => Code::InvalidSettingsEmbedders,
|
||||
UserError::InvalidPromptForEmbeddings(..) => Code::InvalidSettingsEmbedders,
|
||||
UserError::NoPrimaryKeyCandidateFound => Code::IndexPrimaryKeyNoCandidateFound,
|
||||
@ -429,7 +462,8 @@ impl ErrorCode for milli::Error {
|
||||
UserError::InvalidMinTypoWordLenSetting(_, _) => {
|
||||
Code::InvalidSettingsTypoTolerance
|
||||
}
|
||||
UserError::InvalidEmbedder(_) => Code::InvalidEmbedder,
|
||||
UserError::InvalidSearchEmbedder(_) => Code::InvalidSearchEmbedder,
|
||||
UserError::InvalidSimilarEmbedder(_) => Code::InvalidSimilarEmbedder,
|
||||
UserError::VectorEmbeddingError(_) | UserError::DocumentEmbeddingError(_) => {
|
||||
Code::VectorEmbeddingError
|
||||
}
|
||||
@ -470,8 +504,7 @@ impl ErrorCode for HeedError {
|
||||
HeedError::Mdb(_)
|
||||
| HeedError::Encoding(_)
|
||||
| HeedError::Decoding(_)
|
||||
| HeedError::DatabaseClosing
|
||||
| HeedError::BadOpenOptions { .. } => Code::Internal,
|
||||
| HeedError::EnvAlreadyOpened => Code::Internal,
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -547,7 +580,7 @@ impl fmt::Display for deserr_codes::InvalidSimilarId {
|
||||
"the value of `id` is invalid. \
|
||||
A document identifier can be of type integer or string, \
|
||||
only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_), \
|
||||
and can not be more than 512 bytes."
|
||||
and can not be more than 511 bytes."
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -567,6 +600,18 @@ impl fmt::Display for deserr_codes::InvalidSimilarRankingScoreThreshold {
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for deserr_codes::InvalidNetworkUrl {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "the value of `url` is invalid, expected a string.")
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for deserr_codes::InvalidNetworkSearchApiKey {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "the value of `searchApiKey` is invalid, expected a string.")
|
||||
}
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! internal_error {
|
||||
($target:ty : $($other:path), *) => {
|
||||
|
||||
@ -1,8 +1,9 @@
|
||||
use deserr::Deserr;
|
||||
use milli::OrderBy;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Deserr)]
|
||||
#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Deserr, ToSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[deserr(rename_all = camelCase)]
|
||||
pub enum FacetValuesSort {
|
||||
|
||||
@ -1,13 +1,17 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
pub struct RuntimeTogglableFeatures {
|
||||
pub vector_store: bool,
|
||||
pub metrics: bool,
|
||||
pub logs_route: bool,
|
||||
pub edit_documents_by_function: bool,
|
||||
pub contains_filter: bool,
|
||||
pub network: bool,
|
||||
pub get_task_documents_route: bool,
|
||||
pub composite_embedders: bool,
|
||||
}
|
||||
|
||||
#[derive(Default, Debug, Clone, Copy)]
|
||||
@ -16,3 +20,20 @@ pub struct InstanceTogglableFeatures {
|
||||
pub logs_route: bool,
|
||||
pub contains_filter: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Remote {
|
||||
pub url: String,
|
||||
#[serde(default)]
|
||||
pub search_api_key: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Network {
|
||||
#[serde(default, rename = "self")]
|
||||
pub local: Option<String>,
|
||||
#[serde(default)]
|
||||
pub remotes: BTreeMap<String, Remote>,
|
||||
}
|
||||
|
||||
@ -4,13 +4,16 @@ use std::fmt;
|
||||
use std::str::FromStr;
|
||||
|
||||
use deserr::Deserr;
|
||||
use serde::Serialize;
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use crate::error::{Code, ErrorCode};
|
||||
|
||||
/// An index uid is composed of only ascii alphanumeric characters, - and _, between 1 and 400
|
||||
/// bytes long
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Deserr, PartialOrd, Ord)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Deserr, PartialOrd, Ord, Serialize, ToSchema)]
|
||||
#[deserr(try_from(String) = IndexUid::try_from -> IndexUidFormatError)]
|
||||
#[schema(value_type = String, example = "movies")]
|
||||
pub struct IndexUid(String);
|
||||
|
||||
impl IndexUid {
|
||||
|
||||
@ -10,6 +10,7 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
use time::macros::{format_description, time};
|
||||
use time::{Date, OffsetDateTime, PrimitiveDateTime};
|
||||
use utoipa::ToSchema;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::deserr::{immutable_field_error, DeserrError, DeserrJsonError};
|
||||
@ -33,19 +34,31 @@ impl<C: Default + ErrorCode> MergeWithError<IndexUidPatternFormatError> for Dese
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserr)]
|
||||
#[derive(Debug, Deserr, ToSchema)]
|
||||
#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
|
||||
#[schema(rename_all = "camelCase")]
|
||||
pub struct CreateApiKey {
|
||||
/// A description for the key. `null` if empty.
|
||||
#[schema(example = json!(null))]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidApiKeyDescription>)]
|
||||
pub description: Option<String>,
|
||||
/// A human-readable name for the key. `null` if empty.
|
||||
#[schema(example = "Indexing Products API key")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidApiKeyName>)]
|
||||
pub name: Option<String>,
|
||||
/// A uuid v4 to identify the API Key. If not specified, it's generated by Meilisearch.
|
||||
#[schema(value_type = Uuid, example = json!(null))]
|
||||
#[deserr(default = Uuid::new_v4(), error = DeserrJsonError<InvalidApiKeyUid>, try_from(&String) = Uuid::from_str -> uuid::Error)]
|
||||
pub uid: KeyId,
|
||||
/// A list of actions permitted for the key. `["*"]` for all actions. The `*` character can be used as a wildcard when located at the last position. e.g. `documents.*` to authorize access on all documents endpoints.
|
||||
#[schema(example = json!(["documents.add"]))]
|
||||
#[deserr(error = DeserrJsonError<InvalidApiKeyActions>, missing_field_error = DeserrJsonError::missing_api_key_actions)]
|
||||
pub actions: Vec<Action>,
|
||||
/// A list of accesible indexes permitted for the key. `["*"]` for all indexes. The `*` character can be used as a wildcard when located at the last position. e.g. `products_*` to allow access to all indexes whose names start with `products_`.
|
||||
#[deserr(error = DeserrJsonError<InvalidApiKeyIndexes>, missing_field_error = DeserrJsonError::missing_api_key_indexes)]
|
||||
#[schema(value_type = Vec<String>, example = json!(["products"]))]
|
||||
pub indexes: Vec<IndexUidPattern>,
|
||||
/// Represent the expiration date and time as RFC 3339 format. `null` equals to no expiration time.
|
||||
#[deserr(error = DeserrJsonError<InvalidApiKeyExpiresAt>, try_from(Option<String>) = parse_expiration_date -> ParseOffsetDateTimeError, missing_field_error = DeserrJsonError::missing_api_key_expires_at)]
|
||||
pub expires_at: Option<OffsetDateTime>,
|
||||
}
|
||||
@ -87,12 +100,15 @@ fn deny_immutable_fields_api_key(
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserr)]
|
||||
#[derive(Debug, Deserr, ToSchema)]
|
||||
#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields = deny_immutable_fields_api_key)]
|
||||
#[schema(rename_all = "camelCase")]
|
||||
pub struct PatchApiKey {
|
||||
#[deserr(default, error = DeserrJsonError<InvalidApiKeyDescription>)]
|
||||
#[schema(value_type = Option<String>, example = "This key is used to update documents in the products index")]
|
||||
pub description: Setting<String>,
|
||||
#[deserr(default, error = DeserrJsonError<InvalidApiKeyName>)]
|
||||
#[schema(value_type = Option<String>, example = "Indexing Products API key")]
|
||||
pub name: Setting<String>,
|
||||
}
|
||||
|
||||
@ -180,11 +196,13 @@ fn parse_expiration_date(
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, PartialOrd, Ord, ToSchema)]
|
||||
#[repr(transparent)]
|
||||
pub struct Action(u32);
|
||||
|
||||
bitflags! {
|
||||
#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, PartialOrd, Ord)]
|
||||
#[repr(transparent)]
|
||||
// NOTE: For `Sequence` impl to work, the values of these must be in ascending order
|
||||
pub struct Action: u32 {
|
||||
impl Action: u32 {
|
||||
const Search = 1;
|
||||
// Documents
|
||||
const DocumentsAdd = 1 << 1;
|
||||
@ -225,15 +243,19 @@ bitflags! {
|
||||
const KeysGet = 1 << 20;
|
||||
const KeysUpdate = 1 << 21;
|
||||
const KeysDelete = 1 << 22;
|
||||
// Experimental Features
|
||||
const ExperimentalFeaturesGet = 1 << 23;
|
||||
const ExperimentalFeaturesUpdate = 1 << 24;
|
||||
// Network
|
||||
const NetworkGet = 1 << 25;
|
||||
const NetworkUpdate = 1 << 26;
|
||||
// All
|
||||
const All = 0xFFFFFFFF >> (32 - 1 - 24);
|
||||
const All = 0xFFFFFFFF >> (32 - 1 - 26);
|
||||
}
|
||||
}
|
||||
|
||||
impl Action {
|
||||
const SERDE_MAP_ARR: [(&'static str, Self); 34] = [
|
||||
const SERDE_MAP_ARR: [(&'static str, Self); 36] = [
|
||||
("search", Self::Search),
|
||||
("documents.add", Self::DocumentsAdd),
|
||||
("documents.get", Self::DocumentsGet),
|
||||
@ -267,6 +289,8 @@ impl Action {
|
||||
("keys.delete", Self::KeysDelete),
|
||||
("experimental.get", Self::ExperimentalFeaturesGet),
|
||||
("experimental.update", Self::ExperimentalFeaturesUpdate),
|
||||
("network.get", Self::NetworkGet),
|
||||
("network.update", Self::NetworkUpdate),
|
||||
("*", Self::All),
|
||||
];
|
||||
|
||||
@ -335,6 +359,8 @@ pub mod actions {
|
||||
pub const KEYS_DELETE: u32 = A::KeysDelete.bits();
|
||||
pub const EXPERIMENTAL_FEATURES_GET: u32 = A::ExperimentalFeaturesGet.bits();
|
||||
pub const EXPERIMENTAL_FEATURES_UPDATE: u32 = A::ExperimentalFeaturesUpdate.bits();
|
||||
pub const NETWORK_GET: u32 = A::NetworkGet.bits();
|
||||
pub const NETWORK_UPDATE: u32 = A::NetworkUpdate.bits();
|
||||
pub const ALL: u32 = A::All.bits();
|
||||
}
|
||||
|
||||
@ -382,7 +408,7 @@ impl<'de> Deserialize<'de> for Action {
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
struct Visitor;
|
||||
impl<'de> serde::de::Visitor<'de> for Visitor {
|
||||
impl serde::de::Visitor<'_> for Visitor {
|
||||
type Value = Action;
|
||||
|
||||
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
|
||||
@ -1,12 +1,13 @@
|
||||
use deserr::Deserr;
|
||||
use milli::LocalizedAttributesRule;
|
||||
use milli::{AttributePatterns, LocalizedAttributesRule};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize, ToSchema)]
|
||||
#[deserr(rename_all = camelCase)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct LocalizedAttributesRuleView {
|
||||
pub attribute_patterns: Vec<String>,
|
||||
pub attribute_patterns: AttributePatterns,
|
||||
pub locales: Vec<Locale>,
|
||||
}
|
||||
|
||||
@ -33,7 +34,7 @@ impl From<LocalizedAttributesRuleView> for LocalizedAttributesRule {
|
||||
/// this enum implements `Deserr` in order to be used in the API.
|
||||
macro_rules! make_locale {
|
||||
($(($iso_639_1:ident, $iso_639_1_str:expr) => ($iso_639_3:ident, $iso_639_3_str:expr),)+) => {
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize, Ord, PartialOrd)]
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize, Ord, PartialOrd, ToSchema)]
|
||||
#[deserr(rename_all = camelCase)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Locale {
|
||||
|
||||
@ -8,11 +8,12 @@ use std::str::FromStr;
|
||||
|
||||
use deserr::{DeserializeError, Deserr, ErrorKind, MergeWithError, ValuePointerRef};
|
||||
use fst::IntoStreamer;
|
||||
use milli::index::IndexEmbeddingConfig;
|
||||
use milli::index::{IndexEmbeddingConfig, PrefixSearch};
|
||||
use milli::proximity::ProximityPrecision;
|
||||
use milli::update::Setting;
|
||||
use milli::{Criterion, CriterionError, Index, DEFAULT_VALUES_PER_FACET};
|
||||
use milli::{Criterion, CriterionError, FilterableAttributesRule, Index, DEFAULT_VALUES_PER_FACET};
|
||||
use serde::{Deserialize, Serialize, Serializer};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use crate::deserr::DeserrJsonError;
|
||||
use crate::error::deserr_codes::*;
|
||||
@ -39,10 +40,10 @@ where
|
||||
.serialize(s)
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Clone, Default, Debug, Serialize, PartialEq, Eq, ToSchema)]
|
||||
pub struct Checked;
|
||||
|
||||
#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, ToSchema)]
|
||||
pub struct Unchecked;
|
||||
|
||||
impl<E> Deserr<E> for Unchecked
|
||||
@ -69,54 +70,63 @@ fn validate_min_word_size_for_typo_setting<E: DeserializeError>(
|
||||
Ok(s)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr)]
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)]
|
||||
#[serde(deny_unknown_fields, rename_all = "camelCase")]
|
||||
#[deserr(deny_unknown_fields, rename_all = camelCase, validate = validate_min_word_size_for_typo_setting -> DeserrJsonError<InvalidSettingsTypoTolerance>)]
|
||||
pub struct MinWordSizeTyposSetting {
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default)]
|
||||
#[schema(value_type = Option<u8>, example = json!(5))]
|
||||
pub one_typo: Setting<u8>,
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default)]
|
||||
#[schema(value_type = Option<u8>, example = json!(9))]
|
||||
pub two_typos: Setting<u8>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr)]
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)]
|
||||
#[serde(deny_unknown_fields, rename_all = "camelCase")]
|
||||
#[deserr(deny_unknown_fields, rename_all = camelCase, where_predicate = __Deserr_E: deserr::MergeWithError<DeserrJsonError<InvalidSettingsTypoTolerance>>)]
|
||||
pub struct TypoSettings {
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default)]
|
||||
#[schema(value_type = Option<bool>, example = json!(true))]
|
||||
pub enabled: Setting<bool>,
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsTypoTolerance>)]
|
||||
#[schema(value_type = Option<MinWordSizeTyposSetting>, example = json!({ "oneTypo": 5, "twoTypo": 9 }))]
|
||||
pub min_word_size_for_typos: Setting<MinWordSizeTyposSetting>,
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default)]
|
||||
#[schema(value_type = Option<BTreeSet<String>>, example = json!(["iPhone", "phone"]))]
|
||||
pub disable_on_words: Setting<BTreeSet<String>>,
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default)]
|
||||
#[schema(value_type = Option<BTreeSet<String>>, example = json!(["uuid", "url"]))]
|
||||
pub disable_on_attributes: Setting<BTreeSet<String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr)]
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)]
|
||||
#[serde(deny_unknown_fields, rename_all = "camelCase")]
|
||||
#[deserr(rename_all = camelCase, deny_unknown_fields)]
|
||||
pub struct FacetingSettings {
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default)]
|
||||
#[schema(value_type = Option<usize>, example = json!(10))]
|
||||
pub max_values_per_facet: Setting<usize>,
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default)]
|
||||
#[schema(value_type = Option<BTreeMap<String, FacetValuesSort>>, example = json!({ "genre": FacetValuesSort::Count }))]
|
||||
pub sort_facet_values_by: Setting<BTreeMap<String, FacetValuesSort>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr)]
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)]
|
||||
#[serde(deny_unknown_fields, rename_all = "camelCase")]
|
||||
#[deserr(rename_all = camelCase, deny_unknown_fields)]
|
||||
pub struct PaginationSettings {
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default)]
|
||||
#[schema(value_type = Option<usize>, example = json!(250))]
|
||||
pub max_total_hits: Setting<usize>,
|
||||
}
|
||||
|
||||
@ -134,74 +144,150 @@ impl MergeWithError<milli::CriterionError> for DeserrJsonError<InvalidSettingsRa
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Serialize, Deserialize, PartialEq, Eq, Clone, ToSchema)]
|
||||
#[repr(transparent)]
|
||||
#[serde(transparent)]
|
||||
/// "Technical" type that is required due to utoipa.
|
||||
///
|
||||
/// We did not find a way to implement [`utoipa::ToSchema`] for the [`Setting`] enum,
|
||||
/// but most types can use the `value_type` macro parameter to workaround that issue.
|
||||
///
|
||||
/// However that type is used in the settings route, including through the macro that auto-generate
|
||||
/// all the settings route, so we can't remap the `value_type`.
|
||||
pub struct SettingEmbeddingSettings {
|
||||
#[schema(inline, value_type = Option<crate::milli::vector::settings::EmbeddingSettings>)]
|
||||
pub inner: Setting<crate::milli::vector::settings::EmbeddingSettings>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for SettingEmbeddingSettings {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
self.inner.fmt(f)
|
||||
}
|
||||
}
|
||||
|
||||
impl<E: DeserializeError> Deserr<E> for SettingEmbeddingSettings {
|
||||
fn deserialize_from_value<V: deserr::IntoValue>(
|
||||
value: deserr::Value<V>,
|
||||
location: ValuePointerRef,
|
||||
) -> Result<Self, E> {
|
||||
Setting::<crate::milli::vector::settings::EmbeddingSettings>::deserialize_from_value(
|
||||
value, location,
|
||||
)
|
||||
.map(|inner| Self { inner })
|
||||
}
|
||||
}
|
||||
|
||||
/// Holds all the settings for an index. `T` can either be `Checked` if they represents settings
|
||||
/// whose validity is guaranteed, or `Unchecked` if they need to be validated. In the later case, a
|
||||
/// call to `check` will return a `Settings<Checked>` from a `Settings<Unchecked>`.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr)]
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)]
|
||||
#[serde(
|
||||
deny_unknown_fields,
|
||||
rename_all = "camelCase",
|
||||
bound(serialize = "T: Serialize", deserialize = "T: Deserialize<'static>")
|
||||
)]
|
||||
#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
|
||||
#[schema(rename_all = "camelCase")]
|
||||
pub struct Settings<T> {
|
||||
/// Fields displayed in the returned documents.
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsDisplayedAttributes>)]
|
||||
#[schema(value_type = Option<Vec<String>>, example = json!(["id", "title", "description", "url"]))]
|
||||
pub displayed_attributes: WildcardSetting,
|
||||
|
||||
/// Fields in which to search for matching query words sorted by order of importance.
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsSearchableAttributes>)]
|
||||
#[schema(value_type = Option<Vec<String>>, example = json!(["title", "description"]))]
|
||||
pub searchable_attributes: WildcardSetting,
|
||||
|
||||
/// Attributes to use for faceting and filtering. See [Filtering and Faceted Search](https://www.meilisearch.com/docs/learn/filtering_and_sorting/search_with_facet_filters).
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsFilterableAttributes>)]
|
||||
pub filterable_attributes: Setting<BTreeSet<String>>,
|
||||
#[schema(value_type = Option<Vec<FilterableAttributesRule>>, example = json!(["release_date", "genre"]))]
|
||||
pub filterable_attributes: Setting<Vec<FilterableAttributesRule>>,
|
||||
/// Attributes to use when sorting search results.
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsSortableAttributes>)]
|
||||
#[schema(value_type = Option<Vec<String>>, example = json!(["release_date"]))]
|
||||
pub sortable_attributes: Setting<BTreeSet<String>>,
|
||||
/// List of ranking rules sorted by order of importance. The order is customizable.
|
||||
/// [A list of ordered built-in ranking rules](https://www.meilisearch.com/docs/learn/relevancy/relevancy).
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsRankingRules>)]
|
||||
#[schema(value_type = Option<Vec<String>>, example = json!([RankingRuleView::Words, RankingRuleView::Typo, RankingRuleView::Proximity, RankingRuleView::Attribute, RankingRuleView::Exactness]))]
|
||||
pub ranking_rules: Setting<Vec<RankingRuleView>>,
|
||||
/// List of words ignored when present in search queries.
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsStopWords>)]
|
||||
#[schema(value_type = Option<Vec<String>>, example = json!(["the", "a", "them", "their"]))]
|
||||
pub stop_words: Setting<BTreeSet<String>>,
|
||||
/// List of characters not delimiting where one term begins and ends.
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsNonSeparatorTokens>)]
|
||||
#[schema(value_type = Option<Vec<String>>, example = json!([" ", "\n"]))]
|
||||
pub non_separator_tokens: Setting<BTreeSet<String>>,
|
||||
/// List of characters delimiting where one term begins and ends.
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsSeparatorTokens>)]
|
||||
#[schema(value_type = Option<Vec<String>>, example = json!(["S"]))]
|
||||
pub separator_tokens: Setting<BTreeSet<String>>,
|
||||
/// List of strings Meilisearch should parse as a single term.
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsDictionary>)]
|
||||
#[schema(value_type = Option<Vec<String>>, example = json!(["iPhone pro"]))]
|
||||
pub dictionary: Setting<BTreeSet<String>>,
|
||||
/// List of associated words treated similarly. A word associated to an array of word as synonyms.
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsSynonyms>)]
|
||||
#[schema(value_type = Option<BTreeMap<String, Vec<String>>>, example = json!({ "he": ["she", "they", "them"], "phone": ["iPhone", "android"]}))]
|
||||
pub synonyms: Setting<BTreeMap<String, Vec<String>>>,
|
||||
/// Search returns documents with distinct (different) values of the given field.
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsDistinctAttribute>)]
|
||||
#[schema(value_type = Option<String>, example = json!("sku"))]
|
||||
pub distinct_attribute: Setting<String>,
|
||||
/// Precision level when calculating the proximity ranking rule.
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsProximityPrecision>)]
|
||||
#[schema(value_type = Option<String>, example = json!(ProximityPrecisionView::ByAttribute))]
|
||||
pub proximity_precision: Setting<ProximityPrecisionView>,
|
||||
/// Customize typo tolerance feature.
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsTypoTolerance>)]
|
||||
#[schema(value_type = Option<TypoSettings>, example = json!({ "enabled": true, "disableOnAttributes": ["title"]}))]
|
||||
pub typo_tolerance: Setting<TypoSettings>,
|
||||
/// Faceting settings.
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsFaceting>)]
|
||||
#[schema(value_type = Option<FacetingSettings>, example = json!({ "maxValuesPerFacet": 10, "sortFacetValuesBy": { "genre": FacetValuesSort::Count }}))]
|
||||
pub faceting: Setting<FacetingSettings>,
|
||||
/// Pagination settings.
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsPagination>)]
|
||||
#[schema(value_type = Option<PaginationSettings>, example = json!({ "maxValuesPerFacet": 10, "sortFacetValuesBy": { "genre": FacetValuesSort::Count }}))]
|
||||
pub pagination: Setting<PaginationSettings>,
|
||||
|
||||
/// Embedder required for performing semantic search queries.
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsEmbedders>)]
|
||||
pub embedders: Setting<BTreeMap<String, Setting<milli::vector::settings::EmbeddingSettings>>>,
|
||||
#[schema(value_type = Option<BTreeMap<String, SettingEmbeddingSettings>>)]
|
||||
pub embedders: Setting<BTreeMap<String, SettingEmbeddingSettings>>,
|
||||
/// Maximum duration of a search query.
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsSearchCutoffMs>)]
|
||||
#[schema(value_type = Option<u64>, example = json!(50))]
|
||||
pub search_cutoff_ms: Setting<u64>,
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsLocalizedAttributes>)]
|
||||
#[schema(value_type = Option<Vec<LocalizedAttributesRuleView>>, example = json!(50))]
|
||||
pub localized_attributes: Setting<Vec<LocalizedAttributesRuleView>>,
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsFacetSearch>)]
|
||||
#[schema(value_type = Option<bool>, example = json!(true))]
|
||||
pub facet_search: Setting<bool>,
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsPrefixSearch>)]
|
||||
#[schema(value_type = Option<PrefixSearchSettings>, example = json!("Hemlo"))]
|
||||
pub prefix_search: Setting<PrefixSearchSettings>,
|
||||
|
||||
#[serde(skip)]
|
||||
#[deserr(skip)]
|
||||
@ -215,7 +301,7 @@ impl<T> Settings<T> {
|
||||
};
|
||||
|
||||
for mut embedder in embedders.values_mut() {
|
||||
let Setting::Set(embedder) = &mut embedder else {
|
||||
let SettingEmbeddingSettings { inner: Setting::Set(embedder) } = &mut embedder else {
|
||||
continue;
|
||||
};
|
||||
|
||||
@ -266,6 +352,8 @@ impl Settings<Checked> {
|
||||
embedders: Setting::Reset,
|
||||
search_cutoff_ms: Setting::Reset,
|
||||
localized_attributes: Setting::Reset,
|
||||
facet_search: Setting::Reset,
|
||||
prefix_search: Setting::Reset,
|
||||
_kind: PhantomData,
|
||||
}
|
||||
}
|
||||
@ -290,6 +378,8 @@ impl Settings<Checked> {
|
||||
embedders,
|
||||
search_cutoff_ms,
|
||||
localized_attributes: localized_attributes_rules,
|
||||
facet_search,
|
||||
prefix_search,
|
||||
_kind,
|
||||
} = self;
|
||||
|
||||
@ -312,6 +402,8 @@ impl Settings<Checked> {
|
||||
embedders,
|
||||
search_cutoff_ms,
|
||||
localized_attributes: localized_attributes_rules,
|
||||
facet_search,
|
||||
prefix_search,
|
||||
_kind: PhantomData,
|
||||
}
|
||||
}
|
||||
@ -360,6 +452,8 @@ impl Settings<Unchecked> {
|
||||
embedders: self.embedders,
|
||||
search_cutoff_ms: self.search_cutoff_ms,
|
||||
localized_attributes: self.localized_attributes,
|
||||
facet_search: self.facet_search,
|
||||
prefix_search: self.prefix_search,
|
||||
_kind: PhantomData,
|
||||
}
|
||||
}
|
||||
@ -372,8 +466,9 @@ impl Settings<Unchecked> {
|
||||
let Setting::Set(mut configs) = self.embedders else { return Ok(self) };
|
||||
for (name, config) in configs.iter_mut() {
|
||||
let config_to_check = std::mem::take(config);
|
||||
let checked_config = milli::update::validate_embedding_settings(config_to_check, name)?;
|
||||
*config = checked_config
|
||||
let checked_config =
|
||||
milli::update::validate_embedding_settings(config_to_check.inner, name)?;
|
||||
*config = SettingEmbeddingSettings { inner: checked_config };
|
||||
}
|
||||
self.embedders = Setting::Set(configs);
|
||||
Ok(self)
|
||||
@ -433,6 +528,8 @@ impl Settings<Unchecked> {
|
||||
Setting::Set(this)
|
||||
}
|
||||
},
|
||||
prefix_search: other.prefix_search.or(self.prefix_search),
|
||||
facet_search: other.facet_search.or(self.facet_search),
|
||||
_kind: PhantomData,
|
||||
}
|
||||
}
|
||||
@ -469,6 +566,8 @@ pub fn apply_settings_to_builder(
|
||||
embedders,
|
||||
search_cutoff_ms,
|
||||
localized_attributes: localized_attributes_rules,
|
||||
facet_search,
|
||||
prefix_search,
|
||||
_kind,
|
||||
} = settings;
|
||||
|
||||
@ -647,7 +746,9 @@ pub fn apply_settings_to_builder(
|
||||
}
|
||||
|
||||
match embedders {
|
||||
Setting::Set(value) => builder.set_embedder_settings(value.clone()),
|
||||
Setting::Set(value) => builder.set_embedder_settings(
|
||||
value.iter().map(|(k, v)| (k.clone(), v.inner.clone())).collect(),
|
||||
),
|
||||
Setting::Reset => builder.reset_embedder_settings(),
|
||||
Setting::NotSet => (),
|
||||
}
|
||||
@ -657,6 +758,20 @@ pub fn apply_settings_to_builder(
|
||||
Setting::Reset => builder.reset_search_cutoff(),
|
||||
Setting::NotSet => (),
|
||||
}
|
||||
|
||||
match prefix_search {
|
||||
Setting::Set(prefix_search) => {
|
||||
builder.set_prefix_search(PrefixSearch::from(*prefix_search))
|
||||
}
|
||||
Setting::Reset => builder.reset_prefix_search(),
|
||||
Setting::NotSet => (),
|
||||
}
|
||||
|
||||
match facet_search {
|
||||
Setting::Set(facet_search) => builder.set_facet_search(*facet_search),
|
||||
Setting::Reset => builder.reset_facet_search(),
|
||||
Setting::NotSet => (),
|
||||
}
|
||||
}
|
||||
|
||||
pub enum SecretPolicy {
|
||||
@ -676,7 +791,7 @@ pub fn settings(
|
||||
.user_defined_searchable_fields(rtxn)?
|
||||
.map(|fields| fields.into_iter().map(String::from).collect());
|
||||
|
||||
let filterable_attributes = index.filterable_fields(rtxn)?.into_iter().collect();
|
||||
let filterable_attributes = index.filterable_attributes_rules(rtxn)?.into_iter().collect();
|
||||
|
||||
let sortable_attributes = index.sortable_fields(rtxn)?.into_iter().collect();
|
||||
|
||||
@ -747,14 +862,20 @@ pub fn settings(
|
||||
let embedders: BTreeMap<_, _> = index
|
||||
.embedding_configs(rtxn)?
|
||||
.into_iter()
|
||||
.map(|IndexEmbeddingConfig { name, config, .. }| (name, Setting::Set(config.into())))
|
||||
.map(|IndexEmbeddingConfig { name, config, .. }| {
|
||||
(name, SettingEmbeddingSettings { inner: Setting::Set(config.into()) })
|
||||
})
|
||||
.collect();
|
||||
let embedders = if embedders.is_empty() { Setting::NotSet } else { Setting::Set(embedders) };
|
||||
let embedders = Setting::Set(embedders);
|
||||
|
||||
let search_cutoff_ms = index.search_cutoff(rtxn)?;
|
||||
|
||||
let localized_attributes_rules = index.localized_attributes_rules(rtxn)?;
|
||||
|
||||
let prefix_search = index.prefix_search(rtxn)?.map(PrefixSearchSettings::from);
|
||||
|
||||
let facet_search = index.facet_search(rtxn)?;
|
||||
|
||||
let mut settings = Settings {
|
||||
displayed_attributes: match displayed_attributes {
|
||||
Some(attrs) => Setting::Set(attrs),
|
||||
@ -791,17 +912,18 @@ pub fn settings(
|
||||
Some(rules) => Setting::Set(rules.into_iter().map(|r| r.into()).collect()),
|
||||
None => Setting::Reset,
|
||||
},
|
||||
prefix_search: Setting::Set(prefix_search.unwrap_or_default()),
|
||||
facet_search: Setting::Set(facet_search),
|
||||
_kind: PhantomData,
|
||||
};
|
||||
|
||||
if let SecretPolicy::HideSecrets = secret_policy {
|
||||
settings.hide_secrets()
|
||||
}
|
||||
|
||||
Ok(settings)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Deserr)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Deserr, ToSchema)]
|
||||
#[deserr(try_from(&String) = FromStr::from_str -> CriterionError)]
|
||||
pub enum RankingRuleView {
|
||||
/// Sorted by decreasing number of matched query terms.
|
||||
@ -838,7 +960,7 @@ impl<'de> Deserialize<'de> for RankingRuleView {
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
struct Visitor;
|
||||
impl<'de> serde::de::Visitor<'de> for Visitor {
|
||||
impl serde::de::Visitor<'_> for Visitor {
|
||||
type Value = RankingRuleView;
|
||||
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
write!(formatter, "the name of a valid ranking rule (string)")
|
||||
@ -897,7 +1019,7 @@ impl From<RankingRuleView> for Criterion {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Deserr, Serialize, Deserialize)]
|
||||
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Deserr, Serialize, Deserialize, ToSchema)]
|
||||
#[serde(deny_unknown_fields, rename_all = "camelCase")]
|
||||
#[deserr(error = DeserrJsonError<InvalidSettingsProximityPrecision>, rename_all = camelCase, deny_unknown_fields)]
|
||||
pub enum ProximityPrecisionView {
|
||||
@ -964,6 +1086,33 @@ impl std::ops::Deref for WildcardSetting {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Deserr, Serialize, Deserialize, ToSchema)]
|
||||
#[serde(deny_unknown_fields, rename_all = "camelCase")]
|
||||
#[schema(rename_all = "camelCase")]
|
||||
#[deserr(error = DeserrJsonError<InvalidSettingsPrefixSearch>, rename_all = camelCase, deny_unknown_fields)]
|
||||
pub enum PrefixSearchSettings {
|
||||
#[default]
|
||||
IndexingTime,
|
||||
Disabled,
|
||||
}
|
||||
|
||||
impl From<PrefixSearch> for PrefixSearchSettings {
|
||||
fn from(value: PrefixSearch) -> Self {
|
||||
match value {
|
||||
PrefixSearch::IndexingTime => PrefixSearchSettings::IndexingTime,
|
||||
PrefixSearch::Disabled => PrefixSearchSettings::Disabled,
|
||||
}
|
||||
}
|
||||
}
|
||||
impl From<PrefixSearchSettings> for PrefixSearch {
|
||||
fn from(value: PrefixSearchSettings) -> Self {
|
||||
match value {
|
||||
PrefixSearchSettings::IndexingTime => PrefixSearch::IndexingTime,
|
||||
PrefixSearchSettings::Disabled => PrefixSearch::Disabled,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test {
|
||||
use super::*;
|
||||
@ -990,6 +1139,8 @@ pub(crate) mod test {
|
||||
embedders: Setting::NotSet,
|
||||
localized_attributes: Setting::NotSet,
|
||||
search_cutoff_ms: Setting::NotSet,
|
||||
facet_search: Setting::NotSet,
|
||||
prefix_search: Setting::NotSet,
|
||||
_kind: PhantomData::<Unchecked>,
|
||||
};
|
||||
|
||||
@ -1019,6 +1170,8 @@ pub(crate) mod test {
|
||||
embedders: Setting::NotSet,
|
||||
localized_attributes: Setting::NotSet,
|
||||
search_cutoff_ms: Setting::NotSet,
|
||||
facet_search: Setting::NotSet,
|
||||
prefix_search: Setting::NotSet,
|
||||
_kind: PhantomData::<Unchecked>,
|
||||
};
|
||||
|
||||
|
||||
@ -6,6 +6,7 @@ use std::str::FromStr;
|
||||
use deserr::{DeserializeError, Deserr, MergeWithError, ValueKind};
|
||||
use serde::de::Visitor;
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
use utoipa::PartialSchema;
|
||||
|
||||
use crate::deserr::query_params::FromQueryParameter;
|
||||
|
||||
@ -65,7 +66,7 @@ where
|
||||
/// not supported on untagged enums.
|
||||
struct StarOrVisitor<T>(PhantomData<T>);
|
||||
|
||||
impl<'de, T, FE> Visitor<'de> for StarOrVisitor<T>
|
||||
impl<T, FE> Visitor<'_> for StarOrVisitor<T>
|
||||
where
|
||||
T: FromStr<Err = FE>,
|
||||
FE: fmt::Display,
|
||||
@ -229,7 +230,7 @@ pub enum OptionStarOrList<T> {
|
||||
List(Vec<T>),
|
||||
}
|
||||
|
||||
impl<T> OptionStarOrList<T> {
|
||||
impl<T: PartialSchema> OptionStarOrList<T> {
|
||||
pub fn is_some(&self) -> bool {
|
||||
match self {
|
||||
Self::None => false,
|
||||
|
||||
@ -1,32 +1,49 @@
|
||||
use milli::Object;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use time::{Duration, OffsetDateTime};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use crate::batches::BatchId;
|
||||
use crate::error::ResponseError;
|
||||
use crate::settings::{Settings, Unchecked};
|
||||
use crate::tasks::{serialize_duration, Details, IndexSwap, Kind, Status, Task, TaskId};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, ToSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[schema(rename_all = "camelCase")]
|
||||
pub struct TaskView {
|
||||
/// The unique sequential identifier of the task.
|
||||
#[schema(value_type = u32, example = 4312)]
|
||||
pub uid: TaskId,
|
||||
/// The unique identifier of the index where this task is operated.
|
||||
#[schema(value_type = Option<u32>, example = json!("movies"))]
|
||||
pub batch_uid: Option<BatchId>,
|
||||
#[serde(default)]
|
||||
pub index_uid: Option<String>,
|
||||
pub status: Status,
|
||||
/// The type of the task.
|
||||
#[serde(rename = "type")]
|
||||
pub kind: Kind,
|
||||
/// The uid of the task that performed the taskCancelation if the task has been canceled.
|
||||
#[schema(value_type = Option<u32>, example = json!(4326))]
|
||||
pub canceled_by: Option<TaskId>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub details: Option<DetailsView>,
|
||||
pub error: Option<ResponseError>,
|
||||
/// Total elasped time the engine was in processing state expressed as a `ISO-8601` duration format.
|
||||
#[schema(value_type = Option<String>, example = json!(null))]
|
||||
#[serde(serialize_with = "serialize_duration", default)]
|
||||
pub duration: Option<Duration>,
|
||||
/// An `RFC 3339` format for date/time/duration.
|
||||
#[schema(value_type = String, example = json!("2024-08-08_14:12:09.393Z"))]
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
pub enqueued_at: OffsetDateTime,
|
||||
/// An `RFC 3339` format for date/time/duration.
|
||||
#[schema(value_type = String, example = json!("2024-08-08_14:12:09.393Z"))]
|
||||
#[serde(with = "time::serde::rfc3339::option", default)]
|
||||
pub started_at: Option<OffsetDateTime>,
|
||||
/// An `RFC 3339` format for date/time/duration.
|
||||
#[schema(value_type = String, example = json!("2024-08-08_14:12:09.393Z"))]
|
||||
#[serde(with = "time::serde::rfc3339::option", default)]
|
||||
pub finished_at: Option<OffsetDateTime>,
|
||||
}
|
||||
@ -50,40 +67,57 @@ impl TaskView {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
|
||||
#[derive(Default, Debug, PartialEq, Eq, Clone, Serialize, Deserialize, ToSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[schema(rename_all = "camelCase")]
|
||||
pub struct DetailsView {
|
||||
/// Number of documents received for documentAdditionOrUpdate task.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub received_documents: Option<u64>,
|
||||
/// Number of documents finally indexed for documentAdditionOrUpdate task or a documentAdditionOrUpdate batch of tasks.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub indexed_documents: Option<Option<u64>>,
|
||||
/// Number of documents edited for editDocumentByFunction task.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub edited_documents: Option<Option<u64>>,
|
||||
/// Value for the primaryKey field encountered if any for indexCreation or indexUpdate task.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub primary_key: Option<Option<String>>,
|
||||
/// Number of provided document ids for the documentDeletion task.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub provided_ids: Option<usize>,
|
||||
/// Number of documents finally deleted for documentDeletion and indexDeletion tasks.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub deleted_documents: Option<Option<u64>>,
|
||||
/// Number of tasks that match the request for taskCancelation or taskDeletion tasks.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub matched_tasks: Option<u64>,
|
||||
/// Number of tasks canceled for taskCancelation.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub canceled_tasks: Option<Option<u64>>,
|
||||
/// Number of tasks deleted for taskDeletion.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub deleted_tasks: Option<Option<u64>>,
|
||||
/// Original filter query for taskCancelation or taskDeletion tasks.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub original_filter: Option<Option<String>>,
|
||||
/// Identifier generated for the dump for dumpCreation task.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub dump_uid: Option<Option<String>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub context: Option<Option<Object>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub function: Option<String>,
|
||||
/// [Learn more about the settings in this guide](https://www.meilisearch.com/docs/reference/api/settings).
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[serde(flatten)]
|
||||
pub settings: Option<Box<Settings<Unchecked>>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub swaps: Option<Vec<IndexSwap>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub upgrade_from: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub upgrade_to: Option<String>,
|
||||
}
|
||||
|
||||
impl DetailsView {
|
||||
@ -204,6 +238,18 @@ impl DetailsView {
|
||||
Some(left)
|
||||
}
|
||||
},
|
||||
// We want the earliest version
|
||||
upgrade_from: match (self.upgrade_from.clone(), other.upgrade_from.clone()) {
|
||||
(None, None) => None,
|
||||
(None, Some(from)) | (Some(from), None) => Some(from),
|
||||
(Some(from), Some(_)) => Some(from),
|
||||
},
|
||||
// And the latest
|
||||
upgrade_to: match (self.upgrade_to.clone(), other.upgrade_to.clone()) {
|
||||
(None, None) => None,
|
||||
(None, Some(to)) | (Some(to), None) => Some(to),
|
||||
(Some(_), Some(to)) => Some(to),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -281,6 +327,11 @@ impl From<Details> for DetailsView {
|
||||
Details::IndexSwap { swaps } => {
|
||||
DetailsView { swaps: Some(swaps), ..Default::default() }
|
||||
}
|
||||
Details::UpgradeDatabase { from, to } => DetailsView {
|
||||
upgrade_from: Some(format!("v{}.{}.{}", from.0, from.1, from.2)),
|
||||
upgrade_to: Some(format!("v{}.{}.{}", to.0, to.1, to.2)),
|
||||
..Default::default()
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -4,19 +4,19 @@ use std::fmt::{Display, Write};
|
||||
use std::str::FromStr;
|
||||
|
||||
use enum_iterator::Sequence;
|
||||
use milli::update::new::indexer::document_changes::Progress;
|
||||
use milli::update::IndexDocumentsMethod;
|
||||
use milli::Object;
|
||||
use roaring::RoaringBitmap;
|
||||
use serde::{Deserialize, Serialize, Serializer};
|
||||
use time::{Duration, OffsetDateTime};
|
||||
use utoipa::ToSchema;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::batches::BatchId;
|
||||
use crate::error::ResponseError;
|
||||
use crate::keys::Key;
|
||||
use crate::settings::{Settings, Unchecked};
|
||||
use crate::InstanceUid;
|
||||
use crate::{versioning, InstanceUid};
|
||||
|
||||
pub type TaskId = u32;
|
||||
|
||||
@ -41,62 +41,6 @@ pub struct Task {
|
||||
pub kind: KindWithContent,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TaskProgress {
|
||||
pub current_step: &'static str,
|
||||
pub finished_steps: u16,
|
||||
pub total_steps: u16,
|
||||
pub finished_substeps: Option<u32>,
|
||||
pub total_substeps: Option<u32>,
|
||||
}
|
||||
|
||||
impl Default for TaskProgress {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl TaskProgress {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
current_step: "start",
|
||||
finished_steps: 0,
|
||||
total_steps: 1,
|
||||
finished_substeps: None,
|
||||
total_substeps: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update(&mut self, progress: Progress) -> TaskProgress {
|
||||
if self.finished_steps > progress.finished_steps {
|
||||
return *self;
|
||||
}
|
||||
|
||||
if self.current_step != progress.step_name {
|
||||
self.current_step = progress.step_name
|
||||
}
|
||||
|
||||
self.total_steps = progress.total_steps;
|
||||
|
||||
if self.finished_steps < progress.finished_steps {
|
||||
self.finished_substeps = None;
|
||||
self.total_substeps = None;
|
||||
}
|
||||
self.finished_steps = progress.finished_steps;
|
||||
if let Some((finished_substeps, total_substeps)) = progress.finished_total_substep {
|
||||
if let Some(task_finished_substeps) = self.finished_substeps {
|
||||
if task_finished_substeps > finished_substeps {
|
||||
return *self;
|
||||
}
|
||||
}
|
||||
self.finished_substeps = Some(finished_substeps);
|
||||
self.total_substeps = Some(total_substeps);
|
||||
}
|
||||
*self
|
||||
}
|
||||
}
|
||||
|
||||
impl Task {
|
||||
pub fn index_uid(&self) -> Option<&str> {
|
||||
use KindWithContent::*;
|
||||
@ -106,6 +50,7 @@ impl Task {
|
||||
| SnapshotCreation
|
||||
| TaskCancelation { .. }
|
||||
| TaskDeletion { .. }
|
||||
| UpgradeDatabase { .. }
|
||||
| IndexSwap { .. } => None,
|
||||
DocumentAdditionOrUpdate { index_uid, .. }
|
||||
| DocumentEdition { index_uid, .. }
|
||||
@ -140,7 +85,8 @@ impl Task {
|
||||
| KindWithContent::TaskCancelation { .. }
|
||||
| KindWithContent::TaskDeletion { .. }
|
||||
| KindWithContent::DumpCreation { .. }
|
||||
| KindWithContent::SnapshotCreation => None,
|
||||
| KindWithContent::SnapshotCreation
|
||||
| KindWithContent::UpgradeDatabase { .. } => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -206,9 +152,12 @@ pub enum KindWithContent {
|
||||
instance_uid: Option<InstanceUid>,
|
||||
},
|
||||
SnapshotCreation,
|
||||
UpgradeDatabase {
|
||||
from: (u32, u32, u32),
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct IndexSwap {
|
||||
pub indexes: (String, String),
|
||||
@ -231,6 +180,7 @@ impl KindWithContent {
|
||||
KindWithContent::TaskDeletion { .. } => Kind::TaskDeletion,
|
||||
KindWithContent::DumpCreation { .. } => Kind::DumpCreation,
|
||||
KindWithContent::SnapshotCreation => Kind::SnapshotCreation,
|
||||
KindWithContent::UpgradeDatabase { .. } => Kind::UpgradeDatabase,
|
||||
}
|
||||
}
|
||||
|
||||
@ -241,7 +191,8 @@ impl KindWithContent {
|
||||
DumpCreation { .. }
|
||||
| SnapshotCreation
|
||||
| TaskCancelation { .. }
|
||||
| TaskDeletion { .. } => vec![],
|
||||
| TaskDeletion { .. }
|
||||
| UpgradeDatabase { .. } => vec![],
|
||||
DocumentAdditionOrUpdate { index_uid, .. }
|
||||
| DocumentEdition { index_uid, .. }
|
||||
| DocumentDeletion { index_uid, .. }
|
||||
@ -318,6 +269,14 @@ impl KindWithContent {
|
||||
}),
|
||||
KindWithContent::DumpCreation { .. } => Some(Details::Dump { dump_uid: None }),
|
||||
KindWithContent::SnapshotCreation => None,
|
||||
KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase {
|
||||
from: (from.0, from.1, from.2),
|
||||
to: (
|
||||
versioning::VERSION_MAJOR.parse().unwrap(),
|
||||
versioning::VERSION_MINOR.parse().unwrap(),
|
||||
versioning::VERSION_PATCH.parse().unwrap(),
|
||||
),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@ -376,6 +335,14 @@ impl KindWithContent {
|
||||
}),
|
||||
KindWithContent::DumpCreation { .. } => Some(Details::Dump { dump_uid: None }),
|
||||
KindWithContent::SnapshotCreation => None,
|
||||
KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase {
|
||||
from: *from,
|
||||
to: (
|
||||
versioning::VERSION_MAJOR.parse().unwrap(),
|
||||
versioning::VERSION_MINOR.parse().unwrap(),
|
||||
versioning::VERSION_PATCH.parse().unwrap(),
|
||||
),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -416,13 +383,34 @@ impl From<&KindWithContent> for Option<Details> {
|
||||
}),
|
||||
KindWithContent::DumpCreation { .. } => Some(Details::Dump { dump_uid: None }),
|
||||
KindWithContent::SnapshotCreation => None,
|
||||
KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase {
|
||||
from: *from,
|
||||
to: (
|
||||
versioning::VERSION_MAJOR.parse().unwrap(),
|
||||
versioning::VERSION_MINOR.parse().unwrap(),
|
||||
versioning::VERSION_PATCH.parse().unwrap(),
|
||||
),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The status of a task.
|
||||
#[derive(
|
||||
Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Sequence, PartialOrd, Ord,
|
||||
Debug,
|
||||
Clone,
|
||||
Copy,
|
||||
PartialEq,
|
||||
Eq,
|
||||
Hash,
|
||||
Serialize,
|
||||
Deserialize,
|
||||
Sequence,
|
||||
PartialOrd,
|
||||
Ord,
|
||||
ToSchema,
|
||||
)]
|
||||
#[schema(example = json!(Status::Processing))]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Status {
|
||||
Enqueued,
|
||||
@ -481,10 +469,23 @@ impl fmt::Display for ParseTaskStatusError {
|
||||
}
|
||||
impl std::error::Error for ParseTaskStatusError {}
|
||||
|
||||
/// The type of the task.
|
||||
#[derive(
|
||||
Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Sequence, PartialOrd, Ord,
|
||||
Debug,
|
||||
Clone,
|
||||
Copy,
|
||||
PartialEq,
|
||||
Eq,
|
||||
Hash,
|
||||
Serialize,
|
||||
Deserialize,
|
||||
Sequence,
|
||||
PartialOrd,
|
||||
Ord,
|
||||
ToSchema,
|
||||
)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[schema(rename_all = "camelCase", example = json!(enum_iterator::all::<Kind>().collect::<Vec<_>>()))]
|
||||
pub enum Kind {
|
||||
DocumentAdditionOrUpdate,
|
||||
DocumentEdition,
|
||||
@ -498,6 +499,7 @@ pub enum Kind {
|
||||
TaskDeletion,
|
||||
DumpCreation,
|
||||
SnapshotCreation,
|
||||
UpgradeDatabase,
|
||||
}
|
||||
|
||||
impl Kind {
|
||||
@ -514,6 +516,7 @@ impl Kind {
|
||||
| Kind::TaskCancelation
|
||||
| Kind::TaskDeletion
|
||||
| Kind::DumpCreation
|
||||
| Kind::UpgradeDatabase
|
||||
| Kind::SnapshotCreation => false,
|
||||
}
|
||||
}
|
||||
@ -533,6 +536,7 @@ impl Display for Kind {
|
||||
Kind::TaskDeletion => write!(f, "taskDeletion"),
|
||||
Kind::DumpCreation => write!(f, "dumpCreation"),
|
||||
Kind::SnapshotCreation => write!(f, "snapshotCreation"),
|
||||
Kind::UpgradeDatabase => write!(f, "upgradeDatabase"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -564,6 +568,8 @@ impl FromStr for Kind {
|
||||
Ok(Kind::DumpCreation)
|
||||
} else if kind.eq_ignore_ascii_case("snapshotCreation") {
|
||||
Ok(Kind::SnapshotCreation)
|
||||
} else if kind.eq_ignore_ascii_case("upgradeDatabase") {
|
||||
Ok(Kind::UpgradeDatabase)
|
||||
} else {
|
||||
Err(ParseTaskKindError(kind.to_owned()))
|
||||
}
|
||||
@ -637,6 +643,10 @@ pub enum Details {
|
||||
IndexSwap {
|
||||
swaps: Vec<IndexSwap>,
|
||||
},
|
||||
UpgradeDatabase {
|
||||
from: (u32, u32, u32),
|
||||
to: (u32, u32, u32),
|
||||
},
|
||||
}
|
||||
|
||||
impl Details {
|
||||
@ -657,6 +667,7 @@ impl Details {
|
||||
Self::SettingsUpdate { .. }
|
||||
| Self::IndexInfo { .. }
|
||||
| Self::Dump { .. }
|
||||
| Self::UpgradeDatabase { .. }
|
||||
| Self::IndexSwap { .. } => (),
|
||||
}
|
||||
|
||||
@ -717,7 +728,9 @@ pub fn serialize_duration<S: Serializer>(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::Details;
|
||||
use std::str::FromStr;
|
||||
|
||||
use super::{Details, Kind};
|
||||
use crate::heed::types::SerdeJson;
|
||||
use crate::heed::{BytesDecode, BytesEncode};
|
||||
|
||||
@ -733,4 +746,13 @@ mod tests {
|
||||
meili_snap::snapshot!(format!("{:?}", details), @r###"TaskDeletion { matched_tasks: 1, deleted_tasks: None, original_filter: "hello" }"###);
|
||||
meili_snap::snapshot!(format!("{:?}", deserialised), @r###"TaskDeletion { matched_tasks: 1, deleted_tasks: None, original_filter: "hello" }"###);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_kind_can_be_from_str() {
|
||||
for kind in enum_iterator::all::<Kind>() {
|
||||
let s = kind.to_string();
|
||||
let k = Kind::from_str(&s).map_err(|e| format!("Could not from_str {s}: {e}")).unwrap();
|
||||
assert_eq!(kind, k, "{kind}.to_string() returned {s} which was parsed as {k}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,16 +1,19 @@
|
||||
use std::fs;
|
||||
use std::io::{self, ErrorKind};
|
||||
use std::io::{ErrorKind, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use milli::heed;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
/// The name of the file that contains the version of the database.
|
||||
pub const VERSION_FILE_NAME: &str = "VERSION";
|
||||
|
||||
static VERSION_MAJOR: &str = env!("CARGO_PKG_VERSION_MAJOR");
|
||||
static VERSION_MINOR: &str = env!("CARGO_PKG_VERSION_MINOR");
|
||||
static VERSION_PATCH: &str = env!("CARGO_PKG_VERSION_PATCH");
|
||||
pub static VERSION_MAJOR: &str = env!("CARGO_PKG_VERSION_MAJOR");
|
||||
pub static VERSION_MINOR: &str = env!("CARGO_PKG_VERSION_MINOR");
|
||||
pub static VERSION_PATCH: &str = env!("CARGO_PKG_VERSION_PATCH");
|
||||
|
||||
/// Persists the version of the current Meilisearch binary to a VERSION file
|
||||
pub fn create_current_version_file(db_path: &Path) -> io::Result<()> {
|
||||
pub fn create_current_version_file(db_path: &Path) -> anyhow::Result<()> {
|
||||
create_version_file(db_path, VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH)
|
||||
}
|
||||
|
||||
@ -19,39 +22,50 @@ pub fn create_version_file(
|
||||
major: &str,
|
||||
minor: &str,
|
||||
patch: &str,
|
||||
) -> io::Result<()> {
|
||||
) -> anyhow::Result<()> {
|
||||
let version_path = db_path.join(VERSION_FILE_NAME);
|
||||
fs::write(version_path, format!("{}.{}.{}", major, minor, patch))
|
||||
}
|
||||
|
||||
/// Ensures Meilisearch version is compatible with the database, returns an error versions mismatch.
|
||||
pub fn check_version_file(db_path: &Path) -> anyhow::Result<()> {
|
||||
let (major, minor, patch) = get_version(db_path)?;
|
||||
|
||||
if major != VERSION_MAJOR || minor != VERSION_MINOR {
|
||||
return Err(VersionFileError::VersionMismatch { major, minor, patch }.into());
|
||||
}
|
||||
|
||||
// In order to persist the file later we must create it in the `data.ms` and not in `/tmp`
|
||||
let mut file = NamedTempFile::new_in(db_path)?;
|
||||
file.write_all(format!("{}.{}.{}", major, minor, patch).as_bytes())?;
|
||||
file.flush()?;
|
||||
file.persist(version_path)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_version(db_path: &Path) -> Result<(String, String, String), VersionFileError> {
|
||||
pub fn get_version(db_path: &Path) -> Result<(u32, u32, u32), VersionFileError> {
|
||||
let version_path = db_path.join(VERSION_FILE_NAME);
|
||||
|
||||
match fs::read_to_string(version_path) {
|
||||
Ok(version) => parse_version(&version),
|
||||
Err(error) => match error.kind() {
|
||||
ErrorKind::NotFound => Err(VersionFileError::MissingVersionFile),
|
||||
_ => Err(error.into()),
|
||||
_ => Err(anyhow::Error::from(error).into()),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_version(version: &str) -> Result<(String, String, String), VersionFileError> {
|
||||
let version_components = version.split('.').collect::<Vec<_>>();
|
||||
pub fn parse_version(version: &str) -> Result<(u32, u32, u32), VersionFileError> {
|
||||
let version_components = version.trim().split('.').collect::<Vec<_>>();
|
||||
let (major, minor, patch) = match &version_components[..] {
|
||||
[major, minor, patch] => (major.to_string(), minor.to_string(), patch.to_string()),
|
||||
_ => return Err(VersionFileError::MalformedVersionFile),
|
||||
[major, minor, patch] => (
|
||||
major.parse().map_err(|e| VersionFileError::MalformedVersionFile {
|
||||
context: format!("Could not parse the major: {e}"),
|
||||
})?,
|
||||
minor.parse().map_err(|e| VersionFileError::MalformedVersionFile {
|
||||
context: format!("Could not parse the minor: {e}"),
|
||||
})?,
|
||||
patch.parse().map_err(|e| VersionFileError::MalformedVersionFile {
|
||||
context: format!("Could not parse the patch: {e}"),
|
||||
})?,
|
||||
),
|
||||
_ => {
|
||||
return Err(VersionFileError::MalformedVersionFile {
|
||||
context: format!(
|
||||
"The version contains {} parts instead of 3 (major, minor and patch)",
|
||||
version_components.len()
|
||||
),
|
||||
})
|
||||
}
|
||||
};
|
||||
Ok((major, minor, patch))
|
||||
}
|
||||
@ -64,15 +78,21 @@ pub enum VersionFileError {
|
||||
env!("CARGO_PKG_VERSION").to_string()
|
||||
)]
|
||||
MissingVersionFile,
|
||||
#[error("Version file is corrupted and thus Meilisearch is unable to determine the version of the database.")]
|
||||
MalformedVersionFile,
|
||||
#[error("Version file is corrupted and thus Meilisearch is unable to determine the version of the database. {context}")]
|
||||
MalformedVersionFile { context: String },
|
||||
#[error(
|
||||
"Your database version ({major}.{minor}.{patch}) is incompatible with your current engine version ({}).\n\
|
||||
To migrate data between Meilisearch versions, please follow our guide on https://www.meilisearch.com/docs/learn/update_and_migration/updating.",
|
||||
env!("CARGO_PKG_VERSION").to_string()
|
||||
)]
|
||||
VersionMismatch { major: String, minor: String, patch: String },
|
||||
VersionMismatch { major: u32, minor: u32, patch: u32 },
|
||||
#[error("Database version {major}.{minor}.{patch} is higher than the Meilisearch version {VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_PATCH}. Downgrade is not supported")]
|
||||
DowngradeNotSupported { major: u32, minor: u32, patch: u32 },
|
||||
#[error("Database version {major}.{minor}.{patch} is too old for the experimental dumpless upgrade feature. Please generate a dump using the v{major}.{minor}.{patch} and import it in the v{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_PATCH}")]
|
||||
TooOldForAutomaticUpgrade { major: u32, minor: u32, patch: u32 },
|
||||
#[error("Error while modifying the database: {0}")]
|
||||
ErrorWhileModifyingTheDatabase(#[from] heed::Error),
|
||||
|
||||
#[error(transparent)]
|
||||
IoError(#[from] std::io::Error),
|
||||
AnyhowError(#[from] anyhow::Error),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user