mirror of
https://github.com/meilisearch/meilisearch.git
synced 2025-09-16 09:46:25 +00:00
Move crates under a sub folder to clean up the code
This commit is contained in:
310
crates/dump/src/reader/v4/errors.rs
Normal file
310
crates/dump/src/reader/v4/errors.rs
Normal file
@ -0,0 +1,310 @@
|
||||
use std::fmt;
|
||||
|
||||
use http::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ResponseError {
|
||||
#[serde(skip)]
|
||||
pub code: StatusCode,
|
||||
pub message: String,
|
||||
#[serde(rename = "code")]
|
||||
pub error_code: String,
|
||||
#[serde(rename = "type")]
|
||||
pub error_type: String,
|
||||
#[serde(rename = "link")]
|
||||
pub error_link: String,
|
||||
}
|
||||
|
||||
impl ResponseError {
|
||||
pub fn from_msg(message: String, code: Code) -> Self {
|
||||
Self {
|
||||
code: code.http(),
|
||||
message,
|
||||
error_code: code.err_code().error_name.to_string(),
|
||||
error_type: code.type_(),
|
||||
error_link: code.url(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ResponseError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
self.message.fmt(f)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ResponseError {}
|
||||
|
||||
impl<T> From<T> for ResponseError
|
||||
where
|
||||
T: ErrorCode,
|
||||
{
|
||||
fn from(other: T) -> Self {
|
||||
Self {
|
||||
code: other.http_status(),
|
||||
message: other.to_string(),
|
||||
error_code: other.error_name(),
|
||||
error_type: other.error_type(),
|
||||
error_link: other.error_url(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub trait ErrorCode: std::error::Error {
|
||||
fn error_code(&self) -> Code;
|
||||
|
||||
/// returns the HTTP status code ascociated with the error
|
||||
fn http_status(&self) -> StatusCode {
|
||||
self.error_code().http()
|
||||
}
|
||||
|
||||
/// returns the doc url ascociated with the error
|
||||
fn error_url(&self) -> String {
|
||||
self.error_code().url()
|
||||
}
|
||||
|
||||
/// returns error name, used as error code
|
||||
fn error_name(&self) -> String {
|
||||
self.error_code().name()
|
||||
}
|
||||
|
||||
/// return the error type
|
||||
fn error_type(&self) -> String {
|
||||
self.error_code().type_()
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::enum_variant_names)]
|
||||
enum ErrorType {
|
||||
InternalError,
|
||||
InvalidRequestError,
|
||||
AuthenticationError,
|
||||
}
|
||||
|
||||
impl fmt::Display for ErrorType {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
use ErrorType::*;
|
||||
|
||||
match self {
|
||||
InternalError => write!(f, "internal"),
|
||||
InvalidRequestError => write!(f, "invalid_request"),
|
||||
AuthenticationError => write!(f, "auth"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::enum_variant_names)]
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Copy)]
|
||||
pub enum Code {
|
||||
// index related error
|
||||
CreateIndex,
|
||||
IndexAlreadyExists,
|
||||
IndexNotFound,
|
||||
InvalidIndexUid,
|
||||
InvalidMinWordLengthForTypo,
|
||||
|
||||
// invalid state error
|
||||
InvalidState,
|
||||
MissingPrimaryKey,
|
||||
PrimaryKeyAlreadyPresent,
|
||||
|
||||
MaxFieldsLimitExceeded,
|
||||
MissingDocumentId,
|
||||
InvalidDocumentId,
|
||||
|
||||
Filter,
|
||||
Sort,
|
||||
|
||||
BadParameter,
|
||||
BadRequest,
|
||||
DatabaseSizeLimitReached,
|
||||
DocumentNotFound,
|
||||
Internal,
|
||||
InvalidGeoField,
|
||||
InvalidRankingRule,
|
||||
InvalidStore,
|
||||
InvalidToken,
|
||||
MissingAuthorizationHeader,
|
||||
NoSpaceLeftOnDevice,
|
||||
DumpNotFound,
|
||||
TaskNotFound,
|
||||
PayloadTooLarge,
|
||||
RetrieveDocument,
|
||||
SearchDocuments,
|
||||
UnsupportedMediaType,
|
||||
|
||||
DumpAlreadyInProgress,
|
||||
DumpProcessFailed,
|
||||
|
||||
InvalidContentType,
|
||||
MissingContentType,
|
||||
MalformedPayload,
|
||||
MissingPayload,
|
||||
|
||||
ApiKeyNotFound,
|
||||
MissingParameter,
|
||||
InvalidApiKeyActions,
|
||||
InvalidApiKeyIndexes,
|
||||
InvalidApiKeyExpiresAt,
|
||||
InvalidApiKeyDescription,
|
||||
|
||||
UnretrievableErrorCode,
|
||||
MalformedDump,
|
||||
}
|
||||
|
||||
impl Code {
|
||||
/// ascociate a `Code` variant to the actual ErrCode
|
||||
fn err_code(&self) -> ErrCode {
|
||||
use Code::*;
|
||||
|
||||
match self {
|
||||
// index related errors
|
||||
// create index is thrown on internal error while creating an index.
|
||||
CreateIndex => {
|
||||
ErrCode::internal("index_creation_failed", StatusCode::INTERNAL_SERVER_ERROR)
|
||||
}
|
||||
IndexAlreadyExists => ErrCode::invalid("index_already_exists", StatusCode::CONFLICT),
|
||||
// thrown when requesting an unexisting index
|
||||
IndexNotFound => ErrCode::invalid("index_not_found", StatusCode::NOT_FOUND),
|
||||
InvalidIndexUid => ErrCode::invalid("invalid_index_uid", StatusCode::BAD_REQUEST),
|
||||
|
||||
// invalid state error
|
||||
InvalidState => ErrCode::internal("invalid_state", StatusCode::INTERNAL_SERVER_ERROR),
|
||||
// thrown when no primary key has been set
|
||||
MissingPrimaryKey => {
|
||||
ErrCode::invalid("primary_key_inference_failed", StatusCode::BAD_REQUEST)
|
||||
}
|
||||
// error thrown when trying to set an already existing primary key
|
||||
PrimaryKeyAlreadyPresent => {
|
||||
ErrCode::invalid("index_primary_key_already_exists", StatusCode::BAD_REQUEST)
|
||||
}
|
||||
// invalid ranking rule
|
||||
InvalidRankingRule => ErrCode::invalid("invalid_ranking_rule", StatusCode::BAD_REQUEST),
|
||||
|
||||
// invalid database
|
||||
InvalidStore => {
|
||||
ErrCode::internal("invalid_store_file", StatusCode::INTERNAL_SERVER_ERROR)
|
||||
}
|
||||
|
||||
// invalid document
|
||||
MaxFieldsLimitExceeded => {
|
||||
ErrCode::invalid("max_fields_limit_exceeded", StatusCode::BAD_REQUEST)
|
||||
}
|
||||
MissingDocumentId => ErrCode::invalid("missing_document_id", StatusCode::BAD_REQUEST),
|
||||
InvalidDocumentId => ErrCode::invalid("invalid_document_id", StatusCode::BAD_REQUEST),
|
||||
|
||||
// error related to filters
|
||||
Filter => ErrCode::invalid("invalid_filter", StatusCode::BAD_REQUEST),
|
||||
// error related to sorts
|
||||
Sort => ErrCode::invalid("invalid_sort", StatusCode::BAD_REQUEST),
|
||||
|
||||
BadParameter => ErrCode::invalid("bad_parameter", StatusCode::BAD_REQUEST),
|
||||
BadRequest => ErrCode::invalid("bad_request", StatusCode::BAD_REQUEST),
|
||||
DatabaseSizeLimitReached => {
|
||||
ErrCode::internal("database_size_limit_reached", StatusCode::INTERNAL_SERVER_ERROR)
|
||||
}
|
||||
DocumentNotFound => ErrCode::invalid("document_not_found", StatusCode::NOT_FOUND),
|
||||
Internal => ErrCode::internal("internal", StatusCode::INTERNAL_SERVER_ERROR),
|
||||
InvalidGeoField => ErrCode::invalid("invalid_geo_field", StatusCode::BAD_REQUEST),
|
||||
InvalidToken => ErrCode::authentication("invalid_api_key", StatusCode::FORBIDDEN),
|
||||
MissingAuthorizationHeader => {
|
||||
ErrCode::authentication("missing_authorization_header", StatusCode::UNAUTHORIZED)
|
||||
}
|
||||
TaskNotFound => ErrCode::invalid("task_not_found", StatusCode::NOT_FOUND),
|
||||
DumpNotFound => ErrCode::invalid("dump_not_found", StatusCode::NOT_FOUND),
|
||||
NoSpaceLeftOnDevice => {
|
||||
ErrCode::internal("no_space_left_on_device", StatusCode::INTERNAL_SERVER_ERROR)
|
||||
}
|
||||
PayloadTooLarge => ErrCode::invalid("payload_too_large", StatusCode::PAYLOAD_TOO_LARGE),
|
||||
RetrieveDocument => {
|
||||
ErrCode::internal("unretrievable_document", StatusCode::BAD_REQUEST)
|
||||
}
|
||||
SearchDocuments => ErrCode::internal("search_error", StatusCode::BAD_REQUEST),
|
||||
UnsupportedMediaType => {
|
||||
ErrCode::invalid("unsupported_media_type", StatusCode::UNSUPPORTED_MEDIA_TYPE)
|
||||
}
|
||||
|
||||
// error related to dump
|
||||
DumpAlreadyInProgress => {
|
||||
ErrCode::invalid("dump_already_processing", StatusCode::CONFLICT)
|
||||
}
|
||||
DumpProcessFailed => {
|
||||
ErrCode::internal("dump_process_failed", StatusCode::INTERNAL_SERVER_ERROR)
|
||||
}
|
||||
MissingContentType => {
|
||||
ErrCode::invalid("missing_content_type", StatusCode::UNSUPPORTED_MEDIA_TYPE)
|
||||
}
|
||||
MalformedPayload => ErrCode::invalid("malformed_payload", StatusCode::BAD_REQUEST),
|
||||
InvalidContentType => {
|
||||
ErrCode::invalid("invalid_content_type", StatusCode::UNSUPPORTED_MEDIA_TYPE)
|
||||
}
|
||||
MissingPayload => ErrCode::invalid("missing_payload", StatusCode::BAD_REQUEST),
|
||||
|
||||
// error related to keys
|
||||
ApiKeyNotFound => ErrCode::invalid("api_key_not_found", StatusCode::NOT_FOUND),
|
||||
MissingParameter => ErrCode::invalid("missing_parameter", StatusCode::BAD_REQUEST),
|
||||
InvalidApiKeyActions => {
|
||||
ErrCode::invalid("invalid_api_key_actions", StatusCode::BAD_REQUEST)
|
||||
}
|
||||
InvalidApiKeyIndexes => {
|
||||
ErrCode::invalid("invalid_api_key_indexes", StatusCode::BAD_REQUEST)
|
||||
}
|
||||
InvalidApiKeyExpiresAt => {
|
||||
ErrCode::invalid("invalid_api_key_expires_at", StatusCode::BAD_REQUEST)
|
||||
}
|
||||
InvalidApiKeyDescription => {
|
||||
ErrCode::invalid("invalid_api_key_description", StatusCode::BAD_REQUEST)
|
||||
}
|
||||
InvalidMinWordLengthForTypo => {
|
||||
ErrCode::invalid("invalid_min_word_length_for_typo", StatusCode::BAD_REQUEST)
|
||||
}
|
||||
UnretrievableErrorCode => {
|
||||
ErrCode::invalid("unretrievable_error_code", StatusCode::BAD_REQUEST)
|
||||
}
|
||||
MalformedDump => ErrCode::invalid("malformed_dump", StatusCode::BAD_REQUEST),
|
||||
}
|
||||
}
|
||||
|
||||
/// return the HTTP status code ascociated with the `Code`
|
||||
fn http(&self) -> StatusCode {
|
||||
self.err_code().status_code
|
||||
}
|
||||
|
||||
/// return error name, used as error code
|
||||
fn name(&self) -> String {
|
||||
self.err_code().error_name.to_string()
|
||||
}
|
||||
|
||||
/// return the error type
|
||||
fn type_(&self) -> String {
|
||||
self.err_code().error_type.to_string()
|
||||
}
|
||||
|
||||
/// return the doc url ascociated with the error
|
||||
fn url(&self) -> String {
|
||||
format!("https://docs.meilisearch.com/errors#{}", self.name())
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal structure providing a convenient way to create error codes
|
||||
struct ErrCode {
|
||||
status_code: StatusCode,
|
||||
error_type: ErrorType,
|
||||
error_name: &'static str,
|
||||
}
|
||||
|
||||
impl ErrCode {
|
||||
fn authentication(error_name: &'static str, status_code: StatusCode) -> ErrCode {
|
||||
ErrCode { status_code, error_name, error_type: ErrorType::AuthenticationError }
|
||||
}
|
||||
|
||||
fn internal(error_name: &'static str, status_code: StatusCode) -> ErrCode {
|
||||
ErrCode { status_code, error_name, error_type: ErrorType::InternalError }
|
||||
}
|
||||
|
||||
fn invalid(error_name: &'static str, status_code: StatusCode) -> ErrCode {
|
||||
ErrCode { status_code, error_name, error_type: ErrorType::InvalidRequestError }
|
||||
}
|
||||
}
|
77
crates/dump/src/reader/v4/keys.rs
Normal file
77
crates/dump/src/reader/v4/keys.rs
Normal file
@ -0,0 +1,77 @@
|
||||
use serde::Deserialize;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
pub const KEY_ID_LENGTH: usize = 8;
|
||||
pub type KeyId = [u8; KEY_ID_LENGTH];
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
pub struct Key {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub description: Option<String>,
|
||||
pub id: KeyId,
|
||||
pub actions: Vec<Action>,
|
||||
pub indexes: Vec<String>,
|
||||
#[serde(with = "time::serde::rfc3339::option")]
|
||||
pub expires_at: Option<OffsetDateTime>,
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
pub created_at: OffsetDateTime,
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
pub updated_at: OffsetDateTime,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Deserialize, Debug, Eq, PartialEq)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
#[repr(u8)]
|
||||
pub enum Action {
|
||||
#[serde(rename = "*")]
|
||||
All = 0,
|
||||
#[serde(rename = "search")]
|
||||
Search = actions::SEARCH,
|
||||
#[serde(rename = "documents.add")]
|
||||
DocumentsAdd = actions::DOCUMENTS_ADD,
|
||||
#[serde(rename = "documents.get")]
|
||||
DocumentsGet = actions::DOCUMENTS_GET,
|
||||
#[serde(rename = "documents.delete")]
|
||||
DocumentsDelete = actions::DOCUMENTS_DELETE,
|
||||
#[serde(rename = "indexes.create")]
|
||||
IndexesAdd = actions::INDEXES_CREATE,
|
||||
#[serde(rename = "indexes.get")]
|
||||
IndexesGet = actions::INDEXES_GET,
|
||||
#[serde(rename = "indexes.update")]
|
||||
IndexesUpdate = actions::INDEXES_UPDATE,
|
||||
#[serde(rename = "indexes.delete")]
|
||||
IndexesDelete = actions::INDEXES_DELETE,
|
||||
#[serde(rename = "tasks.get")]
|
||||
TasksGet = actions::TASKS_GET,
|
||||
#[serde(rename = "settings.get")]
|
||||
SettingsGet = actions::SETTINGS_GET,
|
||||
#[serde(rename = "settings.update")]
|
||||
SettingsUpdate = actions::SETTINGS_UPDATE,
|
||||
#[serde(rename = "stats.get")]
|
||||
StatsGet = actions::STATS_GET,
|
||||
#[serde(rename = "dumps.create")]
|
||||
DumpsCreate = actions::DUMPS_CREATE,
|
||||
#[serde(rename = "dumps.get")]
|
||||
DumpsGet = actions::DUMPS_GET,
|
||||
#[serde(rename = "version")]
|
||||
Version = actions::VERSION,
|
||||
}
|
||||
|
||||
pub mod actions {
|
||||
pub const SEARCH: u8 = 1;
|
||||
pub const DOCUMENTS_ADD: u8 = 2;
|
||||
pub const DOCUMENTS_GET: u8 = 3;
|
||||
pub const DOCUMENTS_DELETE: u8 = 4;
|
||||
pub const INDEXES_CREATE: u8 = 5;
|
||||
pub const INDEXES_GET: u8 = 6;
|
||||
pub const INDEXES_UPDATE: u8 = 7;
|
||||
pub const INDEXES_DELETE: u8 = 8;
|
||||
pub const TASKS_GET: u8 = 9;
|
||||
pub const SETTINGS_GET: u8 = 10;
|
||||
pub const SETTINGS_UPDATE: u8 = 11;
|
||||
pub const STATS_GET: u8 = 12;
|
||||
pub const DUMPS_CREATE: u8 = 13;
|
||||
pub const DUMPS_GET: u8 = 14;
|
||||
pub const VERSION: u8 = 15;
|
||||
}
|
140
crates/dump/src/reader/v4/meta.rs
Normal file
140
crates/dump/src/reader/v4/meta.rs
Normal file
@ -0,0 +1,140 @@
|
||||
use std::fmt::{self, Display, Formatter};
|
||||
use std::marker::PhantomData;
|
||||
use std::str::FromStr;
|
||||
|
||||
use serde::de::Visitor;
|
||||
use serde::{Deserialize, Deserializer};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::settings::{Settings, Unchecked};
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
pub struct IndexUuid {
|
||||
pub uid: String,
|
||||
pub index_meta: IndexMeta,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
pub struct IndexMeta {
|
||||
pub uuid: Uuid,
|
||||
pub creation_task_id: usize,
|
||||
}
|
||||
|
||||
// There is one in each indexes under `meta.json`.
|
||||
#[derive(Deserialize)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
pub struct DumpMeta {
|
||||
pub settings: Settings<Unchecked>,
|
||||
pub primary_key: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone, PartialEq, Eq)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
pub struct IndexUid(pub String);
|
||||
|
||||
impl TryFrom<String> for IndexUid {
|
||||
type Error = IndexUidFormatError;
|
||||
|
||||
fn try_from(uid: String) -> Result<Self, Self::Error> {
|
||||
if !uid.chars().all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_')
|
||||
|| uid.is_empty()
|
||||
|| uid.len() > 400
|
||||
{
|
||||
Err(IndexUidFormatError { invalid_uid: uid })
|
||||
} else {
|
||||
Ok(IndexUid(uid))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for IndexUid {
|
||||
type Err = IndexUidFormatError;
|
||||
|
||||
fn from_str(uid: &str) -> Result<IndexUid, IndexUidFormatError> {
|
||||
uid.to_string().try_into()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<IndexUid> for String {
|
||||
fn from(uid: IndexUid) -> Self {
|
||||
uid.into_inner()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct IndexUidFormatError {
|
||||
pub invalid_uid: String,
|
||||
}
|
||||
|
||||
impl Display for IndexUidFormatError {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"invalid index uid `{}`, the uid must be an integer \
|
||||
or a string containing only alphanumeric characters \
|
||||
a-z A-Z 0-9, hyphens - and underscores _, \
|
||||
and can not be more than 400 bytes.",
|
||||
self.invalid_uid,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for IndexUidFormatError {}
|
||||
|
||||
/// A type that tries to match either a star (*) or
|
||||
/// any other thing that implements `FromStr`.
|
||||
#[derive(Debug)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
pub enum StarOr<T> {
|
||||
Star,
|
||||
Other(T),
|
||||
}
|
||||
|
||||
impl<'de, T, E> Deserialize<'de> for StarOr<T>
|
||||
where
|
||||
T: FromStr<Err = E>,
|
||||
E: Display,
|
||||
{
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
/// Serde can't differentiate between `StarOr::Star` and `StarOr::Other` without a tag.
|
||||
/// Simply using `#[serde(untagged)]` + `#[serde(rename="*")]` will lead to attempting to
|
||||
/// deserialize everything as a `StarOr::Other`, including "*".
|
||||
/// [`#[serde(other)]`](https://serde.rs/variant-attrs.html#other) might have helped but is
|
||||
/// not supported on untagged enums.
|
||||
struct StarOrVisitor<T>(PhantomData<T>);
|
||||
|
||||
impl<'de, T, FE> Visitor<'de> for StarOrVisitor<T>
|
||||
where
|
||||
T: FromStr<Err = FE>,
|
||||
FE: Display,
|
||||
{
|
||||
type Value = StarOr<T>;
|
||||
|
||||
fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result {
|
||||
formatter.write_str("a string")
|
||||
}
|
||||
|
||||
fn visit_str<SE>(self, v: &str) -> Result<Self::Value, SE>
|
||||
where
|
||||
SE: serde::de::Error,
|
||||
{
|
||||
match v {
|
||||
"*" => Ok(StarOr::Star),
|
||||
v => {
|
||||
let other = FromStr::from_str(v).map_err(|e: T::Err| {
|
||||
SE::custom(format!("Invalid `other` value: {}", e))
|
||||
})?;
|
||||
Ok(StarOr::Other(other))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_str(StarOrVisitor(PhantomData))
|
||||
}
|
||||
}
|
338
crates/dump/src/reader/v4/mod.rs
Normal file
338
crates/dump/src/reader/v4/mod.rs
Normal file
@ -0,0 +1,338 @@
|
||||
use std::fs::{self, File};
|
||||
use std::io::{BufRead, BufReader, ErrorKind};
|
||||
use std::path::Path;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tempfile::TempDir;
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub mod errors;
|
||||
pub mod keys;
|
||||
pub mod meta;
|
||||
pub mod settings;
|
||||
pub mod tasks;
|
||||
|
||||
use self::meta::{DumpMeta, IndexMeta, IndexUuid};
|
||||
use super::compat::v4_to_v5::CompatV4ToV5;
|
||||
use crate::{Error, IndexMetadata, Result, Version};
|
||||
|
||||
pub type Document = serde_json::Map<String, serde_json::Value>;
|
||||
pub type Settings<T> = settings::Settings<T>;
|
||||
pub type Checked = settings::Checked;
|
||||
pub type Unchecked = settings::Unchecked;
|
||||
|
||||
pub type Task = tasks::Task;
|
||||
pub type Key = keys::Key;
|
||||
|
||||
// everything related to the settings
|
||||
pub type Setting<T> = settings::Setting<T>;
|
||||
|
||||
// everything related to the api keys
|
||||
pub type Action = keys::Action;
|
||||
|
||||
// everything related to the errors
|
||||
pub type ResponseError = errors::ResponseError;
|
||||
pub type Code = errors::Code;
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Metadata {
|
||||
db_version: String,
|
||||
index_db_size: usize,
|
||||
update_db_size: usize,
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
dump_date: OffsetDateTime,
|
||||
}
|
||||
|
||||
pub struct V4Reader {
|
||||
dump: TempDir,
|
||||
metadata: Metadata,
|
||||
tasks: BufReader<File>,
|
||||
keys: BufReader<File>,
|
||||
index_uuid: Vec<IndexUuid>,
|
||||
}
|
||||
|
||||
impl V4Reader {
|
||||
pub fn open(dump: TempDir) -> Result<Self> {
|
||||
let meta_file = fs::read(dump.path().join("metadata.json"))?;
|
||||
let metadata = serde_json::from_reader(&*meta_file)?;
|
||||
let index_uuid = File::open(dump.path().join("index_uuids/data.jsonl"))?;
|
||||
let index_uuid = BufReader::new(index_uuid);
|
||||
let index_uuid = index_uuid
|
||||
.lines()
|
||||
.map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) })
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
|
||||
Ok(V4Reader {
|
||||
metadata,
|
||||
tasks: BufReader::new(
|
||||
File::open(dump.path().join("updates").join("data.jsonl")).unwrap(),
|
||||
),
|
||||
keys: BufReader::new(File::open(dump.path().join("keys"))?),
|
||||
index_uuid,
|
||||
dump,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn to_v5(self) -> CompatV4ToV5 {
|
||||
CompatV4ToV5::new(self)
|
||||
}
|
||||
|
||||
pub fn version(&self) -> Version {
|
||||
Version::V4
|
||||
}
|
||||
|
||||
pub fn date(&self) -> Option<OffsetDateTime> {
|
||||
Some(self.metadata.dump_date)
|
||||
}
|
||||
|
||||
pub fn instance_uid(&self) -> Result<Option<Uuid>> {
|
||||
match fs::read_to_string(self.dump.path().join("instance-uid")) {
|
||||
Ok(uuid) => Ok(Some(Uuid::parse_str(&uuid)?)),
|
||||
Err(e) if e.kind() == ErrorKind::NotFound => Ok(None),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn indexes(&self) -> Result<impl Iterator<Item = Result<V4IndexReader>> + '_> {
|
||||
Ok(self.index_uuid.iter().map(|index| -> Result<_> {
|
||||
V4IndexReader::new(
|
||||
index.uid.clone(),
|
||||
&self.dump.path().join("indexes").join(index.index_meta.uuid.to_string()),
|
||||
&index.index_meta,
|
||||
BufReader::new(
|
||||
File::open(self.dump.path().join("updates").join("data.jsonl")).unwrap(),
|
||||
),
|
||||
)
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn tasks(
|
||||
&mut self,
|
||||
) -> Box<dyn Iterator<Item = Result<(Task, Option<Box<super::UpdateFile>>)>> + '_> {
|
||||
Box::new((&mut self.tasks).lines().map(|line| -> Result<_> {
|
||||
let task: Task = serde_json::from_str(&line?)?;
|
||||
if !task.is_finished() {
|
||||
if let Some(uuid) = task.get_content_uuid() {
|
||||
let update_file_path = self
|
||||
.dump
|
||||
.path()
|
||||
.join("updates")
|
||||
.join("updates_files")
|
||||
.join(uuid.to_string());
|
||||
Ok((
|
||||
task,
|
||||
Some(
|
||||
Box::new(UpdateFile::new(&update_file_path)?) as Box<super::UpdateFile>
|
||||
),
|
||||
))
|
||||
} else {
|
||||
Ok((task, None))
|
||||
}
|
||||
} else {
|
||||
Ok((task, None))
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<Key>> + '_> {
|
||||
Box::new(
|
||||
(&mut self.keys).lines().map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct V4IndexReader {
|
||||
metadata: IndexMetadata,
|
||||
settings: Settings<Checked>,
|
||||
|
||||
documents: BufReader<File>,
|
||||
}
|
||||
|
||||
impl V4IndexReader {
|
||||
pub fn new(
|
||||
name: String,
|
||||
path: &Path,
|
||||
index_metadata: &IndexMeta,
|
||||
tasks: BufReader<File>,
|
||||
) -> Result<Self> {
|
||||
let meta = File::open(path.join("meta.json"))?;
|
||||
let meta: DumpMeta = serde_json::from_reader(meta)?;
|
||||
|
||||
let mut created_at = None;
|
||||
let mut updated_at = None;
|
||||
|
||||
for line in tasks.lines() {
|
||||
let task: Task = serde_json::from_str(&line?)?;
|
||||
|
||||
if task.index_uid.to_string() == name {
|
||||
// The first task to match our index_uid that succeeded (ie. processed_at returns Some)
|
||||
// is our `last_updated_at`.
|
||||
if updated_at.is_none() {
|
||||
updated_at = task.processed_at()
|
||||
}
|
||||
|
||||
// Once we reach the `creation_task_id` we can stop iterating on the task queue and
|
||||
// this task represents our `created_at`.
|
||||
if task.id as usize == index_metadata.creation_task_id {
|
||||
created_at = task.created_at();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let current_time = OffsetDateTime::now_utc();
|
||||
|
||||
let metadata = IndexMetadata {
|
||||
uid: name,
|
||||
primary_key: meta.primary_key,
|
||||
created_at: created_at.unwrap_or(current_time),
|
||||
updated_at: updated_at.unwrap_or(current_time),
|
||||
};
|
||||
|
||||
let ret = V4IndexReader {
|
||||
metadata,
|
||||
settings: meta.settings.check(),
|
||||
documents: BufReader::new(File::open(path.join("documents.jsonl"))?),
|
||||
};
|
||||
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
pub fn metadata(&self) -> &IndexMetadata {
|
||||
&self.metadata
|
||||
}
|
||||
|
||||
pub fn documents(&mut self) -> Result<impl Iterator<Item = Result<Document>> + '_> {
|
||||
Ok((&mut self.documents)
|
||||
.lines()
|
||||
.map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }))
|
||||
}
|
||||
|
||||
pub fn settings(&mut self) -> Result<Settings<Checked>> {
|
||||
Ok(self.settings.clone())
|
||||
}
|
||||
}
|
||||
|
||||
pub struct UpdateFile {
|
||||
reader: BufReader<File>,
|
||||
}
|
||||
|
||||
impl UpdateFile {
|
||||
fn new(path: &Path) -> Result<Self> {
|
||||
Ok(UpdateFile { reader: BufReader::new(File::open(path)?) })
|
||||
}
|
||||
}
|
||||
|
||||
impl Iterator for UpdateFile {
|
||||
type Item = Result<Document>;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
(&mut self.reader)
|
||||
.lines()
|
||||
.map(|line| {
|
||||
line.map_err(Error::from)
|
||||
.and_then(|line| serde_json::from_str(&line).map_err(Error::from))
|
||||
})
|
||||
.next()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test {
|
||||
use std::fs::File;
|
||||
use std::io::BufReader;
|
||||
|
||||
use flate2::bufread::GzDecoder;
|
||||
use meili_snap::insta;
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn read_dump_v4() {
|
||||
let dump = File::open("tests/assets/v4.dump").unwrap();
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut dump = BufReader::new(dump);
|
||||
let gz = GzDecoder::new(&mut dump);
|
||||
let mut archive = tar::Archive::new(gz);
|
||||
archive.unpack(dir.path()).unwrap();
|
||||
|
||||
let mut dump = V4Reader::open(dir).unwrap();
|
||||
|
||||
// top level infos
|
||||
insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
|
||||
insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
|
||||
|
||||
// tasks
|
||||
let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();
|
||||
let (tasks, mut update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
|
||||
meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"f4efacbea0c1a4400873f4b2ee33f975");
|
||||
assert_eq!(update_files.len(), 10);
|
||||
assert!(update_files[0].is_some()); // the enqueued document addition
|
||||
assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
|
||||
|
||||
let update_file = update_files.remove(0).unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
meili_snap::snapshot_hash!(meili_snap::json_string!(update_file), @"7b8889539b669c7b9ddba448bafa385d");
|
||||
|
||||
// keys
|
||||
let keys = dump.keys().collect::<Result<Vec<_>>>().unwrap();
|
||||
meili_snap::snapshot_hash!(meili_snap::json_string!(keys, { "[].uid" => "[uuid]" }), @"9240300dca8f962cdf58359ef4c76f09");
|
||||
|
||||
// indexes
|
||||
let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
// the index are not ordered in any way by default
|
||||
indexes.sort_by_key(|index| index.metadata().uid.to_string());
|
||||
|
||||
let mut products = indexes.pop().unwrap();
|
||||
let mut movies = indexes.pop().unwrap();
|
||||
let mut spells = indexes.pop().unwrap();
|
||||
assert!(indexes.is_empty());
|
||||
|
||||
// products
|
||||
insta::assert_json_snapshot!(products.metadata(), @r###"
|
||||
{
|
||||
"uid": "products",
|
||||
"primaryKey": "sku",
|
||||
"createdAt": "2022-10-06T12:53:39.360187055Z",
|
||||
"updatedAt": "2022-10-06T12:53:40.603035979Z"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(products.settings().unwrap());
|
||||
let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 10);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
|
||||
|
||||
// movies
|
||||
insta::assert_json_snapshot!(movies.metadata(), @r###"
|
||||
{
|
||||
"uid": "movies",
|
||||
"primaryKey": "id",
|
||||
"createdAt": "2022-10-06T12:53:38.710611568Z",
|
||||
"updatedAt": "2022-10-06T12:53:49.785862546Z"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(movies.settings().unwrap());
|
||||
let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 110);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"786022a66ecb992c8a2a60fee070a5ab");
|
||||
|
||||
// spells
|
||||
insta::assert_json_snapshot!(spells.metadata(), @r###"
|
||||
{
|
||||
"uid": "dnd_spells",
|
||||
"primaryKey": "index",
|
||||
"createdAt": "2022-10-06T12:53:40.831649057Z",
|
||||
"updatedAt": "2022-10-06T12:53:41.116036186Z"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(spells.settings().unwrap());
|
||||
let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 10);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
|
||||
}
|
||||
}
|
262
crates/dump/src/reader/v4/settings.rs
Normal file
262
crates/dump/src/reader/v4/settings.rs
Normal file
@ -0,0 +1,262 @@
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::marker::PhantomData;
|
||||
use std::num::NonZeroUsize;
|
||||
|
||||
use serde::{Deserialize, Deserializer};
|
||||
|
||||
#[cfg(test)]
|
||||
fn serialize_with_wildcard<S>(
|
||||
field: &Setting<Vec<String>>,
|
||||
s: S,
|
||||
) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
use serde::Serialize;
|
||||
|
||||
let wildcard = vec!["*".to_string()];
|
||||
match field {
|
||||
Setting::Set(value) => Some(value),
|
||||
Setting::Reset => Some(&wildcard),
|
||||
Setting::NotSet => None,
|
||||
}
|
||||
.serialize(s)
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Debug, PartialEq, Eq)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
pub struct Checked;
|
||||
|
||||
#[derive(Clone, Default, Debug, Deserialize, PartialEq, Eq)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
pub struct Unchecked;
|
||||
|
||||
#[derive(Debug, Clone, Default, Deserialize, PartialEq, Eq)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MinWordSizeTyposSetting {
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
pub one_typo: Setting<u8>,
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
pub two_typos: Setting<u8>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Deserialize, PartialEq, Eq)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TypoSettings {
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
pub enabled: Setting<bool>,
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
pub min_word_size_for_typos: Setting<MinWordSizeTyposSetting>,
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
pub disable_on_words: Setting<BTreeSet<String>>,
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
pub disable_on_attributes: Setting<BTreeSet<String>>,
|
||||
}
|
||||
/// Holds all the settings for an index. `T` can either be `Checked` if they represents settings
|
||||
/// whose validity is guaranteed, or `Unchecked` if they need to be validated. In the later case, a
|
||||
/// call to `check` will return a `Settings<Checked>` from a `Settings<Unchecked>`.
|
||||
#[derive(Debug, Clone, Default, Deserialize, PartialEq, Eq)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(bound(serialize = "T: serde::Serialize", deserialize = "T: Deserialize<'static>"))]
|
||||
pub struct Settings<T> {
|
||||
#[serde(
|
||||
default,
|
||||
serialize_with = "serialize_with_wildcard",
|
||||
skip_serializing_if = "Setting::is_not_set"
|
||||
)]
|
||||
pub displayed_attributes: Setting<Vec<String>>,
|
||||
|
||||
#[serde(
|
||||
default,
|
||||
serialize_with = "serialize_with_wildcard",
|
||||
skip_serializing_if = "Setting::is_not_set"
|
||||
)]
|
||||
pub searchable_attributes: Setting<Vec<String>>,
|
||||
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
pub filterable_attributes: Setting<BTreeSet<String>>,
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
pub sortable_attributes: Setting<BTreeSet<String>>,
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
pub ranking_rules: Setting<Vec<String>>,
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
pub stop_words: Setting<BTreeSet<String>>,
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
pub synonyms: Setting<BTreeMap<String, Vec<String>>>,
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
pub distinct_attribute: Setting<String>,
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
pub typo_tolerance: Setting<TypoSettings>,
|
||||
|
||||
#[serde(skip)]
|
||||
pub _kind: PhantomData<T>,
|
||||
}
|
||||
|
||||
impl Settings<Checked> {
|
||||
pub fn cleared() -> Settings<Checked> {
|
||||
Settings {
|
||||
displayed_attributes: Setting::Reset,
|
||||
searchable_attributes: Setting::Reset,
|
||||
filterable_attributes: Setting::Reset,
|
||||
sortable_attributes: Setting::Reset,
|
||||
ranking_rules: Setting::Reset,
|
||||
stop_words: Setting::Reset,
|
||||
synonyms: Setting::Reset,
|
||||
distinct_attribute: Setting::Reset,
|
||||
typo_tolerance: Setting::Reset,
|
||||
_kind: PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn into_unchecked(self) -> Settings<Unchecked> {
|
||||
let Self {
|
||||
displayed_attributes,
|
||||
searchable_attributes,
|
||||
filterable_attributes,
|
||||
sortable_attributes,
|
||||
ranking_rules,
|
||||
stop_words,
|
||||
synonyms,
|
||||
distinct_attribute,
|
||||
typo_tolerance,
|
||||
..
|
||||
} = self;
|
||||
|
||||
Settings {
|
||||
displayed_attributes,
|
||||
searchable_attributes,
|
||||
filterable_attributes,
|
||||
sortable_attributes,
|
||||
ranking_rules,
|
||||
stop_words,
|
||||
synonyms,
|
||||
distinct_attribute,
|
||||
typo_tolerance,
|
||||
_kind: PhantomData,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Settings<Unchecked> {
|
||||
pub fn check(self) -> Settings<Checked> {
|
||||
let displayed_attributes = match self.displayed_attributes {
|
||||
Setting::Set(fields) => {
|
||||
if fields.iter().any(|f| f == "*") {
|
||||
Setting::Reset
|
||||
} else {
|
||||
Setting::Set(fields)
|
||||
}
|
||||
}
|
||||
otherwise => otherwise,
|
||||
};
|
||||
|
||||
let searchable_attributes = match self.searchable_attributes {
|
||||
Setting::Set(fields) => {
|
||||
if fields.iter().any(|f| f == "*") {
|
||||
Setting::Reset
|
||||
} else {
|
||||
Setting::Set(fields)
|
||||
}
|
||||
}
|
||||
otherwise => otherwise,
|
||||
};
|
||||
|
||||
Settings {
|
||||
displayed_attributes,
|
||||
searchable_attributes,
|
||||
filterable_attributes: self.filterable_attributes,
|
||||
sortable_attributes: self.sortable_attributes,
|
||||
ranking_rules: self.ranking_rules,
|
||||
stop_words: self.stop_words,
|
||||
synonyms: self.synonyms,
|
||||
distinct_attribute: self.distinct_attribute,
|
||||
typo_tolerance: self.typo_tolerance,
|
||||
_kind: PhantomData,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)] // otherwise rustc complains that the fields go unused
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Facets {
|
||||
pub level_group_size: Option<NonZeroUsize>,
|
||||
pub min_level_size: Option<NonZeroUsize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Copy)]
|
||||
pub enum Setting<T> {
|
||||
Set(T),
|
||||
Reset,
|
||||
NotSet,
|
||||
}
|
||||
|
||||
impl<T> Default for Setting<T> {
|
||||
fn default() -> Self {
|
||||
Self::NotSet
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Setting<T> {
|
||||
pub fn set(self) -> Option<T> {
|
||||
match self {
|
||||
Self::Set(value) => Some(value),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn as_ref(&self) -> Setting<&T> {
|
||||
match *self {
|
||||
Self::Set(ref value) => Setting::Set(value),
|
||||
Self::Reset => Setting::Reset,
|
||||
Self::NotSet => Setting::NotSet,
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn is_not_set(&self) -> bool {
|
||||
matches!(self, Self::NotSet)
|
||||
}
|
||||
|
||||
/// If `Self` is `Reset`, then map self to `Set` with the provided `val`.
|
||||
pub fn or_reset(self, val: T) -> Self {
|
||||
match self {
|
||||
Self::Reset => Self::Set(val),
|
||||
otherwise => otherwise,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl<T: serde::Serialize> serde::Serialize for Setting<T> {
|
||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
match self {
|
||||
Self::Set(value) => Some(value),
|
||||
// Usually not_set isn't serialized by setting skip_serializing_if field attribute
|
||||
Self::NotSet | Self::Reset => None,
|
||||
}
|
||||
.serialize(serializer)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de, T: Deserialize<'de>> Deserialize<'de> for Setting<T> {
|
||||
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
Deserialize::deserialize(deserializer).map(|x| match x {
|
||||
Some(x) => Self::Set(x),
|
||||
None => Self::Reset, // Reset is forced by sending null value
|
||||
})
|
||||
}
|
||||
}
|
@ -0,0 +1,40 @@
|
||||
---
|
||||
source: dump/src/reader/v4/mod.rs
|
||||
expression: movies.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": [
|
||||
"*"
|
||||
],
|
||||
"searchableAttributes": [
|
||||
"*"
|
||||
],
|
||||
"filterableAttributes": [
|
||||
"genres",
|
||||
"id"
|
||||
],
|
||||
"sortableAttributes": [
|
||||
"release_date"
|
||||
],
|
||||
"rankingRules": [
|
||||
"words",
|
||||
"typo",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"sort",
|
||||
"exactness",
|
||||
"release_date:asc"
|
||||
],
|
||||
"stopWords": [],
|
||||
"synonyms": {},
|
||||
"distinctAttribute": null,
|
||||
"typoTolerance": {
|
||||
"enabled": true,
|
||||
"minWordSizeForTypos": {
|
||||
"oneTypo": 5,
|
||||
"twoTypos": 9
|
||||
},
|
||||
"disableOnWords": [],
|
||||
"disableOnAttributes": []
|
||||
}
|
||||
}
|
@ -0,0 +1,34 @@
|
||||
---
|
||||
source: dump/src/reader/v4/mod.rs
|
||||
expression: spells.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": [
|
||||
"*"
|
||||
],
|
||||
"searchableAttributes": [
|
||||
"*"
|
||||
],
|
||||
"filterableAttributes": [],
|
||||
"sortableAttributes": [],
|
||||
"rankingRules": [
|
||||
"words",
|
||||
"typo",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"sort",
|
||||
"exactness"
|
||||
],
|
||||
"stopWords": [],
|
||||
"synonyms": {},
|
||||
"distinctAttribute": null,
|
||||
"typoTolerance": {
|
||||
"enabled": true,
|
||||
"minWordSizeForTypos": {
|
||||
"oneTypo": 5,
|
||||
"twoTypos": 9
|
||||
},
|
||||
"disableOnWords": [],
|
||||
"disableOnAttributes": []
|
||||
}
|
||||
}
|
@ -0,0 +1,48 @@
|
||||
---
|
||||
source: dump/src/reader/v4/mod.rs
|
||||
expression: products.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": [
|
||||
"*"
|
||||
],
|
||||
"searchableAttributes": [
|
||||
"*"
|
||||
],
|
||||
"filterableAttributes": [],
|
||||
"sortableAttributes": [],
|
||||
"rankingRules": [
|
||||
"words",
|
||||
"typo",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"sort",
|
||||
"exactness"
|
||||
],
|
||||
"stopWords": [],
|
||||
"synonyms": {
|
||||
"android": [
|
||||
"phone",
|
||||
"smartphone"
|
||||
],
|
||||
"iphone": [
|
||||
"phone",
|
||||
"smartphone"
|
||||
],
|
||||
"phone": [
|
||||
"android",
|
||||
"iphone",
|
||||
"smartphone"
|
||||
]
|
||||
},
|
||||
"distinctAttribute": null,
|
||||
"typoTolerance": {
|
||||
"enabled": true,
|
||||
"minWordSizeForTypos": {
|
||||
"oneTypo": 5,
|
||||
"twoTypos": 9
|
||||
},
|
||||
"disableOnWords": [],
|
||||
"disableOnAttributes": []
|
||||
}
|
||||
}
|
170
crates/dump/src/reader/v4/tasks.rs
Normal file
170
crates/dump/src/reader/v4/tasks.rs
Normal file
@ -0,0 +1,170 @@
|
||||
use serde::Deserialize;
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::errors::ResponseError;
|
||||
use super::meta::IndexUid;
|
||||
use super::settings::{Settings, Unchecked};
|
||||
|
||||
pub type TaskId = u32;
|
||||
pub type BatchId = u32;
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
pub struct Task {
|
||||
pub id: TaskId,
|
||||
pub index_uid: IndexUid,
|
||||
pub content: TaskContent,
|
||||
pub events: Vec<TaskEvent>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
pub enum TaskContent {
|
||||
DocumentAddition {
|
||||
content_uuid: Uuid,
|
||||
merge_strategy: IndexDocumentsMethod,
|
||||
primary_key: Option<String>,
|
||||
documents_count: usize,
|
||||
allow_index_creation: bool,
|
||||
},
|
||||
DocumentDeletion(DocumentDeletion),
|
||||
SettingsUpdate {
|
||||
settings: Settings<Unchecked>,
|
||||
/// Indicates whether the task was a deletion
|
||||
is_deletion: bool,
|
||||
allow_index_creation: bool,
|
||||
},
|
||||
IndexDeletion,
|
||||
IndexCreation {
|
||||
primary_key: Option<String>,
|
||||
},
|
||||
IndexUpdate {
|
||||
primary_key: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
pub enum IndexDocumentsMethod {
|
||||
/// Replace the previous document with the new one,
|
||||
/// removing all the already known attributes.
|
||||
ReplaceDocuments,
|
||||
|
||||
/// Merge the previous version of the document with the new version,
|
||||
/// replacing old attributes values with the new ones and add the new attributes.
|
||||
UpdateDocuments,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
pub enum DocumentDeletion {
|
||||
Clear,
|
||||
Ids(Vec<String>),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
pub enum TaskEvent {
|
||||
Created(#[serde(with = "time::serde::rfc3339")] OffsetDateTime),
|
||||
Batched {
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
timestamp: OffsetDateTime,
|
||||
batch_id: BatchId,
|
||||
},
|
||||
Processing(#[serde(with = "time::serde::rfc3339")] OffsetDateTime),
|
||||
Succeded {
|
||||
result: TaskResult,
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
timestamp: OffsetDateTime,
|
||||
},
|
||||
Failed {
|
||||
error: ResponseError,
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
timestamp: OffsetDateTime,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
|
||||
#[cfg_attr(test, derive(serde::Serialize))]
|
||||
pub enum TaskResult {
|
||||
DocumentAddition { indexed_documents: u64 },
|
||||
DocumentDeletion { deleted_documents: u64 },
|
||||
ClearAll { deleted_documents: u64 },
|
||||
Other,
|
||||
}
|
||||
|
||||
impl Task {
|
||||
/// Return true when a task is finished.
|
||||
/// A task is finished when its last state is either `Succeeded` or `Failed`.
|
||||
pub fn is_finished(&self) -> bool {
|
||||
self.events.last().map_or(false, |event| {
|
||||
matches!(event, TaskEvent::Succeded { .. } | TaskEvent::Failed { .. })
|
||||
})
|
||||
}
|
||||
|
||||
pub fn processed_at(&self) -> Option<OffsetDateTime> {
|
||||
match self.events.last() {
|
||||
Some(TaskEvent::Succeded { result: _, timestamp }) => Some(*timestamp),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn created_at(&self) -> Option<OffsetDateTime> {
|
||||
match &self.content {
|
||||
TaskContent::IndexCreation { primary_key: _ } => match self.events.first() {
|
||||
Some(TaskEvent::Created(ts)) => Some(*ts),
|
||||
_ => None,
|
||||
},
|
||||
TaskContent::DocumentAddition {
|
||||
content_uuid: _,
|
||||
merge_strategy: _,
|
||||
primary_key: _,
|
||||
documents_count: _,
|
||||
allow_index_creation: _,
|
||||
} => match self.events.first() {
|
||||
Some(TaskEvent::Created(ts)) => Some(*ts),
|
||||
_ => None,
|
||||
},
|
||||
TaskContent::SettingsUpdate {
|
||||
settings: _,
|
||||
is_deletion: _,
|
||||
allow_index_creation: _,
|
||||
} => match self.events.first() {
|
||||
Some(TaskEvent::Created(ts)) => Some(*ts),
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the content_uuid of the `Task` if there is one.
|
||||
pub fn get_content_uuid(&self) -> Option<Uuid> {
|
||||
match self {
|
||||
Task { content: TaskContent::DocumentAddition { content_uuid, .. }, .. } => {
|
||||
Some(*content_uuid)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl IndexUid {
|
||||
pub fn into_inner(self) -> String {
|
||||
self.0
|
||||
}
|
||||
|
||||
/// Return a reference over the inner str.
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl std::ops::Deref for IndexUid {
|
||||
type Target = str;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
Reference in New Issue
Block a user