mirror of
https://github.com/meilisearch/meilisearch.git
synced 2025-07-27 16:51:01 +00:00
Move crates under a sub folder to clean up the code
This commit is contained in:
Clément Renault
5
crates/dump/src/reader/compat/mod.rs
Normal file
5
crates/dump/src/reader/compat/mod.rs
Normal file
@ -0,0 +1,5 @@
|
||||
pub mod v1_to_v2;
|
||||
pub mod v2_to_v3;
|
||||
pub mod v3_to_v4;
|
||||
pub mod v4_to_v5;
|
||||
pub mod v5_to_v6;
|
||||
@ -0,0 +1,38 @@
|
||||
---
|
||||
source: dump/src/reader/compat/v1_to_v2.rs
|
||||
expression: products.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": [
|
||||
"*"
|
||||
],
|
||||
"searchableAttributes": [
|
||||
"*"
|
||||
],
|
||||
"filterableAttributes": [],
|
||||
"sortableAttributes": [],
|
||||
"rankingRules": [
|
||||
"typo",
|
||||
"words",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"exactness"
|
||||
],
|
||||
"stopWords": [],
|
||||
"synonyms": {
|
||||
"android": [
|
||||
"phone",
|
||||
"smartphone"
|
||||
],
|
||||
"iphone": [
|
||||
"phone",
|
||||
"smartphone"
|
||||
],
|
||||
"phone": [
|
||||
"android",
|
||||
"iphone",
|
||||
"smartphone"
|
||||
]
|
||||
},
|
||||
"distinctAttribute": null
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
---
|
||||
source: dump/src/reader/compat/v1_to_v2.rs
|
||||
expression: movies.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": [
|
||||
"*"
|
||||
],
|
||||
"searchableAttributes": [
|
||||
"*"
|
||||
],
|
||||
"filterableAttributes": [
|
||||
"genres",
|
||||
"id"
|
||||
],
|
||||
"sortableAttributes": [
|
||||
"genres",
|
||||
"id"
|
||||
],
|
||||
"rankingRules": [
|
||||
"typo",
|
||||
"words",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"exactness",
|
||||
"release_date:asc"
|
||||
],
|
||||
"stopWords": [],
|
||||
"synonyms": {},
|
||||
"distinctAttribute": null
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
---
|
||||
source: dump/src/reader/compat/v1_to_v2.rs
|
||||
expression: spells.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": [
|
||||
"*"
|
||||
],
|
||||
"searchableAttributes": [
|
||||
"*"
|
||||
],
|
||||
"filterableAttributes": [],
|
||||
"sortableAttributes": [],
|
||||
"rankingRules": [
|
||||
"typo",
|
||||
"words",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"exactness"
|
||||
],
|
||||
"stopWords": [],
|
||||
"synonyms": {},
|
||||
"distinctAttribute": null
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
---
|
||||
source: dump/src/reader/compat/v2_to_v3.rs
|
||||
expression: movies2.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": [
|
||||
"*"
|
||||
],
|
||||
"searchableAttributes": [
|
||||
"*"
|
||||
],
|
||||
"filterableAttributes": [],
|
||||
"rankingRules": [
|
||||
"words",
|
||||
"typo",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"exactness"
|
||||
],
|
||||
"stopWords": [],
|
||||
"synonyms": {},
|
||||
"distinctAttribute": null
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
---
|
||||
source: dump/src/reader/compat/v2_to_v3.rs
|
||||
expression: spells.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": [
|
||||
"*"
|
||||
],
|
||||
"searchableAttributes": [
|
||||
"*"
|
||||
],
|
||||
"filterableAttributes": [],
|
||||
"rankingRules": [
|
||||
"words",
|
||||
"typo",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"exactness"
|
||||
],
|
||||
"stopWords": [],
|
||||
"synonyms": {},
|
||||
"distinctAttribute": null
|
||||
}
|
||||
@ -0,0 +1,37 @@
|
||||
---
|
||||
source: dump/src/reader/compat/v2_to_v3.rs
|
||||
expression: products.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": [
|
||||
"*"
|
||||
],
|
||||
"searchableAttributes": [
|
||||
"*"
|
||||
],
|
||||
"filterableAttributes": [],
|
||||
"rankingRules": [
|
||||
"words",
|
||||
"typo",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"exactness"
|
||||
],
|
||||
"stopWords": [],
|
||||
"synonyms": {
|
||||
"android": [
|
||||
"phone",
|
||||
"smartphone"
|
||||
],
|
||||
"iphone": [
|
||||
"phone",
|
||||
"smartphone"
|
||||
],
|
||||
"phone": [
|
||||
"android",
|
||||
"iphone",
|
||||
"smartphone"
|
||||
]
|
||||
},
|
||||
"distinctAttribute": null
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
---
|
||||
source: dump/src/reader/compat/v2_to_v3.rs
|
||||
expression: movies.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": [
|
||||
"*"
|
||||
],
|
||||
"searchableAttributes": [
|
||||
"*"
|
||||
],
|
||||
"filterableAttributes": [],
|
||||
"rankingRules": [
|
||||
"words",
|
||||
"typo",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"exactness",
|
||||
"release_date:asc"
|
||||
],
|
||||
"stopWords": [],
|
||||
"synonyms": {},
|
||||
"distinctAttribute": null
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
---
|
||||
source: dump/src/reader/compat/v3_to_v4.rs
|
||||
expression: movies2.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": [
|
||||
"*"
|
||||
],
|
||||
"searchableAttributes": [
|
||||
"*"
|
||||
],
|
||||
"filterableAttributes": [],
|
||||
"sortableAttributes": [],
|
||||
"rankingRules": [
|
||||
"words",
|
||||
"typo",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"sort",
|
||||
"exactness"
|
||||
],
|
||||
"stopWords": [],
|
||||
"synonyms": {},
|
||||
"distinctAttribute": null
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
---
|
||||
source: dump/src/reader/compat/v3_to_v4.rs
|
||||
expression: spells.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": [
|
||||
"*"
|
||||
],
|
||||
"searchableAttributes": [
|
||||
"*"
|
||||
],
|
||||
"filterableAttributes": [],
|
||||
"sortableAttributes": [],
|
||||
"rankingRules": [
|
||||
"words",
|
||||
"typo",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"sort",
|
||||
"exactness"
|
||||
],
|
||||
"stopWords": [],
|
||||
"synonyms": {},
|
||||
"distinctAttribute": null
|
||||
}
|
||||
@ -0,0 +1,39 @@
|
||||
---
|
||||
source: dump/src/reader/compat/v3_to_v4.rs
|
||||
expression: products.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": [
|
||||
"*"
|
||||
],
|
||||
"searchableAttributes": [
|
||||
"*"
|
||||
],
|
||||
"filterableAttributes": [],
|
||||
"sortableAttributes": [],
|
||||
"rankingRules": [
|
||||
"words",
|
||||
"typo",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"sort",
|
||||
"exactness"
|
||||
],
|
||||
"stopWords": [],
|
||||
"synonyms": {
|
||||
"android": [
|
||||
"phone",
|
||||
"smartphone"
|
||||
],
|
||||
"iphone": [
|
||||
"phone",
|
||||
"smartphone"
|
||||
],
|
||||
"phone": [
|
||||
"android",
|
||||
"iphone",
|
||||
"smartphone"
|
||||
]
|
||||
},
|
||||
"distinctAttribute": null
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
---
|
||||
source: dump/src/reader/compat/v3_to_v4.rs
|
||||
expression: movies.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": [
|
||||
"*"
|
||||
],
|
||||
"searchableAttributes": [
|
||||
"*"
|
||||
],
|
||||
"filterableAttributes": [
|
||||
"genres",
|
||||
"id"
|
||||
],
|
||||
"sortableAttributes": [
|
||||
"release_date"
|
||||
],
|
||||
"rankingRules": [
|
||||
"words",
|
||||
"typo",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"sort",
|
||||
"exactness",
|
||||
"release_date:asc"
|
||||
],
|
||||
"stopWords": [],
|
||||
"synonyms": {},
|
||||
"distinctAttribute": null
|
||||
}
|
||||
@ -0,0 +1,56 @@
|
||||
---
|
||||
source: dump/src/reader/compat/v4_to_v5.rs
|
||||
expression: spells.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": "Reset",
|
||||
"searchableAttributes": "Reset",
|
||||
"filterableAttributes": {
|
||||
"Set": []
|
||||
},
|
||||
"sortableAttributes": {
|
||||
"Set": []
|
||||
},
|
||||
"rankingRules": {
|
||||
"Set": [
|
||||
"words",
|
||||
"typo",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"sort",
|
||||
"exactness"
|
||||
]
|
||||
},
|
||||
"stopWords": {
|
||||
"Set": []
|
||||
},
|
||||
"synonyms": {
|
||||
"Set": {}
|
||||
},
|
||||
"distinctAttribute": "Reset",
|
||||
"typoTolerance": {
|
||||
"Set": {
|
||||
"enabled": {
|
||||
"Set": true
|
||||
},
|
||||
"minWordSizeForTypos": {
|
||||
"Set": {
|
||||
"oneTypo": {
|
||||
"Set": 5
|
||||
},
|
||||
"twoTypos": {
|
||||
"Set": 9
|
||||
}
|
||||
}
|
||||
},
|
||||
"disableOnWords": {
|
||||
"Set": []
|
||||
},
|
||||
"disableOnAttributes": {
|
||||
"Set": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"faceting": "NotSet",
|
||||
"pagination": "NotSet"
|
||||
}
|
||||
@ -0,0 +1,70 @@
|
||||
---
|
||||
source: dump/src/reader/compat/v4_to_v5.rs
|
||||
expression: products.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": "Reset",
|
||||
"searchableAttributes": "Reset",
|
||||
"filterableAttributes": {
|
||||
"Set": []
|
||||
},
|
||||
"sortableAttributes": {
|
||||
"Set": []
|
||||
},
|
||||
"rankingRules": {
|
||||
"Set": [
|
||||
"words",
|
||||
"typo",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"sort",
|
||||
"exactness"
|
||||
]
|
||||
},
|
||||
"stopWords": {
|
||||
"Set": []
|
||||
},
|
||||
"synonyms": {
|
||||
"Set": {
|
||||
"android": [
|
||||
"phone",
|
||||
"smartphone"
|
||||
],
|
||||
"iphone": [
|
||||
"phone",
|
||||
"smartphone"
|
||||
],
|
||||
"phone": [
|
||||
"android",
|
||||
"iphone",
|
||||
"smartphone"
|
||||
]
|
||||
}
|
||||
},
|
||||
"distinctAttribute": "Reset",
|
||||
"typoTolerance": {
|
||||
"Set": {
|
||||
"enabled": {
|
||||
"Set": true
|
||||
},
|
||||
"minWordSizeForTypos": {
|
||||
"Set": {
|
||||
"oneTypo": {
|
||||
"Set": 5
|
||||
},
|
||||
"twoTypos": {
|
||||
"Set": 9
|
||||
}
|
||||
}
|
||||
},
|
||||
"disableOnWords": {
|
||||
"Set": []
|
||||
},
|
||||
"disableOnAttributes": {
|
||||
"Set": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"faceting": "NotSet",
|
||||
"pagination": "NotSet"
|
||||
}
|
||||
@ -0,0 +1,62 @@
|
||||
---
|
||||
source: dump/src/reader/compat/v4_to_v5.rs
|
||||
expression: movies.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": "Reset",
|
||||
"searchableAttributes": "Reset",
|
||||
"filterableAttributes": {
|
||||
"Set": [
|
||||
"genres",
|
||||
"id"
|
||||
]
|
||||
},
|
||||
"sortableAttributes": {
|
||||
"Set": [
|
||||
"release_date"
|
||||
]
|
||||
},
|
||||
"rankingRules": {
|
||||
"Set": [
|
||||
"words",
|
||||
"typo",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"sort",
|
||||
"exactness",
|
||||
"release_date:asc"
|
||||
]
|
||||
},
|
||||
"stopWords": {
|
||||
"Set": []
|
||||
},
|
||||
"synonyms": {
|
||||
"Set": {}
|
||||
},
|
||||
"distinctAttribute": "Reset",
|
||||
"typoTolerance": {
|
||||
"Set": {
|
||||
"enabled": {
|
||||
"Set": true
|
||||
},
|
||||
"minWordSizeForTypos": {
|
||||
"Set": {
|
||||
"oneTypo": {
|
||||
"Set": 5
|
||||
},
|
||||
"twoTypos": {
|
||||
"Set": 9
|
||||
}
|
||||
}
|
||||
},
|
||||
"disableOnWords": {
|
||||
"Set": []
|
||||
},
|
||||
"disableOnAttributes": {
|
||||
"Set": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"faceting": "NotSet",
|
||||
"pagination": "NotSet"
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
---
|
||||
source: dump/src/reader/compat/v5_to_v6.rs
|
||||
expression: spells.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": [
|
||||
"*"
|
||||
],
|
||||
"searchableAttributes": [
|
||||
"*"
|
||||
],
|
||||
"filterableAttributes": [],
|
||||
"sortableAttributes": [],
|
||||
"rankingRules": [
|
||||
"words",
|
||||
"typo",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"sort",
|
||||
"exactness"
|
||||
],
|
||||
"stopWords": [],
|
||||
"synonyms": {},
|
||||
"distinctAttribute": null,
|
||||
"typoTolerance": {
|
||||
"enabled": true,
|
||||
"minWordSizeForTypos": {
|
||||
"oneTypo": 5,
|
||||
"twoTypos": 9
|
||||
},
|
||||
"disableOnWords": [],
|
||||
"disableOnAttributes": []
|
||||
},
|
||||
"faceting": {
|
||||
"maxValuesPerFacet": 100
|
||||
},
|
||||
"pagination": {
|
||||
"maxTotalHits": 1000
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,54 @@
|
||||
---
|
||||
source: dump/src/reader/compat/v5_to_v6.rs
|
||||
expression: products.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": [
|
||||
"*"
|
||||
],
|
||||
"searchableAttributes": [
|
||||
"*"
|
||||
],
|
||||
"filterableAttributes": [],
|
||||
"sortableAttributes": [],
|
||||
"rankingRules": [
|
||||
"words",
|
||||
"typo",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"sort",
|
||||
"exactness"
|
||||
],
|
||||
"stopWords": [],
|
||||
"synonyms": {
|
||||
"android": [
|
||||
"phone",
|
||||
"smartphone"
|
||||
],
|
||||
"iphone": [
|
||||
"phone",
|
||||
"smartphone"
|
||||
],
|
||||
"phone": [
|
||||
"android",
|
||||
"iphone",
|
||||
"smartphone"
|
||||
]
|
||||
},
|
||||
"distinctAttribute": null,
|
||||
"typoTolerance": {
|
||||
"enabled": true,
|
||||
"minWordSizeForTypos": {
|
||||
"oneTypo": 5,
|
||||
"twoTypos": 9
|
||||
},
|
||||
"disableOnWords": [],
|
||||
"disableOnAttributes": []
|
||||
},
|
||||
"faceting": {
|
||||
"maxValuesPerFacet": 100
|
||||
},
|
||||
"pagination": {
|
||||
"maxTotalHits": 1000
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,46 @@
|
||||
---
|
||||
source: dump/src/reader/compat/v5_to_v6.rs
|
||||
expression: movies.settings().unwrap()
|
||||
---
|
||||
{
|
||||
"displayedAttributes": [
|
||||
"*"
|
||||
],
|
||||
"searchableAttributes": [
|
||||
"*"
|
||||
],
|
||||
"filterableAttributes": [
|
||||
"genres",
|
||||
"id"
|
||||
],
|
||||
"sortableAttributes": [
|
||||
"release_date"
|
||||
],
|
||||
"rankingRules": [
|
||||
"words",
|
||||
"typo",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"sort",
|
||||
"exactness",
|
||||
"release_date:asc"
|
||||
],
|
||||
"stopWords": [],
|
||||
"synonyms": {},
|
||||
"distinctAttribute": null,
|
||||
"typoTolerance": {
|
||||
"enabled": true,
|
||||
"minWordSizeForTypos": {
|
||||
"oneTypo": 5,
|
||||
"twoTypos": 9
|
||||
},
|
||||
"disableOnWords": [],
|
||||
"disableOnAttributes": []
|
||||
},
|
||||
"faceting": {
|
||||
"maxValuesPerFacet": 100
|
||||
},
|
||||
"pagination": {
|
||||
"maxTotalHits": 1000
|
||||
}
|
||||
}
|
||||
410
crates/dump/src/reader/compat/v1_to_v2.rs
Normal file
410
crates/dump/src/reader/compat/v1_to_v2.rs
Normal file
@ -0,0 +1,410 @@
|
||||
use std::str::FromStr;
|
||||
|
||||
use super::v2_to_v3::CompatV2ToV3;
|
||||
use crate::reader::{v1, v2, Document};
|
||||
use crate::Result;
|
||||
|
||||
pub struct CompatV1ToV2 {
|
||||
pub from: v1::V1Reader,
|
||||
}
|
||||
|
||||
impl CompatV1ToV2 {
|
||||
pub fn new(v1: v1::V1Reader) -> Self {
|
||||
Self { from: v1 }
|
||||
}
|
||||
|
||||
pub fn to_v3(self) -> CompatV2ToV3 {
|
||||
CompatV2ToV3::Compat(self)
|
||||
}
|
||||
|
||||
pub fn version(&self) -> crate::Version {
|
||||
self.from.version()
|
||||
}
|
||||
|
||||
pub fn date(&self) -> Option<time::OffsetDateTime> {
|
||||
self.from.date()
|
||||
}
|
||||
|
||||
pub fn index_uuid(&self) -> Vec<v2::meta::IndexUuid> {
|
||||
self.from
|
||||
.index_uuid()
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
// we use the index of the index 😬 as UUID for the index, so that we can link the v2::Task to their index
|
||||
.map(|(index, index_uuid)| v2::meta::IndexUuid {
|
||||
uid: index_uuid.uid,
|
||||
uuid: uuid::Uuid::from_u128(index as u128),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn indexes(&self) -> Result<impl Iterator<Item = Result<CompatIndexV1ToV2>> + '_> {
|
||||
Ok(self.from.indexes()?.map(|index_reader| Ok(CompatIndexV1ToV2 { from: index_reader? })))
|
||||
}
|
||||
|
||||
pub fn tasks(
|
||||
&mut self,
|
||||
) -> Box<dyn Iterator<Item = Result<(v2::Task, Option<v2::UpdateFile>)>> + '_> {
|
||||
// Convert an error here to an iterator yielding the error
|
||||
let indexes = match self.from.indexes() {
|
||||
Ok(indexes) => indexes,
|
||||
Err(err) => return Box::new(std::iter::once(Err(err))),
|
||||
};
|
||||
let it = indexes.enumerate().flat_map(
|
||||
move |(index, index_reader)| -> Box<dyn Iterator<Item = _>> {
|
||||
let index_reader = match index_reader {
|
||||
Ok(index_reader) => index_reader,
|
||||
Err(err) => return Box::new(std::iter::once(Err(err))),
|
||||
};
|
||||
Box::new(
|
||||
index_reader
|
||||
.tasks()
|
||||
// Filter out the UpdateStatus::Customs variant that is not supported in v2
|
||||
// and enqueued tasks, that don't contain the necessary update file in v1
|
||||
.filter_map(move |task| -> Option<_> {
|
||||
let task = match task {
|
||||
Ok(task) => task,
|
||||
Err(err) => return Some(Err(err)),
|
||||
};
|
||||
Some(Ok((
|
||||
v2::Task {
|
||||
uuid: uuid::Uuid::from_u128(index as u128),
|
||||
update: Option::from(task)?,
|
||||
},
|
||||
None,
|
||||
)))
|
||||
}),
|
||||
)
|
||||
},
|
||||
);
|
||||
Box::new(it)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CompatIndexV1ToV2 {
|
||||
pub from: v1::V1IndexReader,
|
||||
}
|
||||
|
||||
impl CompatIndexV1ToV2 {
|
||||
pub fn metadata(&self) -> &crate::IndexMetadata {
|
||||
self.from.metadata()
|
||||
}
|
||||
|
||||
pub fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<Document>> + '_>> {
|
||||
self.from.documents().map(|it| Box::new(it) as Box<dyn Iterator<Item = _>>)
|
||||
}
|
||||
|
||||
pub fn settings(&mut self) -> Result<v2::settings::Settings<v2::settings::Checked>> {
|
||||
Ok(v2::settings::Settings::<v2::settings::Unchecked>::from(self.from.settings()?).check())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<v1::settings::Settings> for v2::Settings<v2::Unchecked> {
|
||||
fn from(source: v1::settings::Settings) -> Self {
|
||||
Self {
|
||||
displayed_attributes: option_to_setting(source.displayed_attributes)
|
||||
.map(|displayed| displayed.into_iter().collect()),
|
||||
searchable_attributes: option_to_setting(source.searchable_attributes),
|
||||
filterable_attributes: option_to_setting(source.attributes_for_faceting.clone())
|
||||
.map(|filterable| filterable.into_iter().collect()),
|
||||
sortable_attributes: option_to_setting(source.attributes_for_faceting)
|
||||
.map(|sortable| sortable.into_iter().collect()),
|
||||
ranking_rules: option_to_setting(source.ranking_rules).map(|ranking_rules| {
|
||||
ranking_rules
|
||||
.into_iter()
|
||||
.filter_map(|ranking_rule| {
|
||||
match v1::settings::RankingRule::from_str(&ranking_rule) {
|
||||
Ok(ranking_rule) => {
|
||||
let criterion: Option<v2::settings::Criterion> =
|
||||
ranking_rule.into();
|
||||
criterion.as_ref().map(ToString::to_string)
|
||||
}
|
||||
Err(()) => {
|
||||
tracing::warn!(
|
||||
"Could not import the following ranking rule: `{}`.",
|
||||
ranking_rule
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}),
|
||||
stop_words: option_to_setting(source.stop_words),
|
||||
synonyms: option_to_setting(source.synonyms),
|
||||
distinct_attribute: option_to_setting(source.distinct_attribute),
|
||||
_kind: std::marker::PhantomData,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn option_to_setting<T>(opt: Option<Option<T>>) -> v2::Setting<T> {
|
||||
match opt {
|
||||
Some(Some(t)) => v2::Setting::Set(t),
|
||||
None => v2::Setting::NotSet,
|
||||
Some(None) => v2::Setting::Reset,
|
||||
}
|
||||
}
|
||||
|
||||
impl From<v1::update::UpdateStatus> for Option<v2::updates::UpdateStatus> {
|
||||
fn from(source: v1::update::UpdateStatus) -> Self {
|
||||
use v1::update::UpdateStatus as UpdateStatusV1;
|
||||
use v2::updates::UpdateStatus as UpdateStatusV2;
|
||||
Some(match source {
|
||||
UpdateStatusV1::Enqueued { content } => {
|
||||
tracing::warn!(
|
||||
"Cannot import task {} (importing enqueued tasks from v1 dumps is unsupported)",
|
||||
content.update_id
|
||||
);
|
||||
tracing::warn!("Task will be skipped in the queue of imported tasks.");
|
||||
|
||||
return None;
|
||||
}
|
||||
UpdateStatusV1::Failed { content } => UpdateStatusV2::Failed(v2::updates::Failed {
|
||||
from: v2::updates::Processing {
|
||||
from: v2::updates::Enqueued {
|
||||
update_id: content.update_id,
|
||||
meta: Option::from(content.update_type)?,
|
||||
enqueued_at: content.enqueued_at,
|
||||
content: None,
|
||||
},
|
||||
started_processing_at: content.processed_at
|
||||
- std::time::Duration::from_secs_f64(content.duration),
|
||||
},
|
||||
error: v2::ResponseError {
|
||||
// error code is ignored by serialization, and so always default in deserialized v2 dumps
|
||||
// that's a good thing, because we don't have them in v1 dump 😅
|
||||
code: http::StatusCode::default(),
|
||||
message: content.error.unwrap_or_default(),
|
||||
// error codes are unchanged between v1 and v2
|
||||
error_code: content.error_code.unwrap_or_default(),
|
||||
// error types are unchanged between v1 and v2
|
||||
error_type: content.error_type.unwrap_or_default(),
|
||||
// error links are unchanged between v1 and v2
|
||||
error_link: content.error_link.unwrap_or_default(),
|
||||
},
|
||||
failed_at: content.processed_at,
|
||||
}),
|
||||
UpdateStatusV1::Processed { content } => {
|
||||
UpdateStatusV2::Processed(v2::updates::Processed {
|
||||
success: match &content.update_type {
|
||||
v1::update::UpdateType::ClearAll => {
|
||||
v2::updates::UpdateResult::DocumentDeletion { deleted: u64::MAX }
|
||||
}
|
||||
v1::update::UpdateType::Customs => v2::updates::UpdateResult::Other,
|
||||
v1::update::UpdateType::DocumentsAddition { number } => {
|
||||
v2::updates::UpdateResult::DocumentsAddition(
|
||||
v2::updates::DocumentAdditionResult { nb_documents: *number },
|
||||
)
|
||||
}
|
||||
v1::update::UpdateType::DocumentsPartial { number } => {
|
||||
v2::updates::UpdateResult::DocumentsAddition(
|
||||
v2::updates::DocumentAdditionResult { nb_documents: *number },
|
||||
)
|
||||
}
|
||||
v1::update::UpdateType::DocumentsDeletion { number } => {
|
||||
v2::updates::UpdateResult::DocumentDeletion { deleted: *number as u64 }
|
||||
}
|
||||
v1::update::UpdateType::Settings { .. } => v2::updates::UpdateResult::Other,
|
||||
},
|
||||
processed_at: content.processed_at,
|
||||
from: v2::updates::Processing {
|
||||
from: v2::updates::Enqueued {
|
||||
update_id: content.update_id,
|
||||
meta: Option::from(content.update_type)?,
|
||||
enqueued_at: content.enqueued_at,
|
||||
content: None,
|
||||
},
|
||||
started_processing_at: content.processed_at
|
||||
- std::time::Duration::from_secs_f64(content.duration),
|
||||
},
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl From<v1::update::UpdateType> for Option<v2::updates::UpdateMeta> {
|
||||
fn from(source: v1::update::UpdateType) -> Self {
|
||||
Some(match source {
|
||||
v1::update::UpdateType::ClearAll => v2::updates::UpdateMeta::ClearDocuments,
|
||||
v1::update::UpdateType::Customs => {
|
||||
tracing::warn!("Ignoring task with type 'Customs' that is no longer supported");
|
||||
return None;
|
||||
}
|
||||
v1::update::UpdateType::DocumentsAddition { .. } => {
|
||||
v2::updates::UpdateMeta::DocumentsAddition {
|
||||
method: v2::updates::IndexDocumentsMethod::ReplaceDocuments,
|
||||
format: v2::updates::UpdateFormat::Json,
|
||||
primary_key: None,
|
||||
}
|
||||
}
|
||||
v1::update::UpdateType::DocumentsPartial { .. } => {
|
||||
v2::updates::UpdateMeta::DocumentsAddition {
|
||||
method: v2::updates::IndexDocumentsMethod::UpdateDocuments,
|
||||
format: v2::updates::UpdateFormat::Json,
|
||||
primary_key: None,
|
||||
}
|
||||
}
|
||||
v1::update::UpdateType::DocumentsDeletion { .. } => {
|
||||
v2::updates::UpdateMeta::DeleteDocuments { ids: vec![] }
|
||||
}
|
||||
v1::update::UpdateType::Settings { settings } => {
|
||||
v2::updates::UpdateMeta::Settings((*settings).into())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl From<v1::settings::SettingsUpdate> for v2::Settings<v2::Unchecked> {
|
||||
fn from(source: v1::settings::SettingsUpdate) -> Self {
|
||||
let ranking_rules = v2::Setting::from(source.ranking_rules);
|
||||
|
||||
// go from the concrete types of v1 (RankingRule) to the concrete type of v2 (Criterion),
|
||||
// and then back to string as this is what the settings manipulate
|
||||
let ranking_rules = ranking_rules.map(|ranking_rules| {
|
||||
ranking_rules
|
||||
.into_iter()
|
||||
// filter out the WordsPosition ranking rule that exists in v1 but not v2
|
||||
.filter_map(Option::<v2::settings::Criterion>::from)
|
||||
.map(|criterion| criterion.to_string())
|
||||
.collect()
|
||||
});
|
||||
|
||||
Self {
|
||||
displayed_attributes: v2::Setting::from(source.displayed_attributes)
|
||||
.map(|displayed_attributes| displayed_attributes.into_iter().collect()),
|
||||
searchable_attributes: source.searchable_attributes.into(),
|
||||
filterable_attributes: v2::Setting::from(source.attributes_for_faceting.clone())
|
||||
.map(|attributes_for_faceting| attributes_for_faceting.into_iter().collect()),
|
||||
sortable_attributes: v2::Setting::from(source.attributes_for_faceting)
|
||||
.map(|attributes_for_faceting| attributes_for_faceting.into_iter().collect()),
|
||||
ranking_rules,
|
||||
stop_words: source.stop_words.into(),
|
||||
synonyms: source.synonyms.into(),
|
||||
distinct_attribute: source.distinct_attribute.into(),
|
||||
_kind: std::marker::PhantomData,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<v1::settings::RankingRule> for Option<v2::settings::Criterion> {
|
||||
fn from(source: v1::settings::RankingRule) -> Self {
|
||||
match source {
|
||||
v1::settings::RankingRule::Typo => Some(v2::settings::Criterion::Typo),
|
||||
v1::settings::RankingRule::Words => Some(v2::settings::Criterion::Words),
|
||||
v1::settings::RankingRule::Proximity => Some(v2::settings::Criterion::Proximity),
|
||||
v1::settings::RankingRule::Attribute => Some(v2::settings::Criterion::Attribute),
|
||||
v1::settings::RankingRule::WordsPosition => {
|
||||
tracing::warn!("Removing the 'WordsPosition' ranking rule that is no longer supported, please check the resulting ranking rules of your indexes");
|
||||
None
|
||||
}
|
||||
v1::settings::RankingRule::Exactness => Some(v2::settings::Criterion::Exactness),
|
||||
v1::settings::RankingRule::Asc(field_name) => {
|
||||
Some(v2::settings::Criterion::Asc(field_name))
|
||||
}
|
||||
v1::settings::RankingRule::Desc(field_name) => {
|
||||
Some(v2::settings::Criterion::Desc(field_name))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> From<v1::settings::UpdateState<T>> for v2::Setting<T> {
|
||||
fn from(source: v1::settings::UpdateState<T>) -> Self {
|
||||
match source {
|
||||
v1::settings::UpdateState::Update(new_value) => v2::Setting::Set(new_value),
|
||||
v1::settings::UpdateState::Clear => v2::Setting::Reset,
|
||||
v1::settings::UpdateState::Nothing => v2::Setting::NotSet,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test {
|
||||
use std::fs::File;
|
||||
use std::io::BufReader;
|
||||
|
||||
use flate2::bufread::GzDecoder;
|
||||
use meili_snap::insta;
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn compat_v1_v2() {
|
||||
let dump = File::open("tests/assets/v1.dump").unwrap();
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut dump = BufReader::new(dump);
|
||||
let gz = GzDecoder::new(&mut dump);
|
||||
let mut archive = tar::Archive::new(gz);
|
||||
archive.unpack(dir.path()).unwrap();
|
||||
|
||||
let mut dump = v1::V1Reader::open(dir).unwrap().to_v2();
|
||||
|
||||
// top level infos
|
||||
assert_eq!(dump.date(), None);
|
||||
|
||||
// tasks
|
||||
let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();
|
||||
let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
|
||||
meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"2298010973ee98cf4670787314176a3a");
|
||||
assert_eq!(update_files.len(), 9);
|
||||
assert!(update_files[..].iter().all(|u| u.is_none())); // no update file in dumps v1
|
||||
|
||||
// indexes
|
||||
let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
// the index are not ordered in any way by default
|
||||
indexes.sort_by_key(|index| index.metadata().uid.to_string());
|
||||
|
||||
let mut products = indexes.pop().unwrap();
|
||||
let mut movies = indexes.pop().unwrap();
|
||||
let mut spells = indexes.pop().unwrap();
|
||||
assert!(indexes.is_empty());
|
||||
|
||||
// products
|
||||
insta::assert_json_snapshot!(products.metadata(), @r###"
|
||||
{
|
||||
"uid": "products",
|
||||
"primaryKey": "sku",
|
||||
"createdAt": "2022-10-02T13:23:39.976870431Z",
|
||||
"updatedAt": "2022-10-02T13:27:54.353262482Z"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(products.settings().unwrap());
|
||||
let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 10);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
|
||||
|
||||
// movies
|
||||
insta::assert_json_snapshot!(movies.metadata(), @r###"
|
||||
{
|
||||
"uid": "movies",
|
||||
"primaryKey": "id",
|
||||
"createdAt": "2022-10-02T13:15:29.477512777Z",
|
||||
"updatedAt": "2022-10-02T13:21:12.671204856Z"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(movies.settings().unwrap());
|
||||
let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 10);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b63dbed5bbc059f3e32bc471ae699bf5");
|
||||
|
||||
// spells
|
||||
insta::assert_json_snapshot!(spells.metadata(), @r###"
|
||||
{
|
||||
"uid": "dnd_spells",
|
||||
"primaryKey": "index",
|
||||
"createdAt": "2022-10-02T13:38:26.358882984Z",
|
||||
"updatedAt": "2022-10-02T13:38:26.385609433Z"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(spells.settings().unwrap());
|
||||
let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 10);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"aa24c0cfc733d66c396237ad44263bed");
|
||||
}
|
||||
}
|
||||
512
crates/dump/src/reader/compat/v2_to_v3.rs
Normal file
512
crates/dump/src/reader/compat/v2_to_v3.rs
Normal file
@ -0,0 +1,512 @@
|
||||
use std::str::FromStr;
|
||||
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::v1_to_v2::{CompatIndexV1ToV2, CompatV1ToV2};
|
||||
use super::v3_to_v4::CompatV3ToV4;
|
||||
use crate::reader::{v2, v3, Document};
|
||||
use crate::Result;
|
||||
|
||||
pub enum CompatV2ToV3 {
|
||||
V2(v2::V2Reader),
|
||||
Compat(CompatV1ToV2),
|
||||
}
|
||||
|
||||
impl CompatV2ToV3 {
|
||||
pub fn new(v2: v2::V2Reader) -> CompatV2ToV3 {
|
||||
CompatV2ToV3::V2(v2)
|
||||
}
|
||||
|
||||
pub fn index_uuid(&self) -> Vec<v3::meta::IndexUuid> {
|
||||
let v2_uuids = match self {
|
||||
CompatV2ToV3::V2(from) => from.index_uuid(),
|
||||
CompatV2ToV3::Compat(compat) => compat.index_uuid(),
|
||||
};
|
||||
v2_uuids
|
||||
.into_iter()
|
||||
.map(|index| v3::meta::IndexUuid { uid: index.uid, uuid: index.uuid })
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn to_v4(self) -> CompatV3ToV4 {
|
||||
CompatV3ToV4::Compat(self)
|
||||
}
|
||||
|
||||
pub fn version(&self) -> crate::Version {
|
||||
match self {
|
||||
CompatV2ToV3::V2(from) => from.version(),
|
||||
CompatV2ToV3::Compat(compat) => compat.version(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn date(&self) -> Option<time::OffsetDateTime> {
|
||||
match self {
|
||||
CompatV2ToV3::V2(from) => from.date(),
|
||||
CompatV2ToV3::Compat(compat) => compat.date(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn instance_uid(&self) -> Result<Option<uuid::Uuid>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
pub fn indexes(&self) -> Result<impl Iterator<Item = Result<CompatIndexV2ToV3>> + '_> {
|
||||
Ok(match self {
|
||||
CompatV2ToV3::V2(from) => Box::new(from.indexes()?.map(|index_reader| -> Result<_> {
|
||||
let compat = CompatIndexV2ToV3::new(index_reader?);
|
||||
Ok(compat)
|
||||
}))
|
||||
as Box<dyn Iterator<Item = Result<CompatIndexV2ToV3>> + '_>,
|
||||
CompatV2ToV3::Compat(compat) => Box::new(compat.indexes()?.map(|index_reader| {
|
||||
let compat = CompatIndexV2ToV3::Compat(Box::new(index_reader?));
|
||||
Ok(compat)
|
||||
}))
|
||||
as Box<dyn Iterator<Item = Result<CompatIndexV2ToV3>> + '_>,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn tasks(
|
||||
&mut self,
|
||||
) -> Box<
|
||||
dyn Iterator<Item = Result<(v3::Task, Option<Box<dyn Iterator<Item = Result<Document>>>>)>>
|
||||
+ '_,
|
||||
> {
|
||||
let tasks = match self {
|
||||
CompatV2ToV3::V2(from) => from.tasks(),
|
||||
CompatV2ToV3::Compat(compat) => compat.tasks(),
|
||||
};
|
||||
|
||||
Box::new(
|
||||
tasks
|
||||
.map(move |task| {
|
||||
task.map(|(task, content_file)| {
|
||||
let task = v3::Task { uuid: task.uuid, update: task.update.into() };
|
||||
|
||||
Some((
|
||||
task,
|
||||
content_file.map(|content_file| {
|
||||
Box::new(content_file) as Box<dyn Iterator<Item = Result<Document>>>
|
||||
}),
|
||||
))
|
||||
})
|
||||
})
|
||||
.filter_map(|res| res.transpose()),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub enum CompatIndexV2ToV3 {
|
||||
V2(v2::V2IndexReader),
|
||||
Compat(Box<CompatIndexV1ToV2>),
|
||||
}
|
||||
|
||||
impl CompatIndexV2ToV3 {
|
||||
pub fn new(v2: v2::V2IndexReader) -> CompatIndexV2ToV3 {
|
||||
CompatIndexV2ToV3::V2(v2)
|
||||
}
|
||||
|
||||
pub fn metadata(&self) -> &crate::IndexMetadata {
|
||||
match self {
|
||||
CompatIndexV2ToV3::V2(from) => from.metadata(),
|
||||
CompatIndexV2ToV3::Compat(compat) => compat.metadata(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<Document>> + '_>> {
|
||||
match self {
|
||||
CompatIndexV2ToV3::V2(from) => from
|
||||
.documents()
|
||||
.map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<Document>> + '_>),
|
||||
CompatIndexV2ToV3::Compat(compat) => compat.documents(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn settings(&mut self) -> Result<v3::Settings<v3::Checked>> {
|
||||
let settings = match self {
|
||||
CompatIndexV2ToV3::V2(from) => from.settings()?,
|
||||
CompatIndexV2ToV3::Compat(compat) => compat.settings()?,
|
||||
};
|
||||
Ok(v3::Settings::<v3::Unchecked>::from(settings).check())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<v2::updates::UpdateStatus> for v3::updates::UpdateStatus {
|
||||
fn from(update: v2::updates::UpdateStatus) -> Self {
|
||||
match update {
|
||||
v2::updates::UpdateStatus::Processing(processing) => {
|
||||
match (processing.from.meta.clone(), processing.from.content).try_into() {
|
||||
Ok(meta) => v3::updates::UpdateStatus::Processing(v3::updates::Processing {
|
||||
from: v3::updates::Enqueued {
|
||||
update_id: processing.from.update_id,
|
||||
meta,
|
||||
enqueued_at: processing.from.enqueued_at,
|
||||
},
|
||||
started_processing_at: processing.started_processing_at,
|
||||
}),
|
||||
Err(e) => {
|
||||
tracing::warn!("Error with task {}: {}", processing.from.update_id, e);
|
||||
tracing::warn!("Task will be marked as `Failed`.");
|
||||
v3::updates::UpdateStatus::Failed(v3::updates::Failed {
|
||||
from: v3::updates::Processing {
|
||||
from: v3::updates::Enqueued {
|
||||
update_id: processing.from.update_id,
|
||||
meta: update_from_unchecked_update_meta(processing.from.meta),
|
||||
enqueued_at: processing.from.enqueued_at,
|
||||
},
|
||||
started_processing_at: processing.started_processing_at,
|
||||
},
|
||||
msg: e.to_string(),
|
||||
code: v3::Code::MalformedDump,
|
||||
failed_at: OffsetDateTime::now_utc(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
v2::updates::UpdateStatus::Enqueued(enqueued) => {
|
||||
match (enqueued.meta.clone(), enqueued.content).try_into() {
|
||||
Ok(meta) => v3::updates::UpdateStatus::Enqueued(v3::updates::Enqueued {
|
||||
update_id: enqueued.update_id,
|
||||
meta,
|
||||
enqueued_at: enqueued.enqueued_at,
|
||||
}),
|
||||
Err(e) => {
|
||||
tracing::warn!("Error with task {}: {}", enqueued.update_id, e);
|
||||
tracing::warn!("Task will be marked as `Failed`.");
|
||||
v3::updates::UpdateStatus::Failed(v3::updates::Failed {
|
||||
from: v3::updates::Processing {
|
||||
from: v3::updates::Enqueued {
|
||||
update_id: enqueued.update_id,
|
||||
meta: update_from_unchecked_update_meta(enqueued.meta),
|
||||
enqueued_at: enqueued.enqueued_at,
|
||||
},
|
||||
started_processing_at: OffsetDateTime::now_utc(),
|
||||
},
|
||||
msg: e.to_string(),
|
||||
code: v3::Code::MalformedDump,
|
||||
failed_at: OffsetDateTime::now_utc(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
v2::updates::UpdateStatus::Processed(processed) => {
|
||||
v3::updates::UpdateStatus::Processed(v3::updates::Processed {
|
||||
success: processed.success.into(),
|
||||
processed_at: processed.processed_at,
|
||||
from: v3::updates::Processing {
|
||||
from: v3::updates::Enqueued {
|
||||
update_id: processed.from.from.update_id,
|
||||
// since we're never going to read the content_file again it's ok to generate a fake one.
|
||||
meta: update_from_unchecked_update_meta(processed.from.from.meta),
|
||||
enqueued_at: processed.from.from.enqueued_at,
|
||||
},
|
||||
started_processing_at: processed.from.started_processing_at,
|
||||
},
|
||||
})
|
||||
}
|
||||
v2::updates::UpdateStatus::Aborted(aborted) => {
|
||||
v3::updates::UpdateStatus::Aborted(v3::updates::Aborted {
|
||||
from: v3::updates::Enqueued {
|
||||
update_id: aborted.from.update_id,
|
||||
// since we're never going to read the content_file again it's ok to generate a fake one.
|
||||
meta: update_from_unchecked_update_meta(aborted.from.meta),
|
||||
enqueued_at: aborted.from.enqueued_at,
|
||||
},
|
||||
aborted_at: aborted.aborted_at,
|
||||
})
|
||||
}
|
||||
v2::updates::UpdateStatus::Failed(failed) => {
|
||||
v3::updates::UpdateStatus::Failed(v3::updates::Failed {
|
||||
from: v3::updates::Processing {
|
||||
from: v3::updates::Enqueued {
|
||||
update_id: failed.from.from.update_id,
|
||||
// since we're never going to read the content_file again it's ok to generate a fake one.
|
||||
meta: update_from_unchecked_update_meta(failed.from.from.meta),
|
||||
enqueued_at: failed.from.from.enqueued_at,
|
||||
},
|
||||
started_processing_at: failed.from.started_processing_at,
|
||||
},
|
||||
msg: failed.error.message,
|
||||
code: failed.error.error_code.into(),
|
||||
failed_at: failed.failed_at,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<(v2::updates::UpdateMeta, Option<Uuid>)> for v3::updates::Update {
|
||||
type Error = crate::Error;
|
||||
|
||||
fn try_from((update, uuid): (v2::updates::UpdateMeta, Option<Uuid>)) -> Result<Self> {
|
||||
Ok(match update {
|
||||
v2::updates::UpdateMeta::DocumentsAddition { method, format: _, primary_key }
|
||||
if uuid.is_some() =>
|
||||
{
|
||||
v3::updates::Update::DocumentAddition {
|
||||
primary_key,
|
||||
method: match method {
|
||||
v2::updates::IndexDocumentsMethod::ReplaceDocuments => {
|
||||
v3::updates::IndexDocumentsMethod::ReplaceDocuments
|
||||
}
|
||||
v2::updates::IndexDocumentsMethod::UpdateDocuments => {
|
||||
v3::updates::IndexDocumentsMethod::UpdateDocuments
|
||||
}
|
||||
},
|
||||
content_uuid: uuid.unwrap(),
|
||||
}
|
||||
}
|
||||
v2::updates::UpdateMeta::DocumentsAddition { .. } => {
|
||||
return Err(crate::Error::MalformedTask)
|
||||
}
|
||||
v2::updates::UpdateMeta::ClearDocuments => v3::updates::Update::ClearDocuments,
|
||||
v2::updates::UpdateMeta::DeleteDocuments { ids } => {
|
||||
v3::updates::Update::DeleteDocuments(ids)
|
||||
}
|
||||
v2::updates::UpdateMeta::Settings(settings) => {
|
||||
v3::updates::Update::Settings(settings.into())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update_from_unchecked_update_meta(update: v2::updates::UpdateMeta) -> v3::updates::Update {
|
||||
match update {
|
||||
v2::updates::UpdateMeta::DocumentsAddition { method, format: _, primary_key } => {
|
||||
v3::updates::Update::DocumentAddition {
|
||||
primary_key,
|
||||
method: match method {
|
||||
v2::updates::IndexDocumentsMethod::ReplaceDocuments => {
|
||||
v3::updates::IndexDocumentsMethod::ReplaceDocuments
|
||||
}
|
||||
v2::updates::IndexDocumentsMethod::UpdateDocuments => {
|
||||
v3::updates::IndexDocumentsMethod::UpdateDocuments
|
||||
}
|
||||
},
|
||||
// we use this special uuid so we can recognize it if one day there is a bug related to this field.
|
||||
content_uuid: Uuid::from_str("00112233-4455-6677-8899-aabbccddeeff").unwrap(),
|
||||
}
|
||||
}
|
||||
v2::updates::UpdateMeta::ClearDocuments => v3::updates::Update::ClearDocuments,
|
||||
v2::updates::UpdateMeta::DeleteDocuments { ids } => {
|
||||
v3::updates::Update::DeleteDocuments(ids)
|
||||
}
|
||||
v2::updates::UpdateMeta::Settings(settings) => {
|
||||
v3::updates::Update::Settings(settings.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<v2::updates::UpdateResult> for v3::updates::UpdateResult {
|
||||
fn from(result: v2::updates::UpdateResult) -> Self {
|
||||
match result {
|
||||
v2::updates::UpdateResult::DocumentsAddition(addition) => {
|
||||
v3::updates::UpdateResult::DocumentsAddition(v3::updates::DocumentAdditionResult {
|
||||
nb_documents: addition.nb_documents,
|
||||
})
|
||||
}
|
||||
v2::updates::UpdateResult::DocumentDeletion { deleted } => {
|
||||
v3::updates::UpdateResult::DocumentDeletion { deleted }
|
||||
}
|
||||
v2::updates::UpdateResult::Other => v3::updates::UpdateResult::Other,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<String> for v3::Code {
|
||||
fn from(code: String) -> Self {
|
||||
match code.as_ref() {
|
||||
"create_index" => v3::Code::CreateIndex,
|
||||
"index_already_exists" => v3::Code::IndexAlreadyExists,
|
||||
"index_not_found" => v3::Code::IndexNotFound,
|
||||
"invalid_index_uid" => v3::Code::InvalidIndexUid,
|
||||
"invalid_state" => v3::Code::InvalidState,
|
||||
"missing_primary_key" => v3::Code::MissingPrimaryKey,
|
||||
"primary_key_already_present" => v3::Code::PrimaryKeyAlreadyPresent,
|
||||
"max_fields_limit_exceeded" => v3::Code::MaxFieldsLimitExceeded,
|
||||
"missing_document_id" => v3::Code::MissingDocumentId,
|
||||
"invalid_document_id" => v3::Code::InvalidDocumentId,
|
||||
"filter" => v3::Code::Filter,
|
||||
"sort" => v3::Code::Sort,
|
||||
"bad_parameter" => v3::Code::BadParameter,
|
||||
"bad_request" => v3::Code::BadRequest,
|
||||
"database_size_limit_reached" => v3::Code::DatabaseSizeLimitReached,
|
||||
"document_not_found" => v3::Code::DocumentNotFound,
|
||||
"internal" => v3::Code::Internal,
|
||||
"invalid_geo_field" => v3::Code::InvalidGeoField,
|
||||
"invalid_ranking_rule" => v3::Code::InvalidRankingRule,
|
||||
"invalid_store" => v3::Code::InvalidStore,
|
||||
"invalid_token" => v3::Code::InvalidToken,
|
||||
"missing_authorization_header" => v3::Code::MissingAuthorizationHeader,
|
||||
"no_space_left_on_device" => v3::Code::NoSpaceLeftOnDevice,
|
||||
"dump_not_found" => v3::Code::DumpNotFound,
|
||||
"task_not_found" => v3::Code::TaskNotFound,
|
||||
"payload_too_large" => v3::Code::PayloadTooLarge,
|
||||
"retrieve_document" => v3::Code::RetrieveDocument,
|
||||
"search_documents" => v3::Code::SearchDocuments,
|
||||
"unsupported_media_type" => v3::Code::UnsupportedMediaType,
|
||||
"dump_already_in_progress" => v3::Code::DumpAlreadyInProgress,
|
||||
"dump_process_failed" => v3::Code::DumpProcessFailed,
|
||||
"invalid_content_type" => v3::Code::InvalidContentType,
|
||||
"missing_content_type" => v3::Code::MissingContentType,
|
||||
"malformed_payload" => v3::Code::MalformedPayload,
|
||||
"missing_payload" => v3::Code::MissingPayload,
|
||||
other => {
|
||||
tracing::warn!("Unknown error code {}", other);
|
||||
v3::Code::UnretrievableErrorCode
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<A> From<v2::Setting<A>> for v3::Setting<A> {
|
||||
fn from(setting: v2::Setting<A>) -> Self {
|
||||
match setting {
|
||||
v2::settings::Setting::Set(a) => v3::settings::Setting::Set(a),
|
||||
v2::settings::Setting::Reset => v3::settings::Setting::Reset,
|
||||
v2::settings::Setting::NotSet => v3::settings::Setting::NotSet,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> From<v2::Settings<T>> for v3::Settings<v3::Unchecked> {
|
||||
fn from(settings: v2::Settings<T>) -> Self {
|
||||
v3::Settings {
|
||||
displayed_attributes: settings.displayed_attributes.into(),
|
||||
searchable_attributes: settings.searchable_attributes.into(),
|
||||
filterable_attributes: settings.filterable_attributes.into(),
|
||||
sortable_attributes: settings.sortable_attributes.into(),
|
||||
ranking_rules: v3::Setting::from(settings.ranking_rules).map(|criteria| {
|
||||
criteria.into_iter().map(|criterion| patch_ranking_rules(&criterion)).collect()
|
||||
}),
|
||||
stop_words: settings.stop_words.into(),
|
||||
synonyms: settings.synonyms.into(),
|
||||
distinct_attribute: settings.distinct_attribute.into(),
|
||||
_kind: std::marker::PhantomData,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn patch_ranking_rules(ranking_rule: &str) -> String {
|
||||
match v2::settings::Criterion::from_str(ranking_rule) {
|
||||
Ok(v2::settings::Criterion::Words) => String::from("words"),
|
||||
Ok(v2::settings::Criterion::Typo) => String::from("typo"),
|
||||
Ok(v2::settings::Criterion::Proximity) => String::from("proximity"),
|
||||
Ok(v2::settings::Criterion::Attribute) => String::from("attribute"),
|
||||
Ok(v2::settings::Criterion::Sort) => String::from("sort"),
|
||||
Ok(v2::settings::Criterion::Exactness) => String::from("exactness"),
|
||||
Ok(v2::settings::Criterion::Asc(name)) => format!("{name}:asc"),
|
||||
Ok(v2::settings::Criterion::Desc(name)) => format!("{name}:desc"),
|
||||
// we want to forward the error to the current version of meilisearch
|
||||
Err(_) => ranking_rule.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test {
|
||||
use std::fs::File;
|
||||
use std::io::BufReader;
|
||||
|
||||
use flate2::bufread::GzDecoder;
|
||||
use meili_snap::insta;
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn compat_v2_v3() {
|
||||
let dump = File::open("tests/assets/v2.dump").unwrap();
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut dump = BufReader::new(dump);
|
||||
let gz = GzDecoder::new(&mut dump);
|
||||
let mut archive = tar::Archive::new(gz);
|
||||
archive.unpack(dir.path()).unwrap();
|
||||
|
||||
let mut dump = v2::V2Reader::open(dir).unwrap().to_v3();
|
||||
|
||||
// top level infos
|
||||
insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
|
||||
|
||||
// tasks
|
||||
let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();
|
||||
let (tasks, mut update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
|
||||
meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"9507711db47c7171c79bc6d57d0bed79");
|
||||
assert_eq!(update_files.len(), 9);
|
||||
assert!(update_files[0].is_some()); // the enqueued document addition
|
||||
assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
|
||||
|
||||
let update_file = update_files.remove(0).unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
meili_snap::snapshot_hash!(meili_snap::json_string!(update_file), @"7b8889539b669c7b9ddba448bafa385d");
|
||||
|
||||
// indexes
|
||||
let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
// the index are not ordered in any way by default
|
||||
indexes.sort_by_key(|index| index.metadata().uid.to_string());
|
||||
|
||||
let mut products = indexes.pop().unwrap();
|
||||
let mut movies2 = indexes.pop().unwrap();
|
||||
let mut movies = indexes.pop().unwrap();
|
||||
let mut spells = indexes.pop().unwrap();
|
||||
assert!(indexes.is_empty());
|
||||
|
||||
// products
|
||||
insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
|
||||
{
|
||||
"uid": "products",
|
||||
"primaryKey": "sku",
|
||||
"createdAt": "[now]",
|
||||
"updatedAt": "[now]"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(products.settings().unwrap());
|
||||
let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 10);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
|
||||
|
||||
// movies
|
||||
insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
|
||||
{
|
||||
"uid": "movies",
|
||||
"primaryKey": "id",
|
||||
"createdAt": "[now]",
|
||||
"updatedAt": "[now]"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(movies.settings().unwrap());
|
||||
let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 110);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022");
|
||||
|
||||
// movies2
|
||||
insta::assert_json_snapshot!(movies2.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
|
||||
{
|
||||
"uid": "movies_2",
|
||||
"primaryKey": null,
|
||||
"createdAt": "[now]",
|
||||
"updatedAt": "[now]"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(movies2.settings().unwrap());
|
||||
let documents = movies2.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 0);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
|
||||
|
||||
// spells
|
||||
insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
|
||||
{
|
||||
"uid": "dnd_spells",
|
||||
"primaryKey": "index",
|
||||
"createdAt": "[now]",
|
||||
"updatedAt": "[now]"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(spells.settings().unwrap());
|
||||
let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 10);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
|
||||
}
|
||||
}
|
||||
449
crates/dump/src/reader/compat/v3_to_v4.rs
Normal file
449
crates/dump/src/reader/compat/v3_to_v4.rs
Normal file
@ -0,0 +1,449 @@
|
||||
use super::v2_to_v3::{CompatIndexV2ToV3, CompatV2ToV3};
|
||||
use super::v4_to_v5::CompatV4ToV5;
|
||||
use crate::reader::{v3, v4, UpdateFile};
|
||||
use crate::Result;
|
||||
|
||||
pub enum CompatV3ToV4 {
|
||||
V3(v3::V3Reader),
|
||||
Compat(CompatV2ToV3),
|
||||
}
|
||||
|
||||
impl CompatV3ToV4 {
|
||||
pub fn new(v3: v3::V3Reader) -> CompatV3ToV4 {
|
||||
CompatV3ToV4::V3(v3)
|
||||
}
|
||||
|
||||
pub fn to_v5(self) -> CompatV4ToV5 {
|
||||
CompatV4ToV5::Compat(self)
|
||||
}
|
||||
|
||||
pub fn version(&self) -> crate::Version {
|
||||
match self {
|
||||
CompatV3ToV4::V3(v3) => v3.version(),
|
||||
CompatV3ToV4::Compat(compat) => compat.version(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn date(&self) -> Option<time::OffsetDateTime> {
|
||||
match self {
|
||||
CompatV3ToV4::V3(v3) => v3.date(),
|
||||
CompatV3ToV4::Compat(compat) => compat.date(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn instance_uid(&self) -> Result<Option<uuid::Uuid>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
pub fn indexes(&self) -> Result<impl Iterator<Item = Result<CompatIndexV3ToV4>> + '_> {
|
||||
Ok(match self {
|
||||
CompatV3ToV4::V3(v3) => {
|
||||
Box::new(v3.indexes()?.map(|index| index.map(CompatIndexV3ToV4::from)))
|
||||
as Box<dyn Iterator<Item = Result<CompatIndexV3ToV4>> + '_>
|
||||
}
|
||||
|
||||
CompatV3ToV4::Compat(compat) => {
|
||||
Box::new(compat.indexes()?.map(|index| index.map(CompatIndexV3ToV4::from)))
|
||||
as Box<dyn Iterator<Item = Result<CompatIndexV3ToV4>> + '_>
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub fn tasks(
|
||||
&mut self,
|
||||
) -> Box<dyn Iterator<Item = Result<(v4::Task, Option<Box<UpdateFile>>)>> + '_> {
|
||||
let indexes = match self {
|
||||
CompatV3ToV4::V3(v3) => v3.index_uuid(),
|
||||
CompatV3ToV4::Compat(compat) => compat.index_uuid(),
|
||||
};
|
||||
let tasks = match self {
|
||||
CompatV3ToV4::V3(v3) => v3.tasks(),
|
||||
CompatV3ToV4::Compat(compat) => compat.tasks(),
|
||||
};
|
||||
|
||||
Box::new(
|
||||
tasks
|
||||
// we need to override the old task ids that were generated
|
||||
// by index in favor of a global unique incremental ID.
|
||||
.enumerate()
|
||||
.map(move |(task_id, task)| {
|
||||
task.map(|(task, content_file)| {
|
||||
let index_uid = indexes
|
||||
.iter()
|
||||
.find(|index| index.uuid == task.uuid)
|
||||
.map(|index| index.uid.clone());
|
||||
|
||||
let index_uid = match index_uid {
|
||||
Some(uid) => uid,
|
||||
None => {
|
||||
tracing::warn!(
|
||||
"Error while importing the update {}.",
|
||||
task.update.id()
|
||||
);
|
||||
tracing::warn!(
|
||||
"The index associated to the uuid `{}` could not be retrieved.",
|
||||
task.uuid.to_string()
|
||||
);
|
||||
if task.update.is_finished() {
|
||||
// we're fucking with his history but not his data, that's ok-ish.
|
||||
tracing::warn!("The index-uuid will be set as `unknown`.");
|
||||
String::from("unknown")
|
||||
} else {
|
||||
tracing::warn!("The task will be ignored.");
|
||||
return None;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let task = v4::Task {
|
||||
id: task_id as u32,
|
||||
index_uid: v4::meta::IndexUid(index_uid),
|
||||
content: match task.update.meta() {
|
||||
v3::Kind::DeleteDocuments(documents) => {
|
||||
v4::tasks::TaskContent::DocumentDeletion(
|
||||
v4::tasks::DocumentDeletion::Ids(documents.clone()),
|
||||
)
|
||||
}
|
||||
v3::Kind::DocumentAddition {
|
||||
primary_key,
|
||||
method,
|
||||
content_uuid,
|
||||
} => v4::tasks::TaskContent::DocumentAddition {
|
||||
merge_strategy: match method {
|
||||
v3::updates::IndexDocumentsMethod::ReplaceDocuments => {
|
||||
v4::tasks::IndexDocumentsMethod::ReplaceDocuments
|
||||
}
|
||||
v3::updates::IndexDocumentsMethod::UpdateDocuments => {
|
||||
v4::tasks::IndexDocumentsMethod::UpdateDocuments
|
||||
}
|
||||
},
|
||||
primary_key: primary_key.clone(),
|
||||
documents_count: 0, // we don't have this info
|
||||
allow_index_creation: true, // there was no API-key in the v3
|
||||
content_uuid: *content_uuid,
|
||||
},
|
||||
v3::Kind::Settings(settings) => {
|
||||
v4::tasks::TaskContent::SettingsUpdate {
|
||||
settings: v4::Settings::from(settings.clone()),
|
||||
is_deletion: false, // that didn't exist at this time
|
||||
allow_index_creation: true, // there was no API-key in the v3
|
||||
}
|
||||
}
|
||||
v3::Kind::ClearDocuments => {
|
||||
v4::tasks::TaskContent::DocumentDeletion(
|
||||
v4::tasks::DocumentDeletion::Clear,
|
||||
)
|
||||
}
|
||||
},
|
||||
events: match task.update {
|
||||
v3::Status::Processing(processing) => {
|
||||
vec![v4::tasks::TaskEvent::Created(processing.from.enqueued_at)]
|
||||
}
|
||||
v3::Status::Enqueued(enqueued) => {
|
||||
vec![v4::tasks::TaskEvent::Created(enqueued.enqueued_at)]
|
||||
}
|
||||
v3::Status::Processed(processed) => {
|
||||
vec![
|
||||
v4::tasks::TaskEvent::Created(
|
||||
processed.from.from.enqueued_at,
|
||||
),
|
||||
v4::tasks::TaskEvent::Processing(
|
||||
processed.from.started_processing_at,
|
||||
),
|
||||
v4::tasks::TaskEvent::Succeded {
|
||||
result: match processed.success {
|
||||
v3::updates::UpdateResult::DocumentsAddition(
|
||||
document_addition,
|
||||
) => v4::tasks::TaskResult::DocumentAddition {
|
||||
indexed_documents: document_addition
|
||||
.nb_documents
|
||||
as u64,
|
||||
},
|
||||
v3::updates::UpdateResult::DocumentDeletion {
|
||||
deleted,
|
||||
} => v4::tasks::TaskResult::DocumentDeletion {
|
||||
deleted_documents: deleted,
|
||||
},
|
||||
v3::updates::UpdateResult::Other => {
|
||||
v4::tasks::TaskResult::Other
|
||||
}
|
||||
},
|
||||
timestamp: processed.processed_at,
|
||||
},
|
||||
]
|
||||
}
|
||||
v3::Status::Failed(failed) => vec![
|
||||
v4::tasks::TaskEvent::Created(failed.from.from.enqueued_at),
|
||||
v4::tasks::TaskEvent::Processing(
|
||||
failed.from.started_processing_at,
|
||||
),
|
||||
v4::tasks::TaskEvent::Failed {
|
||||
error: v4::ResponseError::from_msg(
|
||||
failed.msg.to_string(),
|
||||
failed.code.into(),
|
||||
),
|
||||
timestamp: failed.failed_at,
|
||||
},
|
||||
],
|
||||
v3::Status::Aborted(aborted) => vec![
|
||||
v4::tasks::TaskEvent::Created(aborted.from.enqueued_at),
|
||||
v4::tasks::TaskEvent::Failed {
|
||||
error: v4::ResponseError::from_msg(
|
||||
"Task was aborted in a previous version of meilisearch."
|
||||
.to_string(),
|
||||
v4::errors::Code::UnretrievableErrorCode,
|
||||
),
|
||||
timestamp: aborted.aborted_at,
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
Some((task, content_file))
|
||||
})
|
||||
})
|
||||
.filter_map(|res| res.transpose()),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<v4::Key>> + '_> {
|
||||
Box::new(std::iter::empty())
|
||||
}
|
||||
}
|
||||
|
||||
pub enum CompatIndexV3ToV4 {
|
||||
V3(v3::V3IndexReader),
|
||||
Compat(CompatIndexV2ToV3),
|
||||
}
|
||||
|
||||
impl From<v3::V3IndexReader> for CompatIndexV3ToV4 {
|
||||
fn from(index_reader: v3::V3IndexReader) -> Self {
|
||||
Self::V3(index_reader)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<CompatIndexV2ToV3> for CompatIndexV3ToV4 {
|
||||
fn from(index_reader: CompatIndexV2ToV3) -> Self {
|
||||
Self::Compat(index_reader)
|
||||
}
|
||||
}
|
||||
|
||||
impl CompatIndexV3ToV4 {
|
||||
pub fn new(v3: v3::V3IndexReader) -> CompatIndexV3ToV4 {
|
||||
CompatIndexV3ToV4::V3(v3)
|
||||
}
|
||||
|
||||
pub fn metadata(&self) -> &crate::IndexMetadata {
|
||||
match self {
|
||||
CompatIndexV3ToV4::V3(v3) => v3.metadata(),
|
||||
CompatIndexV3ToV4::Compat(compat) => compat.metadata(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<v4::Document>> + '_>> {
|
||||
match self {
|
||||
CompatIndexV3ToV4::V3(v3) => v3
|
||||
.documents()
|
||||
.map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<v4::Document>> + '_>),
|
||||
|
||||
CompatIndexV3ToV4::Compat(compat) => compat
|
||||
.documents()
|
||||
.map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<v4::Document>> + '_>),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn settings(&mut self) -> Result<v4::Settings<v4::Checked>> {
|
||||
Ok(match self {
|
||||
CompatIndexV3ToV4::V3(v3) => {
|
||||
v4::Settings::<v4::Unchecked>::from(v3.settings()?).check()
|
||||
}
|
||||
CompatIndexV3ToV4::Compat(compat) => {
|
||||
v4::Settings::<v4::Unchecked>::from(compat.settings()?).check()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> From<v3::Setting<T>> for v4::Setting<T> {
|
||||
fn from(setting: v3::Setting<T>) -> Self {
|
||||
match setting {
|
||||
v3::Setting::Set(t) => v4::Setting::Set(t),
|
||||
v3::Setting::Reset => v4::Setting::Reset,
|
||||
v3::Setting::NotSet => v4::Setting::NotSet,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<v3::Code> for v4::Code {
|
||||
fn from(code: v3::Code) -> Self {
|
||||
match code {
|
||||
v3::Code::CreateIndex => v4::Code::CreateIndex,
|
||||
v3::Code::IndexAlreadyExists => v4::Code::IndexAlreadyExists,
|
||||
v3::Code::IndexNotFound => v4::Code::IndexNotFound,
|
||||
v3::Code::InvalidIndexUid => v4::Code::InvalidIndexUid,
|
||||
v3::Code::InvalidState => v4::Code::InvalidState,
|
||||
v3::Code::MissingPrimaryKey => v4::Code::MissingPrimaryKey,
|
||||
v3::Code::PrimaryKeyAlreadyPresent => v4::Code::PrimaryKeyAlreadyPresent,
|
||||
v3::Code::MaxFieldsLimitExceeded => v4::Code::MaxFieldsLimitExceeded,
|
||||
v3::Code::MissingDocumentId => v4::Code::MissingDocumentId,
|
||||
v3::Code::InvalidDocumentId => v4::Code::InvalidDocumentId,
|
||||
v3::Code::Filter => v4::Code::Filter,
|
||||
v3::Code::Sort => v4::Code::Sort,
|
||||
v3::Code::BadParameter => v4::Code::BadParameter,
|
||||
v3::Code::BadRequest => v4::Code::BadRequest,
|
||||
v3::Code::DatabaseSizeLimitReached => v4::Code::DatabaseSizeLimitReached,
|
||||
v3::Code::DocumentNotFound => v4::Code::DocumentNotFound,
|
||||
v3::Code::Internal => v4::Code::Internal,
|
||||
v3::Code::InvalidGeoField => v4::Code::InvalidGeoField,
|
||||
v3::Code::InvalidRankingRule => v4::Code::InvalidRankingRule,
|
||||
v3::Code::InvalidStore => v4::Code::InvalidStore,
|
||||
v3::Code::InvalidToken => v4::Code::InvalidToken,
|
||||
v3::Code::MissingAuthorizationHeader => v4::Code::MissingAuthorizationHeader,
|
||||
v3::Code::NoSpaceLeftOnDevice => v4::Code::NoSpaceLeftOnDevice,
|
||||
v3::Code::DumpNotFound => v4::Code::DumpNotFound,
|
||||
v3::Code::TaskNotFound => v4::Code::TaskNotFound,
|
||||
v3::Code::PayloadTooLarge => v4::Code::PayloadTooLarge,
|
||||
v3::Code::RetrieveDocument => v4::Code::RetrieveDocument,
|
||||
v3::Code::SearchDocuments => v4::Code::SearchDocuments,
|
||||
v3::Code::UnsupportedMediaType => v4::Code::UnsupportedMediaType,
|
||||
v3::Code::DumpAlreadyInProgress => v4::Code::DumpAlreadyInProgress,
|
||||
v3::Code::DumpProcessFailed => v4::Code::DumpProcessFailed,
|
||||
v3::Code::InvalidContentType => v4::Code::InvalidContentType,
|
||||
v3::Code::MissingContentType => v4::Code::MissingContentType,
|
||||
v3::Code::MalformedPayload => v4::Code::MalformedPayload,
|
||||
v3::Code::MissingPayload => v4::Code::MissingPayload,
|
||||
v3::Code::UnretrievableErrorCode => v4::Code::UnretrievableErrorCode,
|
||||
v3::Code::MalformedDump => v4::Code::MalformedDump,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> From<v3::Settings<T>> for v4::Settings<v4::Unchecked> {
|
||||
fn from(settings: v3::Settings<T>) -> Self {
|
||||
v4::Settings {
|
||||
displayed_attributes: settings.displayed_attributes.into(),
|
||||
searchable_attributes: settings.searchable_attributes.into(),
|
||||
filterable_attributes: settings.filterable_attributes.into(),
|
||||
sortable_attributes: settings.sortable_attributes.into(),
|
||||
ranking_rules: settings.ranking_rules.into(),
|
||||
stop_words: settings.stop_words.into(),
|
||||
synonyms: settings.synonyms.into(),
|
||||
distinct_attribute: settings.distinct_attribute.into(),
|
||||
typo_tolerance: v4::Setting::NotSet,
|
||||
_kind: std::marker::PhantomData,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test {
|
||||
use std::fs::File;
|
||||
use std::io::BufReader;
|
||||
|
||||
use flate2::bufread::GzDecoder;
|
||||
use meili_snap::insta;
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn compat_v3_v4() {
|
||||
let dump = File::open("tests/assets/v3.dump").unwrap();
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut dump = BufReader::new(dump);
|
||||
let gz = GzDecoder::new(&mut dump);
|
||||
let mut archive = tar::Archive::new(gz);
|
||||
archive.unpack(dir.path()).unwrap();
|
||||
|
||||
let mut dump = v3::V3Reader::open(dir).unwrap().to_v4();
|
||||
|
||||
// top level infos
|
||||
insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
|
||||
|
||||
// tasks
|
||||
let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();
|
||||
let (tasks, mut update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
|
||||
meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"79bc053583a1a7172bbaaafb1edaeb78");
|
||||
assert_eq!(update_files.len(), 10);
|
||||
assert!(update_files[0].is_some()); // the enqueued document addition
|
||||
assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
|
||||
|
||||
let update_file = update_files.remove(0).unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
meili_snap::snapshot_hash!(meili_snap::json_string!(update_file), @"7b8889539b669c7b9ddba448bafa385d");
|
||||
|
||||
// keys
|
||||
let keys = dump.keys().collect::<Result<Vec<_>>>().unwrap();
|
||||
meili_snap::snapshot_hash!(meili_snap::json_string!(keys, { "[].uid" => "[uuid]" }), @"d751713988987e9331980363e24189ce");
|
||||
|
||||
// indexes
|
||||
let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
// the index are not ordered in any way by default
|
||||
indexes.sort_by_key(|index| index.metadata().uid.to_string());
|
||||
|
||||
let mut products = indexes.pop().unwrap();
|
||||
let mut movies2 = indexes.pop().unwrap();
|
||||
let mut movies = indexes.pop().unwrap();
|
||||
let mut spells = indexes.pop().unwrap();
|
||||
assert!(indexes.is_empty());
|
||||
|
||||
// products
|
||||
insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
|
||||
{
|
||||
"uid": "products",
|
||||
"primaryKey": "sku",
|
||||
"createdAt": "[now]",
|
||||
"updatedAt": "[now]"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(products.settings().unwrap());
|
||||
let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 10);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
|
||||
|
||||
// movies
|
||||
insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
|
||||
{
|
||||
"uid": "movies",
|
||||
"primaryKey": "id",
|
||||
"createdAt": "[now]",
|
||||
"updatedAt": "[now]"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(movies.settings().unwrap());
|
||||
let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 110);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022");
|
||||
|
||||
// movies2
|
||||
insta::assert_json_snapshot!(movies2.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
|
||||
{
|
||||
"uid": "movies_2",
|
||||
"primaryKey": null,
|
||||
"createdAt": "[now]",
|
||||
"updatedAt": "[now]"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(movies2.settings().unwrap());
|
||||
let documents = movies2.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 0);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
|
||||
|
||||
// spells
|
||||
insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
|
||||
{
|
||||
"uid": "dnd_spells",
|
||||
"primaryKey": "index",
|
||||
"createdAt": "[now]",
|
||||
"updatedAt": "[now]"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(spells.settings().unwrap());
|
||||
let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 10);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
|
||||
}
|
||||
}
|
||||
467
crates/dump/src/reader/compat/v4_to_v5.rs
Normal file
467
crates/dump/src/reader/compat/v4_to_v5.rs
Normal file
@ -0,0 +1,467 @@
|
||||
use super::v3_to_v4::{CompatIndexV3ToV4, CompatV3ToV4};
|
||||
use super::v5_to_v6::CompatV5ToV6;
|
||||
use crate::reader::{v4, v5, Document};
|
||||
use crate::Result;
|
||||
|
||||
pub enum CompatV4ToV5 {
|
||||
V4(v4::V4Reader),
|
||||
Compat(CompatV3ToV4),
|
||||
}
|
||||
|
||||
impl CompatV4ToV5 {
|
||||
pub fn new(v4: v4::V4Reader) -> CompatV4ToV5 {
|
||||
CompatV4ToV5::V4(v4)
|
||||
}
|
||||
|
||||
pub fn to_v6(self) -> CompatV5ToV6 {
|
||||
CompatV5ToV6::Compat(self)
|
||||
}
|
||||
|
||||
pub fn version(&self) -> crate::Version {
|
||||
match self {
|
||||
CompatV4ToV5::V4(v4) => v4.version(),
|
||||
CompatV4ToV5::Compat(compat) => compat.version(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn date(&self) -> Option<time::OffsetDateTime> {
|
||||
match self {
|
||||
CompatV4ToV5::V4(v4) => v4.date(),
|
||||
CompatV4ToV5::Compat(compat) => compat.date(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn instance_uid(&self) -> Result<Option<uuid::Uuid>> {
|
||||
match self {
|
||||
CompatV4ToV5::V4(v4) => v4.instance_uid(),
|
||||
CompatV4ToV5::Compat(compat) => compat.instance_uid(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn indexes(&self) -> Result<Box<dyn Iterator<Item = Result<CompatIndexV4ToV5>> + '_>> {
|
||||
Ok(match self {
|
||||
CompatV4ToV5::V4(v4) => {
|
||||
Box::new(v4.indexes()?.map(|index| index.map(CompatIndexV4ToV5::from)))
|
||||
as Box<dyn Iterator<Item = Result<CompatIndexV4ToV5>> + '_>
|
||||
}
|
||||
|
||||
CompatV4ToV5::Compat(compat) => {
|
||||
Box::new(compat.indexes()?.map(|index| index.map(CompatIndexV4ToV5::from)))
|
||||
as Box<dyn Iterator<Item = Result<CompatIndexV4ToV5>> + '_>
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub fn tasks(
|
||||
&mut self,
|
||||
) -> Box<dyn Iterator<Item = Result<(v5::Task, Option<Box<crate::reader::UpdateFile>>)>> + '_>
|
||||
{
|
||||
let tasks = match self {
|
||||
CompatV4ToV5::V4(v4) => v4.tasks(),
|
||||
CompatV4ToV5::Compat(compat) => compat.tasks(),
|
||||
};
|
||||
Box::new(tasks.map(|task| {
|
||||
task.map(|(task, content_file)| {
|
||||
let task = v5::Task {
|
||||
id: task.id,
|
||||
content: match task.content {
|
||||
v4::tasks::TaskContent::DocumentAddition {
|
||||
content_uuid,
|
||||
merge_strategy,
|
||||
primary_key,
|
||||
documents_count,
|
||||
allow_index_creation,
|
||||
} => v5::tasks::TaskContent::DocumentAddition {
|
||||
index_uid: v5::meta::IndexUid(task.index_uid.0),
|
||||
content_uuid,
|
||||
merge_strategy: match merge_strategy {
|
||||
v4::tasks::IndexDocumentsMethod::ReplaceDocuments => {
|
||||
v5::tasks::IndexDocumentsMethod::ReplaceDocuments
|
||||
}
|
||||
v4::tasks::IndexDocumentsMethod::UpdateDocuments => {
|
||||
v5::tasks::IndexDocumentsMethod::UpdateDocuments
|
||||
}
|
||||
},
|
||||
primary_key,
|
||||
documents_count,
|
||||
allow_index_creation,
|
||||
},
|
||||
v4::tasks::TaskContent::DocumentDeletion(deletion) => {
|
||||
v5::tasks::TaskContent::DocumentDeletion {
|
||||
index_uid: v5::meta::IndexUid(task.index_uid.0),
|
||||
deletion: match deletion {
|
||||
v4::tasks::DocumentDeletion::Clear => {
|
||||
v5::tasks::DocumentDeletion::Clear
|
||||
}
|
||||
v4::tasks::DocumentDeletion::Ids(ids) => {
|
||||
v5::tasks::DocumentDeletion::Ids(ids)
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
v4::tasks::TaskContent::SettingsUpdate {
|
||||
settings,
|
||||
is_deletion,
|
||||
allow_index_creation,
|
||||
} => v5::tasks::TaskContent::SettingsUpdate {
|
||||
index_uid: v5::meta::IndexUid(task.index_uid.0),
|
||||
settings: settings.into(),
|
||||
is_deletion,
|
||||
allow_index_creation,
|
||||
},
|
||||
v4::tasks::TaskContent::IndexDeletion => {
|
||||
v5::tasks::TaskContent::IndexDeletion {
|
||||
index_uid: v5::meta::IndexUid(task.index_uid.0),
|
||||
}
|
||||
}
|
||||
v4::tasks::TaskContent::IndexCreation { primary_key } => {
|
||||
v5::tasks::TaskContent::IndexCreation {
|
||||
index_uid: v5::meta::IndexUid(task.index_uid.0),
|
||||
primary_key,
|
||||
}
|
||||
}
|
||||
v4::tasks::TaskContent::IndexUpdate { primary_key } => {
|
||||
v5::tasks::TaskContent::IndexUpdate {
|
||||
index_uid: v5::meta::IndexUid(task.index_uid.0),
|
||||
primary_key,
|
||||
}
|
||||
}
|
||||
},
|
||||
events: task
|
||||
.events
|
||||
.into_iter()
|
||||
.map(|event| match event {
|
||||
v4::tasks::TaskEvent::Created(date) => {
|
||||
v5::tasks::TaskEvent::Created(date)
|
||||
}
|
||||
v4::tasks::TaskEvent::Batched { timestamp, batch_id } => {
|
||||
v5::tasks::TaskEvent::Batched { timestamp, batch_id }
|
||||
}
|
||||
v4::tasks::TaskEvent::Processing(date) => {
|
||||
v5::tasks::TaskEvent::Processing(date)
|
||||
}
|
||||
v4::tasks::TaskEvent::Succeded { result, timestamp } => {
|
||||
v5::tasks::TaskEvent::Succeeded {
|
||||
result: match result {
|
||||
v4::tasks::TaskResult::DocumentAddition {
|
||||
indexed_documents,
|
||||
} => v5::tasks::TaskResult::DocumentAddition {
|
||||
indexed_documents,
|
||||
},
|
||||
v4::tasks::TaskResult::DocumentDeletion {
|
||||
deleted_documents,
|
||||
} => v5::tasks::TaskResult::DocumentDeletion {
|
||||
deleted_documents,
|
||||
},
|
||||
v4::tasks::TaskResult::ClearAll { deleted_documents } => {
|
||||
v5::tasks::TaskResult::ClearAll { deleted_documents }
|
||||
}
|
||||
v4::tasks::TaskResult::Other => {
|
||||
v5::tasks::TaskResult::Other
|
||||
}
|
||||
},
|
||||
timestamp,
|
||||
}
|
||||
}
|
||||
v4::tasks::TaskEvent::Failed { error, timestamp } => {
|
||||
v5::tasks::TaskEvent::Failed {
|
||||
error: v5::ResponseError::from(error),
|
||||
timestamp,
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
|
||||
(task, content_file)
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<v5::Key>> + '_> {
|
||||
let keys = match self {
|
||||
CompatV4ToV5::V4(v4) => v4.keys(),
|
||||
CompatV4ToV5::Compat(compat) => compat.keys(),
|
||||
};
|
||||
Box::new(keys.map(|key| {
|
||||
key.map(|key| v5::Key {
|
||||
description: key.description,
|
||||
name: None,
|
||||
uid: v5::keys::KeyId::new_v4(),
|
||||
actions: key.actions.into_iter().filter_map(|action| action.into()).collect(),
|
||||
indexes: key
|
||||
.indexes
|
||||
.into_iter()
|
||||
.map(|index| match index.as_str() {
|
||||
"*" => v5::StarOr::Star,
|
||||
_ => v5::StarOr::Other(v5::meta::IndexUid(index)),
|
||||
})
|
||||
.collect(),
|
||||
expires_at: key.expires_at,
|
||||
created_at: key.created_at,
|
||||
updated_at: key.updated_at,
|
||||
})
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
pub enum CompatIndexV4ToV5 {
|
||||
V4(v4::V4IndexReader),
|
||||
Compat(CompatIndexV3ToV4),
|
||||
}
|
||||
|
||||
impl From<v4::V4IndexReader> for CompatIndexV4ToV5 {
|
||||
fn from(index_reader: v4::V4IndexReader) -> Self {
|
||||
Self::V4(index_reader)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<CompatIndexV3ToV4> for CompatIndexV4ToV5 {
|
||||
fn from(index_reader: CompatIndexV3ToV4) -> Self {
|
||||
Self::Compat(index_reader)
|
||||
}
|
||||
}
|
||||
|
||||
impl CompatIndexV4ToV5 {
|
||||
pub fn metadata(&self) -> &crate::IndexMetadata {
|
||||
match self {
|
||||
CompatIndexV4ToV5::V4(v4) => v4.metadata(),
|
||||
CompatIndexV4ToV5::Compat(compat) => compat.metadata(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<Document>> + '_>> {
|
||||
match self {
|
||||
CompatIndexV4ToV5::V4(v4) => v4
|
||||
.documents()
|
||||
.map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<Document>> + '_>),
|
||||
CompatIndexV4ToV5::Compat(compat) => compat
|
||||
.documents()
|
||||
.map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<Document>> + '_>),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn settings(&mut self) -> Result<v5::Settings<v5::Checked>> {
|
||||
match self {
|
||||
CompatIndexV4ToV5::V4(v4) => Ok(v5::Settings::from(v4.settings()?).check()),
|
||||
CompatIndexV4ToV5::Compat(compat) => Ok(v5::Settings::from(compat.settings()?).check()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> From<v4::Setting<T>> for v5::Setting<T> {
|
||||
fn from(setting: v4::Setting<T>) -> Self {
|
||||
match setting {
|
||||
v4::Setting::Set(t) => v5::Setting::Set(t),
|
||||
v4::Setting::Reset => v5::Setting::Reset,
|
||||
v4::Setting::NotSet => v5::Setting::NotSet,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<v4::ResponseError> for v5::ResponseError {
|
||||
fn from(error: v4::ResponseError) -> Self {
|
||||
let code = match error.error_code.as_ref() {
|
||||
"index_creation_failed" => v5::Code::CreateIndex,
|
||||
"index_already_exists" => v5::Code::IndexAlreadyExists,
|
||||
"index_not_found" => v5::Code::IndexNotFound,
|
||||
"invalid_index_uid" => v5::Code::InvalidIndexUid,
|
||||
"invalid_min_word_length_for_typo" => v5::Code::InvalidMinWordLengthForTypo,
|
||||
"invalid_state" => v5::Code::InvalidState,
|
||||
"primary_key_inference_failed" => v5::Code::MissingPrimaryKey,
|
||||
"index_primary_key_already_exists" => v5::Code::PrimaryKeyAlreadyPresent,
|
||||
"max_fields_limit_exceeded" => v5::Code::MaxFieldsLimitExceeded,
|
||||
"missing_document_id" => v5::Code::MissingDocumentId,
|
||||
"invalid_document_id" => v5::Code::InvalidDocumentId,
|
||||
"invalid_filter" => v5::Code::Filter,
|
||||
"invalid_sort" => v5::Code::Sort,
|
||||
"bad_parameter" => v5::Code::BadParameter,
|
||||
"bad_request" => v5::Code::BadRequest,
|
||||
"database_size_limit_reached" => v5::Code::DatabaseSizeLimitReached,
|
||||
"document_not_found" => v5::Code::DocumentNotFound,
|
||||
"internal" => v5::Code::Internal,
|
||||
"invalid_geo_field" => v5::Code::InvalidGeoField,
|
||||
"invalid_ranking_rule" => v5::Code::InvalidRankingRule,
|
||||
"invalid_store_file" => v5::Code::InvalidStore,
|
||||
"invalid_api_key" => v5::Code::InvalidToken,
|
||||
"missing_authorization_header" => v5::Code::MissingAuthorizationHeader,
|
||||
"no_space_left_on_device" => v5::Code::NoSpaceLeftOnDevice,
|
||||
"dump_not_found" => v5::Code::DumpNotFound,
|
||||
"task_not_found" => v5::Code::TaskNotFound,
|
||||
"payload_too_large" => v5::Code::PayloadTooLarge,
|
||||
"unretrievable_document" => v5::Code::RetrieveDocument,
|
||||
"search_error" => v5::Code::SearchDocuments,
|
||||
"unsupported_media_type" => v5::Code::UnsupportedMediaType,
|
||||
"dump_already_processing" => v5::Code::DumpAlreadyInProgress,
|
||||
"dump_process_failed" => v5::Code::DumpProcessFailed,
|
||||
"invalid_content_type" => v5::Code::InvalidContentType,
|
||||
"missing_content_type" => v5::Code::MissingContentType,
|
||||
"malformed_payload" => v5::Code::MalformedPayload,
|
||||
"missing_payload" => v5::Code::MissingPayload,
|
||||
"api_key_not_found" => v5::Code::ApiKeyNotFound,
|
||||
"missing_parameter" => v5::Code::MissingParameter,
|
||||
"invalid_api_key_actions" => v5::Code::InvalidApiKeyActions,
|
||||
"invalid_api_key_indexes" => v5::Code::InvalidApiKeyIndexes,
|
||||
"invalid_api_key_expires_at" => v5::Code::InvalidApiKeyExpiresAt,
|
||||
"invalid_api_key_description" => v5::Code::InvalidApiKeyDescription,
|
||||
other => {
|
||||
tracing::warn!("Unknown error code {}", other);
|
||||
v5::Code::UnretrievableErrorCode
|
||||
}
|
||||
};
|
||||
v5::ResponseError::from_msg(error.message, code)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> From<v4::Settings<T>> for v5::Settings<v5::Unchecked> {
|
||||
fn from(settings: v4::Settings<T>) -> Self {
|
||||
v5::Settings {
|
||||
displayed_attributes: settings.displayed_attributes.into(),
|
||||
searchable_attributes: settings.searchable_attributes.into(),
|
||||
filterable_attributes: settings.filterable_attributes.into(),
|
||||
sortable_attributes: settings.sortable_attributes.into(),
|
||||
ranking_rules: settings.ranking_rules.into(),
|
||||
stop_words: settings.stop_words.into(),
|
||||
synonyms: settings.synonyms.into(),
|
||||
distinct_attribute: settings.distinct_attribute.into(),
|
||||
typo_tolerance: match settings.typo_tolerance {
|
||||
v4::Setting::Set(typo) => v5::Setting::Set(v5::TypoTolerance {
|
||||
enabled: typo.enabled.into(),
|
||||
min_word_size_for_typos: match typo.min_word_size_for_typos {
|
||||
v4::Setting::Set(t) => v5::Setting::Set(v5::MinWordSizeForTypos {
|
||||
one_typo: t.one_typo.into(),
|
||||
two_typos: t.two_typos.into(),
|
||||
}),
|
||||
v4::Setting::Reset => v5::Setting::Reset,
|
||||
v4::Setting::NotSet => v5::Setting::NotSet,
|
||||
},
|
||||
disable_on_words: typo.disable_on_words.into(),
|
||||
disable_on_attributes: typo.disable_on_attributes.into(),
|
||||
}),
|
||||
v4::Setting::Reset => v5::Setting::Reset,
|
||||
v4::Setting::NotSet => v5::Setting::NotSet,
|
||||
},
|
||||
faceting: v5::Setting::NotSet,
|
||||
pagination: v5::Setting::NotSet,
|
||||
_kind: std::marker::PhantomData,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<v4::Action> for Option<v5::Action> {
|
||||
fn from(key: v4::Action) -> Self {
|
||||
match key {
|
||||
v4::Action::All => Some(v5::Action::All),
|
||||
v4::Action::Search => Some(v5::Action::Search),
|
||||
v4::Action::DocumentsAdd => Some(v5::Action::DocumentsAdd),
|
||||
v4::Action::DocumentsGet => Some(v5::Action::DocumentsGet),
|
||||
v4::Action::DocumentsDelete => Some(v5::Action::DocumentsDelete),
|
||||
v4::Action::IndexesAdd => Some(v5::Action::IndexesAdd),
|
||||
v4::Action::IndexesGet => Some(v5::Action::IndexesGet),
|
||||
v4::Action::IndexesUpdate => Some(v5::Action::IndexesUpdate),
|
||||
v4::Action::IndexesDelete => Some(v5::Action::IndexesDelete),
|
||||
v4::Action::TasksGet => Some(v5::Action::TasksGet),
|
||||
v4::Action::SettingsGet => Some(v5::Action::SettingsGet),
|
||||
v4::Action::SettingsUpdate => Some(v5::Action::SettingsUpdate),
|
||||
v4::Action::StatsGet => Some(v5::Action::StatsGet),
|
||||
v4::Action::DumpsCreate => Some(v5::Action::DumpsCreate),
|
||||
v4::Action::DumpsGet => None,
|
||||
v4::Action::Version => Some(v5::Action::Version),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test {
|
||||
use std::fs::File;
|
||||
use std::io::BufReader;
|
||||
|
||||
use flate2::bufread::GzDecoder;
|
||||
use meili_snap::insta;
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn compat_v4_v5() {
|
||||
let dump = File::open("tests/assets/v4.dump").unwrap();
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut dump = BufReader::new(dump);
|
||||
let gz = GzDecoder::new(&mut dump);
|
||||
let mut archive = tar::Archive::new(gz);
|
||||
archive.unpack(dir.path()).unwrap();
|
||||
|
||||
let mut dump = v4::V4Reader::open(dir).unwrap().to_v5();
|
||||
|
||||
// top level infos
|
||||
insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
|
||||
insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
|
||||
|
||||
// tasks
|
||||
let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();
|
||||
let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
|
||||
meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"ed9a30cded4c046ef46f7cff7450347e");
|
||||
assert_eq!(update_files.len(), 10);
|
||||
assert!(update_files[0].is_some()); // the enqueued document addition
|
||||
assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
|
||||
|
||||
// keys
|
||||
let keys = dump.keys().collect::<Result<Vec<_>>>().unwrap();
|
||||
meili_snap::snapshot_hash!(meili_snap::json_string!(keys, { "[].uid" => "[uuid]" }), @"1384361d734fd77c23804c9696228660");
|
||||
|
||||
// indexes
|
||||
let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
// the index are not ordered in any way by default
|
||||
indexes.sort_by_key(|index| index.metadata().uid.to_string());
|
||||
|
||||
let mut products = indexes.pop().unwrap();
|
||||
let mut movies = indexes.pop().unwrap();
|
||||
let mut spells = indexes.pop().unwrap();
|
||||
assert!(indexes.is_empty());
|
||||
|
||||
// products
|
||||
insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
|
||||
{
|
||||
"uid": "products",
|
||||
"primaryKey": "sku",
|
||||
"createdAt": "[now]",
|
||||
"updatedAt": "[now]"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(products.settings().unwrap());
|
||||
let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 10);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
|
||||
|
||||
// movies
|
||||
insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
|
||||
{
|
||||
"uid": "movies",
|
||||
"primaryKey": "id",
|
||||
"createdAt": "[now]",
|
||||
"updatedAt": "[now]"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(movies.settings().unwrap());
|
||||
let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 110);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"786022a66ecb992c8a2a60fee070a5ab");
|
||||
|
||||
// spells
|
||||
insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
|
||||
{
|
||||
"uid": "dnd_spells",
|
||||
"primaryKey": "index",
|
||||
"createdAt": "[now]",
|
||||
"updatedAt": "[now]"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(spells.settings().unwrap());
|
||||
let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 10);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
|
||||
}
|
||||
}
|
||||
517
crates/dump/src/reader/compat/v5_to_v6.rs
Normal file
517
crates/dump/src/reader/compat/v5_to_v6.rs
Normal file
@ -0,0 +1,517 @@
|
||||
use std::str::FromStr;
|
||||
|
||||
use super::v4_to_v5::{CompatIndexV4ToV5, CompatV4ToV5};
|
||||
use crate::reader::{v5, v6, Document, UpdateFile};
|
||||
use crate::Result;
|
||||
|
||||
pub enum CompatV5ToV6 {
|
||||
V5(v5::V5Reader),
|
||||
Compat(CompatV4ToV5),
|
||||
}
|
||||
|
||||
impl CompatV5ToV6 {
|
||||
pub fn new_v5(v5: v5::V5Reader) -> CompatV5ToV6 {
|
||||
CompatV5ToV6::V5(v5)
|
||||
}
|
||||
|
||||
pub fn version(&self) -> crate::Version {
|
||||
match self {
|
||||
CompatV5ToV6::V5(v5) => v5.version(),
|
||||
CompatV5ToV6::Compat(compat) => compat.version(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn date(&self) -> Option<time::OffsetDateTime> {
|
||||
match self {
|
||||
CompatV5ToV6::V5(v5) => v5.date(),
|
||||
CompatV5ToV6::Compat(compat) => compat.date(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn instance_uid(&self) -> Result<Option<uuid::Uuid>> {
|
||||
match self {
|
||||
CompatV5ToV6::V5(v5) => v5.instance_uid(),
|
||||
CompatV5ToV6::Compat(compat) => compat.instance_uid(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn indexes(&self) -> Result<Box<dyn Iterator<Item = Result<CompatIndexV5ToV6>> + '_>> {
|
||||
let indexes = match self {
|
||||
CompatV5ToV6::V5(v5) => {
|
||||
Box::new(v5.indexes()?.map(|index| index.map(CompatIndexV5ToV6::from)))
|
||||
as Box<dyn Iterator<Item = Result<CompatIndexV5ToV6>> + '_>
|
||||
}
|
||||
|
||||
CompatV5ToV6::Compat(compat) => {
|
||||
Box::new(compat.indexes()?.map(|index| index.map(CompatIndexV5ToV6::from)))
|
||||
as Box<dyn Iterator<Item = Result<CompatIndexV5ToV6>> + '_>
|
||||
}
|
||||
};
|
||||
Ok(indexes)
|
||||
}
|
||||
|
||||
pub fn tasks(
|
||||
&mut self,
|
||||
) -> Result<Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_>> {
|
||||
let instance_uid = self.instance_uid().ok().flatten();
|
||||
let keys = self.keys()?.collect::<Result<Vec<_>>>()?;
|
||||
|
||||
let tasks = match self {
|
||||
CompatV5ToV6::V5(v5) => v5.tasks(),
|
||||
CompatV5ToV6::Compat(compat) => compat.tasks(),
|
||||
};
|
||||
Ok(Box::new(tasks.map(move |task| {
|
||||
task.map(|(task, content_file)| {
|
||||
let mut task_view: v5::tasks::TaskView = task.clone().into();
|
||||
|
||||
if task_view.status == v5::Status::Processing {
|
||||
task_view.started_at = None;
|
||||
}
|
||||
|
||||
let task = v6::Task {
|
||||
uid: task_view.uid,
|
||||
index_uid: task_view.index_uid,
|
||||
status: match task_view.status {
|
||||
v5::Status::Enqueued => v6::Status::Enqueued,
|
||||
v5::Status::Processing => v6::Status::Enqueued,
|
||||
v5::Status::Succeeded => v6::Status::Succeeded,
|
||||
v5::Status::Failed => v6::Status::Failed,
|
||||
},
|
||||
kind: match task.content {
|
||||
v5::tasks::TaskContent::IndexCreation { primary_key, .. } => {
|
||||
v6::Kind::IndexCreation { primary_key }
|
||||
}
|
||||
v5::tasks::TaskContent::IndexUpdate { primary_key, .. } => {
|
||||
v6::Kind::IndexUpdate { primary_key }
|
||||
}
|
||||
v5::tasks::TaskContent::IndexDeletion { .. } => v6::Kind::IndexDeletion,
|
||||
v5::tasks::TaskContent::DocumentAddition {
|
||||
merge_strategy,
|
||||
allow_index_creation,
|
||||
primary_key,
|
||||
documents_count,
|
||||
..
|
||||
} => v6::Kind::DocumentImport {
|
||||
primary_key,
|
||||
documents_count: documents_count as u64,
|
||||
method: match merge_strategy {
|
||||
v5::tasks::IndexDocumentsMethod::ReplaceDocuments => {
|
||||
v6::milli::update::IndexDocumentsMethod::ReplaceDocuments
|
||||
}
|
||||
v5::tasks::IndexDocumentsMethod::UpdateDocuments => {
|
||||
v6::milli::update::IndexDocumentsMethod::UpdateDocuments
|
||||
}
|
||||
},
|
||||
allow_index_creation,
|
||||
},
|
||||
v5::tasks::TaskContent::DocumentDeletion { deletion, .. } => match deletion
|
||||
{
|
||||
v5::tasks::DocumentDeletion::Clear => v6::Kind::DocumentClear,
|
||||
v5::tasks::DocumentDeletion::Ids(documents_ids) => {
|
||||
v6::Kind::DocumentDeletion { documents_ids }
|
||||
}
|
||||
},
|
||||
v5::tasks::TaskContent::SettingsUpdate {
|
||||
allow_index_creation,
|
||||
is_deletion,
|
||||
settings,
|
||||
..
|
||||
} => v6::Kind::Settings {
|
||||
is_deletion,
|
||||
allow_index_creation,
|
||||
settings: Box::new(settings.into()),
|
||||
},
|
||||
v5::tasks::TaskContent::Dump { uid: _ } => {
|
||||
// in v6 we compute the dump_uid from the started_at processing time
|
||||
v6::Kind::DumpCreation { keys: keys.clone(), instance_uid }
|
||||
}
|
||||
},
|
||||
canceled_by: None,
|
||||
details: task_view.details.map(|details| match details {
|
||||
v5::Details::DocumentAddition { received_documents, indexed_documents } => {
|
||||
v6::Details::DocumentAdditionOrUpdate {
|
||||
received_documents: received_documents as u64,
|
||||
indexed_documents,
|
||||
}
|
||||
}
|
||||
v5::Details::Settings { settings } => {
|
||||
v6::Details::SettingsUpdate { settings: Box::new(settings.into()) }
|
||||
}
|
||||
v5::Details::IndexInfo { primary_key } => {
|
||||
v6::Details::IndexInfo { primary_key }
|
||||
}
|
||||
v5::Details::DocumentDeletion {
|
||||
received_document_ids,
|
||||
deleted_documents,
|
||||
} => v6::Details::DocumentDeletion {
|
||||
provided_ids: received_document_ids,
|
||||
deleted_documents,
|
||||
},
|
||||
v5::Details::ClearAll { deleted_documents } => {
|
||||
v6::Details::ClearAll { deleted_documents }
|
||||
}
|
||||
v5::Details::Dump { dump_uid } => {
|
||||
v6::Details::Dump { dump_uid: Some(dump_uid) }
|
||||
}
|
||||
}),
|
||||
error: task_view.error.map(|e| e.into()),
|
||||
enqueued_at: task_view.enqueued_at,
|
||||
started_at: task_view.started_at,
|
||||
finished_at: task_view.finished_at,
|
||||
};
|
||||
|
||||
(task, content_file)
|
||||
})
|
||||
})))
|
||||
}
|
||||
|
||||
pub fn keys(&mut self) -> Result<Box<dyn Iterator<Item = Result<v6::Key>> + '_>> {
|
||||
let keys = match self {
|
||||
CompatV5ToV6::V5(v5) => v5.keys()?,
|
||||
CompatV5ToV6::Compat(compat) => compat.keys(),
|
||||
};
|
||||
|
||||
Ok(Box::new(keys.map(|key| {
|
||||
key.map(|key| v6::Key {
|
||||
description: key.description,
|
||||
name: key.name,
|
||||
uid: key.uid,
|
||||
actions: key.actions.into_iter().map(|action| action.into()).collect(),
|
||||
indexes: key
|
||||
.indexes
|
||||
.into_iter()
|
||||
.map(|index| match index {
|
||||
v5::StarOr::Star => v6::IndexUidPattern::all(),
|
||||
v5::StarOr::Other(uid) => v6::IndexUidPattern::new_unchecked(uid.as_str()),
|
||||
})
|
||||
.collect(),
|
||||
expires_at: key.expires_at,
|
||||
created_at: key.created_at,
|
||||
updated_at: key.updated_at,
|
||||
})
|
||||
})))
|
||||
}
|
||||
|
||||
pub fn features(&self) -> Result<Option<v6::RuntimeTogglableFeatures>> {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
pub enum CompatIndexV5ToV6 {
|
||||
V5(v5::V5IndexReader),
|
||||
Compat(CompatIndexV4ToV5),
|
||||
}
|
||||
|
||||
impl From<v5::V5IndexReader> for CompatIndexV5ToV6 {
|
||||
fn from(index_reader: v5::V5IndexReader) -> Self {
|
||||
Self::V5(index_reader)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<CompatIndexV4ToV5> for CompatIndexV5ToV6 {
|
||||
fn from(index_reader: CompatIndexV4ToV5) -> Self {
|
||||
Self::Compat(index_reader)
|
||||
}
|
||||
}
|
||||
|
||||
impl CompatIndexV5ToV6 {
|
||||
pub fn new_v5(v5: v5::V5IndexReader) -> CompatIndexV5ToV6 {
|
||||
CompatIndexV5ToV6::V5(v5)
|
||||
}
|
||||
|
||||
pub fn metadata(&self) -> &crate::IndexMetadata {
|
||||
match self {
|
||||
CompatIndexV5ToV6::V5(v5) => v5.metadata(),
|
||||
CompatIndexV5ToV6::Compat(compat) => compat.metadata(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<Document>> + '_>> {
|
||||
match self {
|
||||
CompatIndexV5ToV6::V5(v5) => v5
|
||||
.documents()
|
||||
.map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<Document>> + '_>),
|
||||
CompatIndexV5ToV6::Compat(compat) => compat
|
||||
.documents()
|
||||
.map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<Document>> + '_>),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn settings(&mut self) -> Result<v6::Settings<v6::Checked>> {
|
||||
match self {
|
||||
CompatIndexV5ToV6::V5(v5) => Ok(v6::Settings::from(v5.settings()?).check()),
|
||||
CompatIndexV5ToV6::Compat(compat) => Ok(v6::Settings::from(compat.settings()?).check()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> From<v5::Setting<T>> for v6::Setting<T> {
|
||||
fn from(setting: v5::Setting<T>) -> Self {
|
||||
match setting {
|
||||
v5::Setting::Set(t) => v6::Setting::Set(t),
|
||||
v5::Setting::Reset => v6::Setting::Reset,
|
||||
v5::Setting::NotSet => v6::Setting::NotSet,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<v5::ResponseError> for v6::ResponseError {
|
||||
fn from(error: v5::ResponseError) -> Self {
|
||||
let code = match error.error_code.as_ref() {
|
||||
"index_creation_failed" => v6::Code::IndexCreationFailed,
|
||||
"index_already_exists" => v6::Code::IndexAlreadyExists,
|
||||
"index_not_found" => v6::Code::IndexNotFound,
|
||||
"invalid_index_uid" => v6::Code::InvalidIndexUid,
|
||||
"invalid_min_word_length_for_typo" => v6::Code::InvalidSettingsTypoTolerance,
|
||||
"invalid_state" => v6::Code::InvalidState,
|
||||
"primary_key_inference_failed" => v6::Code::IndexPrimaryKeyNoCandidateFound,
|
||||
"index_primary_key_already_exists" => v6::Code::IndexPrimaryKeyAlreadyExists,
|
||||
"max_fields_limit_exceeded" => v6::Code::MaxFieldsLimitExceeded,
|
||||
"missing_document_id" => v6::Code::MissingDocumentId,
|
||||
"invalid_document_id" => v6::Code::InvalidDocumentId,
|
||||
"invalid_filter" => v6::Code::InvalidSettingsFilterableAttributes,
|
||||
"invalid_sort" => v6::Code::InvalidSettingsSortableAttributes,
|
||||
"bad_parameter" => v6::Code::BadParameter,
|
||||
"bad_request" => v6::Code::BadRequest,
|
||||
"database_size_limit_reached" => v6::Code::DatabaseSizeLimitReached,
|
||||
"document_not_found" => v6::Code::DocumentNotFound,
|
||||
"internal" => v6::Code::Internal,
|
||||
"invalid_geo_field" => v6::Code::InvalidDocumentGeoField,
|
||||
"invalid_ranking_rule" => v6::Code::InvalidSettingsRankingRules,
|
||||
"invalid_store_file" => v6::Code::InvalidStoreFile,
|
||||
"invalid_api_key" => v6::Code::InvalidApiKey,
|
||||
"missing_authorization_header" => v6::Code::MissingAuthorizationHeader,
|
||||
"no_space_left_on_device" => v6::Code::NoSpaceLeftOnDevice,
|
||||
"dump_not_found" => v6::Code::DumpNotFound,
|
||||
"task_not_found" => v6::Code::TaskNotFound,
|
||||
"payload_too_large" => v6::Code::PayloadTooLarge,
|
||||
"unretrievable_document" => v6::Code::UnretrievableDocument,
|
||||
"unsupported_media_type" => v6::Code::UnsupportedMediaType,
|
||||
"dump_already_processing" => v6::Code::DumpAlreadyProcessing,
|
||||
"dump_process_failed" => v6::Code::DumpProcessFailed,
|
||||
"invalid_content_type" => v6::Code::InvalidContentType,
|
||||
"missing_content_type" => v6::Code::MissingContentType,
|
||||
"malformed_payload" => v6::Code::MalformedPayload,
|
||||
"missing_payload" => v6::Code::MissingPayload,
|
||||
"api_key_not_found" => v6::Code::ApiKeyNotFound,
|
||||
"missing_parameter" => v6::Code::BadRequest,
|
||||
"invalid_api_key_actions" => v6::Code::InvalidApiKeyActions,
|
||||
"invalid_api_key_indexes" => v6::Code::InvalidApiKeyIndexes,
|
||||
"invalid_api_key_expires_at" => v6::Code::InvalidApiKeyExpiresAt,
|
||||
"invalid_api_key_description" => v6::Code::InvalidApiKeyDescription,
|
||||
"invalid_api_key_name" => v6::Code::InvalidApiKeyName,
|
||||
"invalid_api_key_uid" => v6::Code::InvalidApiKeyUid,
|
||||
"immutable_field" => v6::Code::BadRequest,
|
||||
"api_key_already_exists" => v6::Code::ApiKeyAlreadyExists,
|
||||
other => {
|
||||
tracing::warn!("Unknown error code {}", other);
|
||||
v6::Code::UnretrievableErrorCode
|
||||
}
|
||||
};
|
||||
v6::ResponseError::from_msg(error.message, code)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
|
||||
fn from(settings: v5::Settings<T>) -> Self {
|
||||
v6::Settings {
|
||||
displayed_attributes: v6::Setting::from(settings.displayed_attributes).into(),
|
||||
searchable_attributes: v6::Setting::from(settings.searchable_attributes).into(),
|
||||
filterable_attributes: settings.filterable_attributes.into(),
|
||||
sortable_attributes: settings.sortable_attributes.into(),
|
||||
ranking_rules: {
|
||||
match settings.ranking_rules {
|
||||
v5::settings::Setting::Set(ranking_rules) => {
|
||||
let mut new_ranking_rules = vec![];
|
||||
for rule in ranking_rules {
|
||||
match v6::RankingRuleView::from_str(&rule) {
|
||||
Ok(new_rule) => {
|
||||
new_ranking_rules.push(new_rule);
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::warn!("Error while importing settings. The ranking rule `{rule}` does not exist anymore.")
|
||||
}
|
||||
}
|
||||
}
|
||||
v6::Setting::Set(new_ranking_rules)
|
||||
}
|
||||
v5::settings::Setting::Reset => v6::Setting::Reset,
|
||||
v5::settings::Setting::NotSet => v6::Setting::NotSet,
|
||||
}
|
||||
},
|
||||
stop_words: settings.stop_words.into(),
|
||||
non_separator_tokens: v6::Setting::NotSet,
|
||||
separator_tokens: v6::Setting::NotSet,
|
||||
dictionary: v6::Setting::NotSet,
|
||||
synonyms: settings.synonyms.into(),
|
||||
distinct_attribute: settings.distinct_attribute.into(),
|
||||
proximity_precision: v6::Setting::NotSet,
|
||||
typo_tolerance: match settings.typo_tolerance {
|
||||
v5::Setting::Set(typo) => v6::Setting::Set(v6::TypoTolerance {
|
||||
enabled: typo.enabled.into(),
|
||||
min_word_size_for_typos: match typo.min_word_size_for_typos {
|
||||
v5::Setting::Set(t) => v6::Setting::Set(v6::MinWordSizeForTypos {
|
||||
one_typo: t.one_typo.into(),
|
||||
two_typos: t.two_typos.into(),
|
||||
}),
|
||||
v5::Setting::Reset => v6::Setting::Reset,
|
||||
v5::Setting::NotSet => v6::Setting::NotSet,
|
||||
},
|
||||
disable_on_words: typo.disable_on_words.into(),
|
||||
disable_on_attributes: typo.disable_on_attributes.into(),
|
||||
}),
|
||||
v5::Setting::Reset => v6::Setting::Reset,
|
||||
v5::Setting::NotSet => v6::Setting::NotSet,
|
||||
},
|
||||
faceting: match settings.faceting {
|
||||
v5::Setting::Set(faceting) => v6::Setting::Set(v6::FacetingSettings {
|
||||
max_values_per_facet: faceting.max_values_per_facet.into(),
|
||||
sort_facet_values_by: v6::Setting::NotSet,
|
||||
}),
|
||||
v5::Setting::Reset => v6::Setting::Reset,
|
||||
v5::Setting::NotSet => v6::Setting::NotSet,
|
||||
},
|
||||
pagination: match settings.pagination {
|
||||
v5::Setting::Set(pagination) => v6::Setting::Set(v6::PaginationSettings {
|
||||
max_total_hits: pagination.max_total_hits.into(),
|
||||
}),
|
||||
v5::Setting::Reset => v6::Setting::Reset,
|
||||
v5::Setting::NotSet => v6::Setting::NotSet,
|
||||
},
|
||||
embedders: v6::Setting::NotSet,
|
||||
localized_attributes: v6::Setting::NotSet,
|
||||
search_cutoff_ms: v6::Setting::NotSet,
|
||||
_kind: std::marker::PhantomData,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<v5::Action> for v6::Action {
|
||||
fn from(key: v5::Action) -> Self {
|
||||
match key {
|
||||
v5::Action::All => v6::Action::All,
|
||||
v5::Action::Search => v6::Action::Search,
|
||||
v5::Action::DocumentsAll => v6::Action::DocumentsAll,
|
||||
v5::Action::DocumentsAdd => v6::Action::DocumentsAdd,
|
||||
v5::Action::DocumentsGet => v6::Action::DocumentsGet,
|
||||
v5::Action::DocumentsDelete => v6::Action::DocumentsDelete,
|
||||
v5::Action::IndexesAll => v6::Action::IndexesAll,
|
||||
v5::Action::IndexesAdd => v6::Action::IndexesAdd,
|
||||
v5::Action::IndexesGet => v6::Action::IndexesGet,
|
||||
v5::Action::IndexesUpdate => v6::Action::IndexesUpdate,
|
||||
v5::Action::IndexesDelete => v6::Action::IndexesDelete,
|
||||
v5::Action::TasksAll => v6::Action::TasksAll,
|
||||
v5::Action::TasksGet => v6::Action::TasksGet,
|
||||
v5::Action::SettingsAll => v6::Action::SettingsAll,
|
||||
v5::Action::SettingsGet => v6::Action::SettingsGet,
|
||||
v5::Action::SettingsUpdate => v6::Action::SettingsUpdate,
|
||||
v5::Action::StatsAll => v6::Action::StatsAll,
|
||||
v5::Action::StatsGet => v6::Action::StatsGet,
|
||||
v5::Action::MetricsAll => v6::Action::MetricsAll,
|
||||
v5::Action::MetricsGet => v6::Action::MetricsGet,
|
||||
v5::Action::DumpsAll => v6::Action::DumpsAll,
|
||||
v5::Action::DumpsCreate => v6::Action::DumpsCreate,
|
||||
v5::Action::Version => v6::Action::Version,
|
||||
v5::Action::KeysAdd => v6::Action::KeysAdd,
|
||||
v5::Action::KeysGet => v6::Action::KeysGet,
|
||||
v5::Action::KeysUpdate => v6::Action::KeysUpdate,
|
||||
v5::Action::KeysDelete => v6::Action::KeysDelete,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test {
|
||||
use std::fs::File;
|
||||
use std::io::BufReader;
|
||||
|
||||
use flate2::bufread::GzDecoder;
|
||||
use meili_snap::insta;
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn compat_v5_v6() {
|
||||
let dump = File::open("tests/assets/v5.dump").unwrap();
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut dump = BufReader::new(dump);
|
||||
let gz = GzDecoder::new(&mut dump);
|
||||
let mut archive = tar::Archive::new(gz);
|
||||
archive.unpack(dir.path()).unwrap();
|
||||
|
||||
let mut dump = v5::V5Reader::open(dir).unwrap().to_v6();
|
||||
|
||||
// top level infos
|
||||
insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
|
||||
insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
|
||||
|
||||
// tasks
|
||||
let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
|
||||
meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"41f91d3a94911b2735ec41b07540df5c");
|
||||
assert_eq!(update_files.len(), 22);
|
||||
assert!(update_files[0].is_none()); // the dump creation
|
||||
assert!(update_files[1].is_some()); // the enqueued document addition
|
||||
assert!(update_files[2..].iter().all(|u| u.is_none())); // everything already processed
|
||||
|
||||
// keys
|
||||
let keys = dump.keys().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"c9d2b467fe2fca0b35580d8a999808fb");
|
||||
|
||||
// indexes
|
||||
let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
// the index are not ordered in any way by default
|
||||
indexes.sort_by_key(|index| index.metadata().uid.to_string());
|
||||
|
||||
let mut products = indexes.pop().unwrap();
|
||||
let mut movies = indexes.pop().unwrap();
|
||||
let mut spells = indexes.pop().unwrap();
|
||||
assert!(indexes.is_empty());
|
||||
|
||||
// products
|
||||
insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
|
||||
{
|
||||
"uid": "products",
|
||||
"primaryKey": "sku",
|
||||
"createdAt": "[now]",
|
||||
"updatedAt": "[now]"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(products.settings().unwrap());
|
||||
let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 10);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
|
||||
|
||||
// movies
|
||||
insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
|
||||
{
|
||||
"uid": "movies",
|
||||
"primaryKey": "id",
|
||||
"createdAt": "[now]",
|
||||
"updatedAt": "[now]"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(movies.settings().unwrap());
|
||||
let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 200);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"e962baafd2fbae4cdd14e876053b0c5a");
|
||||
|
||||
// spells
|
||||
insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
|
||||
{
|
||||
"uid": "dnd_spells",
|
||||
"primaryKey": "index",
|
||||
"createdAt": "[now]",
|
||||
"updatedAt": "[now]"
|
||||
}
|
||||
"###);
|
||||
|
||||
insta::assert_json_snapshot!(spells.settings().unwrap());
|
||||
let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||
assert_eq!(documents.len(), 10);
|
||||
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user