mirror of
https://github.com/meilisearch/meilisearch.git
synced 2025-12-20 19:36:56 +00:00
Compare commits
13 Commits
yoeight/un
...
post-updat
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
22e6ca7ea2 | ||
|
|
6338905314 | ||
|
|
c6b0b22cd1 | ||
|
|
210e704e2a | ||
|
|
c98391eddd | ||
|
|
3769bc5b38 | ||
|
|
9170bc63af | ||
|
|
e168bcbb85 | ||
|
|
b24b1979a1 | ||
|
|
9db2b16eed | ||
|
|
2ba3fafcc3 | ||
|
|
14db3dbcc4 | ||
|
|
a61ef955fc |
2
.github/workflows/test-suite.yml
vendored
2
.github/workflows/test-suite.yml
vendored
@@ -15,7 +15,7 @@ env:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
test-linux:
|
test-linux:
|
||||||
name: Tests on Ubuntu
|
name: Tests on ${{ matrix.runner }} ${{ matrix.features }}
|
||||||
runs-on: ${{ matrix.runner }}
|
runs-on: ${{ matrix.runner }}
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
|
|||||||
4
Cargo.lock
generated
4
Cargo.lock
generated
@@ -2698,9 +2698,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hannoy"
|
name = "hannoy"
|
||||||
version = "0.1.0-nested-rtxns"
|
version = "0.1.2-nested-rtxns"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "be82bf3f2108ddc8885e3d306fcd7f4692066bfe26065ca8b42ba417f3c26dd1"
|
checksum = "533c952127a7e73448f26af313ac7b98012516561e48e953781cd6b30e573436"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytemuck",
|
"bytemuck",
|
||||||
"byteorder",
|
"byteorder",
|
||||||
|
|||||||
@@ -346,6 +346,7 @@ pub(crate) mod test {
|
|||||||
prefix_search: Setting::NotSet,
|
prefix_search: Setting::NotSet,
|
||||||
chat: Setting::NotSet,
|
chat: Setting::NotSet,
|
||||||
vector_store: Setting::NotSet,
|
vector_store: Setting::NotSet,
|
||||||
|
execute_after_update: Setting::NotSet,
|
||||||
_kind: std::marker::PhantomData,
|
_kind: std::marker::PhantomData,
|
||||||
};
|
};
|
||||||
settings.check()
|
settings.check()
|
||||||
|
|||||||
@@ -423,6 +423,7 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
|
|||||||
prefix_search: v6::Setting::NotSet,
|
prefix_search: v6::Setting::NotSet,
|
||||||
chat: v6::Setting::NotSet,
|
chat: v6::Setting::NotSet,
|
||||||
vector_store: v6::Setting::NotSet,
|
vector_store: v6::Setting::NotSet,
|
||||||
|
execute_after_update: v6::Setting::NotSet,
|
||||||
_kind: std::marker::PhantomData,
|
_kind: std::marker::PhantomData,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -306,6 +306,18 @@ fn create_or_open_index(
|
|||||||
) -> Result<Index> {
|
) -> Result<Index> {
|
||||||
let options = EnvOpenOptions::new();
|
let options = EnvOpenOptions::new();
|
||||||
let mut options = options.read_txn_without_tls();
|
let mut options = options.read_txn_without_tls();
|
||||||
|
|
||||||
|
let map_size = match std::env::var("MEILI_MAX_INDEX_SIZE") {
|
||||||
|
Ok(max_size) => {
|
||||||
|
let max_size = max_size.parse().unwrap();
|
||||||
|
map_size.min(max_size)
|
||||||
|
}
|
||||||
|
Err(VarError::NotPresent) => map_size,
|
||||||
|
Err(VarError::NotUnicode(e)) => {
|
||||||
|
panic!("Non unicode max index size in `MEILI_MAX_INDEX_SIZE`: {e:?}")
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
options.map_size(clamp_to_page_size(map_size));
|
options.map_size(clamp_to_page_size(map_size));
|
||||||
|
|
||||||
// You can find more details about this experimental
|
// You can find more details about this experimental
|
||||||
|
|||||||
@@ -328,6 +328,7 @@ InvalidSettingsDisplayedAttributes , InvalidRequest , BAD_REQU
|
|||||||
InvalidSettingsDistinctAttribute , InvalidRequest , BAD_REQUEST ;
|
InvalidSettingsDistinctAttribute , InvalidRequest , BAD_REQUEST ;
|
||||||
InvalidSettingsProximityPrecision , InvalidRequest , BAD_REQUEST ;
|
InvalidSettingsProximityPrecision , InvalidRequest , BAD_REQUEST ;
|
||||||
InvalidSettingsFacetSearch , InvalidRequest , BAD_REQUEST ;
|
InvalidSettingsFacetSearch , InvalidRequest , BAD_REQUEST ;
|
||||||
|
InvalidSettingsexecuteAfterUpdate , InvalidRequest , BAD_REQUEST ;
|
||||||
InvalidSettingsPrefixSearch , InvalidRequest , BAD_REQUEST ;
|
InvalidSettingsPrefixSearch , InvalidRequest , BAD_REQUEST ;
|
||||||
InvalidSettingsFaceting , InvalidRequest , BAD_REQUEST ;
|
InvalidSettingsFaceting , InvalidRequest , BAD_REQUEST ;
|
||||||
InvalidSettingsFilterableAttributes , InvalidRequest , BAD_REQUEST ;
|
InvalidSettingsFilterableAttributes , InvalidRequest , BAD_REQUEST ;
|
||||||
|
|||||||
@@ -326,6 +326,12 @@ pub struct Settings<T> {
|
|||||||
#[schema(value_type = Option<VectorStoreBackend>)]
|
#[schema(value_type = Option<VectorStoreBackend>)]
|
||||||
pub vector_store: Setting<VectorStoreBackend>,
|
pub vector_store: Setting<VectorStoreBackend>,
|
||||||
|
|
||||||
|
/// Function to execute after an update
|
||||||
|
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||||
|
#[deserr(default, error = DeserrJsonError<InvalidSettingsexecuteAfterUpdate>)]
|
||||||
|
#[schema(value_type = Option<String>, example = json!("doc.likes += 1"))]
|
||||||
|
pub execute_after_update: Setting<String>,
|
||||||
|
|
||||||
#[serde(skip)]
|
#[serde(skip)]
|
||||||
#[deserr(skip)]
|
#[deserr(skip)]
|
||||||
pub _kind: PhantomData<T>,
|
pub _kind: PhantomData<T>,
|
||||||
@@ -395,6 +401,7 @@ impl Settings<Checked> {
|
|||||||
prefix_search: Setting::Reset,
|
prefix_search: Setting::Reset,
|
||||||
chat: Setting::Reset,
|
chat: Setting::Reset,
|
||||||
vector_store: Setting::Reset,
|
vector_store: Setting::Reset,
|
||||||
|
execute_after_update: Setting::Reset,
|
||||||
_kind: PhantomData,
|
_kind: PhantomData,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -423,6 +430,7 @@ impl Settings<Checked> {
|
|||||||
prefix_search,
|
prefix_search,
|
||||||
chat,
|
chat,
|
||||||
vector_store,
|
vector_store,
|
||||||
|
execute_after_update,
|
||||||
_kind,
|
_kind,
|
||||||
} = self;
|
} = self;
|
||||||
|
|
||||||
@@ -449,6 +457,7 @@ impl Settings<Checked> {
|
|||||||
prefix_search,
|
prefix_search,
|
||||||
vector_store,
|
vector_store,
|
||||||
chat,
|
chat,
|
||||||
|
execute_after_update,
|
||||||
_kind: PhantomData,
|
_kind: PhantomData,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -501,6 +510,7 @@ impl Settings<Unchecked> {
|
|||||||
prefix_search: self.prefix_search,
|
prefix_search: self.prefix_search,
|
||||||
chat: self.chat,
|
chat: self.chat,
|
||||||
vector_store: self.vector_store,
|
vector_store: self.vector_store,
|
||||||
|
execute_after_update: self.execute_after_update,
|
||||||
_kind: PhantomData,
|
_kind: PhantomData,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -582,6 +592,10 @@ impl Settings<Unchecked> {
|
|||||||
prefix_search: other.prefix_search.or(self.prefix_search),
|
prefix_search: other.prefix_search.or(self.prefix_search),
|
||||||
chat: other.chat.clone().or(self.chat.clone()),
|
chat: other.chat.clone().or(self.chat.clone()),
|
||||||
vector_store: other.vector_store.or(self.vector_store),
|
vector_store: other.vector_store.or(self.vector_store),
|
||||||
|
execute_after_update: other
|
||||||
|
.execute_after_update
|
||||||
|
.clone()
|
||||||
|
.or(self.execute_after_update.clone()),
|
||||||
_kind: PhantomData,
|
_kind: PhantomData,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -622,6 +636,7 @@ pub fn apply_settings_to_builder(
|
|||||||
prefix_search,
|
prefix_search,
|
||||||
chat,
|
chat,
|
||||||
vector_store,
|
vector_store,
|
||||||
|
execute_after_update,
|
||||||
_kind,
|
_kind,
|
||||||
} = settings;
|
} = settings;
|
||||||
|
|
||||||
@@ -845,6 +860,14 @@ pub fn apply_settings_to_builder(
|
|||||||
Setting::Reset => builder.reset_vector_store(),
|
Setting::Reset => builder.reset_vector_store(),
|
||||||
Setting::NotSet => (),
|
Setting::NotSet => (),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
match execute_after_update {
|
||||||
|
Setting::Set(execute_after_update) => {
|
||||||
|
builder.set_execute_after_update(execute_after_update.clone())
|
||||||
|
}
|
||||||
|
Setting::Reset => builder.reset_execute_after_update(),
|
||||||
|
Setting::NotSet => (),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub enum SecretPolicy {
|
pub enum SecretPolicy {
|
||||||
@@ -944,13 +967,13 @@ pub fn settings(
|
|||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let vector_store = index.get_vector_store(rtxn)?;
|
let vector_store = index.get_vector_store(rtxn)?;
|
||||||
|
|
||||||
let embedders = Setting::Set(embedders);
|
let embedders = Setting::Set(embedders);
|
||||||
let search_cutoff_ms = index.search_cutoff(rtxn)?;
|
let search_cutoff_ms = index.search_cutoff(rtxn)?;
|
||||||
let localized_attributes_rules = index.localized_attributes_rules(rtxn)?;
|
let localized_attributes_rules = index.localized_attributes_rules(rtxn)?;
|
||||||
let prefix_search = index.prefix_search(rtxn)?.map(PrefixSearchSettings::from);
|
let prefix_search = index.prefix_search(rtxn)?.map(PrefixSearchSettings::from);
|
||||||
let facet_search = index.facet_search(rtxn)?;
|
let facet_search = index.facet_search(rtxn)?;
|
||||||
let chat = index.chat_config(rtxn).map(ChatSettings::from)?;
|
let chat = index.chat_config(rtxn).map(ChatSettings::from)?;
|
||||||
|
let execute_after_update = index.execute_after_update(rtxn)?;
|
||||||
|
|
||||||
let mut settings = Settings {
|
let mut settings = Settings {
|
||||||
displayed_attributes: match displayed_attributes {
|
displayed_attributes: match displayed_attributes {
|
||||||
@@ -995,6 +1018,10 @@ pub fn settings(
|
|||||||
Some(vector_store) => Setting::Set(vector_store),
|
Some(vector_store) => Setting::Set(vector_store),
|
||||||
None => Setting::Reset,
|
None => Setting::Reset,
|
||||||
},
|
},
|
||||||
|
execute_after_update: match execute_after_update {
|
||||||
|
Some(function) => Setting::Set(function.to_string()),
|
||||||
|
None => Setting::NotSet,
|
||||||
|
},
|
||||||
_kind: PhantomData,
|
_kind: PhantomData,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1225,6 +1252,7 @@ pub(crate) mod test {
|
|||||||
prefix_search: Setting::NotSet,
|
prefix_search: Setting::NotSet,
|
||||||
chat: Setting::NotSet,
|
chat: Setting::NotSet,
|
||||||
vector_store: Setting::NotSet,
|
vector_store: Setting::NotSet,
|
||||||
|
execute_after_update: Setting::NotSet,
|
||||||
_kind: PhantomData::<Unchecked>,
|
_kind: PhantomData::<Unchecked>,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1258,7 +1286,7 @@ pub(crate) mod test {
|
|||||||
prefix_search: Setting::NotSet,
|
prefix_search: Setting::NotSet,
|
||||||
chat: Setting::NotSet,
|
chat: Setting::NotSet,
|
||||||
vector_store: Setting::NotSet,
|
vector_store: Setting::NotSet,
|
||||||
|
execute_after_update: Setting::NotSet,
|
||||||
_kind: PhantomData::<Unchecked>,
|
_kind: PhantomData::<Unchecked>,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -465,6 +465,17 @@ make_setting_routes!(
|
|||||||
camelcase_attr: "facetSearch",
|
camelcase_attr: "facetSearch",
|
||||||
analytics: FacetSearchAnalytics
|
analytics: FacetSearchAnalytics
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
route: "/execute-after-update",
|
||||||
|
update_verb: put,
|
||||||
|
value_type: String,
|
||||||
|
err_type: meilisearch_types::deserr::DeserrJsonError<
|
||||||
|
meilisearch_types::error::deserr_codes::InvalidSettingsexecuteAfterUpdate,
|
||||||
|
>,
|
||||||
|
attr: execute_after_update,
|
||||||
|
camelcase_attr: "executeAfterUpdate",
|
||||||
|
analytics: ExecuteAfterUpdateAnalytics
|
||||||
|
},
|
||||||
{
|
{
|
||||||
route: "/prefix-search",
|
route: "/prefix-search",
|
||||||
update_verb: put,
|
update_verb: put,
|
||||||
@@ -585,6 +596,9 @@ pub async fn update_all(
|
|||||||
new_settings.non_separator_tokens.as_ref().set(),
|
new_settings.non_separator_tokens.as_ref().set(),
|
||||||
),
|
),
|
||||||
facet_search: FacetSearchAnalytics::new(new_settings.facet_search.as_ref().set()),
|
facet_search: FacetSearchAnalytics::new(new_settings.facet_search.as_ref().set()),
|
||||||
|
execute_after_update: ExecuteAfterUpdateAnalytics::new(
|
||||||
|
new_settings.execute_after_update.as_ref().set(),
|
||||||
|
),
|
||||||
prefix_search: PrefixSearchAnalytics::new(new_settings.prefix_search.as_ref().set()),
|
prefix_search: PrefixSearchAnalytics::new(new_settings.prefix_search.as_ref().set()),
|
||||||
chat: ChatAnalytics::new(new_settings.chat.as_ref().set()),
|
chat: ChatAnalytics::new(new_settings.chat.as_ref().set()),
|
||||||
vector_store: VectorStoreAnalytics::new(new_settings.vector_store.as_ref().set()),
|
vector_store: VectorStoreAnalytics::new(new_settings.vector_store.as_ref().set()),
|
||||||
|
|||||||
@@ -42,6 +42,7 @@ pub struct SettingsAnalytics {
|
|||||||
pub prefix_search: PrefixSearchAnalytics,
|
pub prefix_search: PrefixSearchAnalytics,
|
||||||
pub chat: ChatAnalytics,
|
pub chat: ChatAnalytics,
|
||||||
pub vector_store: VectorStoreAnalytics,
|
pub vector_store: VectorStoreAnalytics,
|
||||||
|
pub execute_after_update: ExecuteAfterUpdateAnalytics,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Aggregate for SettingsAnalytics {
|
impl Aggregate for SettingsAnalytics {
|
||||||
@@ -197,6 +198,9 @@ impl Aggregate for SettingsAnalytics {
|
|||||||
set: new.facet_search.set | self.facet_search.set,
|
set: new.facet_search.set | self.facet_search.set,
|
||||||
value: new.facet_search.value.or(self.facet_search.value),
|
value: new.facet_search.value.or(self.facet_search.value),
|
||||||
},
|
},
|
||||||
|
execute_after_update: ExecuteAfterUpdateAnalytics {
|
||||||
|
set: new.execute_after_update.set | self.execute_after_update.set,
|
||||||
|
},
|
||||||
prefix_search: PrefixSearchAnalytics {
|
prefix_search: PrefixSearchAnalytics {
|
||||||
set: new.prefix_search.set | self.prefix_search.set,
|
set: new.prefix_search.set | self.prefix_search.set,
|
||||||
value: new.prefix_search.value.or(self.prefix_search.value),
|
value: new.prefix_search.value.or(self.prefix_search.value),
|
||||||
@@ -669,6 +673,21 @@ impl FacetSearchAnalytics {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Default)]
|
||||||
|
pub struct ExecuteAfterUpdateAnalytics {
|
||||||
|
pub set: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ExecuteAfterUpdateAnalytics {
|
||||||
|
pub fn new(distinct: Option<&String>) -> Self {
|
||||||
|
Self { set: distinct.is_some() }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn into_settings(self) -> SettingsAnalytics {
|
||||||
|
SettingsAnalytics { execute_after_update: self, ..Default::default() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Default)]
|
#[derive(Serialize, Default)]
|
||||||
pub struct PrefixSearchAnalytics {
|
pub struct PrefixSearchAnalytics {
|
||||||
pub set: bool,
|
pub set: bool,
|
||||||
|
|||||||
@@ -91,7 +91,7 @@ rhai = { version = "1.23.6", features = [
|
|||||||
"sync",
|
"sync",
|
||||||
] }
|
] }
|
||||||
arroy = "0.6.4-nested-rtxns"
|
arroy = "0.6.4-nested-rtxns"
|
||||||
hannoy = { version = "0.1.0-nested-rtxns", features = ["arroy"] }
|
hannoy = { version = "0.1.2-nested-rtxns", features = ["arroy"] }
|
||||||
rand = "0.8.5"
|
rand = "0.8.5"
|
||||||
tracing = "0.1.41"
|
tracing = "0.1.41"
|
||||||
ureq = { version = "2.12.1", features = ["json"] }
|
ureq = { version = "2.12.1", features = ["json"] }
|
||||||
|
|||||||
@@ -84,6 +84,7 @@ pub mod main_key {
|
|||||||
pub const SEARCH_CUTOFF: &str = "search_cutoff";
|
pub const SEARCH_CUTOFF: &str = "search_cutoff";
|
||||||
pub const LOCALIZED_ATTRIBUTES_RULES: &str = "localized_attributes_rules";
|
pub const LOCALIZED_ATTRIBUTES_RULES: &str = "localized_attributes_rules";
|
||||||
pub const FACET_SEARCH: &str = "facet_search";
|
pub const FACET_SEARCH: &str = "facet_search";
|
||||||
|
pub const EXECUTE_AFTER_UPDATE: &str = "execute-after-update";
|
||||||
pub const PREFIX_SEARCH: &str = "prefix_search";
|
pub const PREFIX_SEARCH: &str = "prefix_search";
|
||||||
pub const DOCUMENTS_STATS: &str = "documents_stats";
|
pub const DOCUMENTS_STATS: &str = "documents_stats";
|
||||||
pub const DISABLED_TYPOS_TERMS: &str = "disabled_typos_terms";
|
pub const DISABLED_TYPOS_TERMS: &str = "disabled_typos_terms";
|
||||||
@@ -1768,6 +1769,22 @@ impl Index {
|
|||||||
self.main.remap_key_type::<Str>().delete(txn, main_key::CHAT)
|
self.main.remap_key_type::<Str>().delete(txn, main_key::CHAT)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn execute_after_update<'t>(&self, txn: &'t RoTxn<'_>) -> heed::Result<Option<&'t str>> {
|
||||||
|
self.main.remap_types::<Str, Str>().get(txn, main_key::EXECUTE_AFTER_UPDATE)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn put_execute_after_update(
|
||||||
|
&self,
|
||||||
|
txn: &mut RwTxn<'_>,
|
||||||
|
val: &str,
|
||||||
|
) -> heed::Result<()> {
|
||||||
|
self.main.remap_types::<Str, Str>().put(txn, main_key::EXECUTE_AFTER_UPDATE, &val)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn delete_execute_after_update(&self, txn: &mut RwTxn<'_>) -> heed::Result<bool> {
|
||||||
|
self.main.remap_key_type::<Str>().delete(txn, main_key::EXECUTE_AFTER_UPDATE)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn localized_attributes_rules(
|
pub fn localized_attributes_rules(
|
||||||
&self,
|
&self,
|
||||||
rtxn: &RoTxn<'_>,
|
rtxn: &RoTxn<'_>,
|
||||||
|
|||||||
@@ -470,6 +470,71 @@ impl<'doc> Versions<'doc> {
|
|||||||
Ok(Some(Self::single(data)))
|
Ok(Some(Self::single(data)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn multiple_with_edits(
|
||||||
|
doc: Option<rhai::Map>,
|
||||||
|
mut versions: impl Iterator<Item = Result<RawMap<'doc, FxBuildHasher>>>,
|
||||||
|
engine: &rhai::Engine,
|
||||||
|
edit_function: &rhai::AST,
|
||||||
|
doc_alloc: &'doc bumpalo::Bump,
|
||||||
|
) -> Result<Option<Option<Self>>> {
|
||||||
|
let Some(data) = versions.next() else { return Ok(None) };
|
||||||
|
|
||||||
|
let mut doc = doc.unwrap_or_default();
|
||||||
|
let mut data = data?;
|
||||||
|
for version in versions {
|
||||||
|
let version = version?;
|
||||||
|
for (field, value) in version {
|
||||||
|
data.insert(field, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut scope = rhai::Scope::new();
|
||||||
|
data.iter().for_each(|(k, v)| {
|
||||||
|
doc.insert(k.into(), serde_json::from_str(v.get()).unwrap());
|
||||||
|
});
|
||||||
|
scope.push("doc", doc.clone());
|
||||||
|
|
||||||
|
let _ = engine.eval_ast_with_scope::<rhai::Dynamic>(&mut scope, edit_function).unwrap();
|
||||||
|
data = RawMap::with_hasher_in(FxBuildHasher, doc_alloc);
|
||||||
|
match scope.get_value::<rhai::Map>("doc") {
|
||||||
|
Some(map) => {
|
||||||
|
for (key, value) in map {
|
||||||
|
let mut vec = bumpalo::collections::Vec::new_in(doc_alloc);
|
||||||
|
serde_json::to_writer(&mut vec, &value).unwrap();
|
||||||
|
let key = doc_alloc.alloc_str(key.as_str());
|
||||||
|
let raw_value = serde_json::from_slice(vec.into_bump_slice()).unwrap();
|
||||||
|
data.insert(key, raw_value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// In case the deletes the document and it's not the last change
|
||||||
|
// we simply set the document to an empty one and await the next change.
|
||||||
|
None => (),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// We must also run the code after the last change
|
||||||
|
let mut scope = rhai::Scope::new();
|
||||||
|
data.iter().for_each(|(k, v)| {
|
||||||
|
doc.insert(k.into(), serde_json::from_str(v.get()).unwrap());
|
||||||
|
});
|
||||||
|
scope.push("doc", doc);
|
||||||
|
|
||||||
|
let _ = engine.eval_ast_with_scope::<rhai::Dynamic>(&mut scope, edit_function).unwrap();
|
||||||
|
data = RawMap::with_hasher_in(FxBuildHasher, doc_alloc);
|
||||||
|
match scope.get_value::<rhai::Map>("doc") {
|
||||||
|
Some(map) => {
|
||||||
|
for (key, value) in map {
|
||||||
|
let mut vec = bumpalo::collections::Vec::new_in(doc_alloc);
|
||||||
|
serde_json::to_writer(&mut vec, &value).unwrap();
|
||||||
|
let key = doc_alloc.alloc_str(key.as_str());
|
||||||
|
let raw_value = serde_json::from_slice(vec.into_bump_slice()).unwrap();
|
||||||
|
data.insert(key, raw_value);
|
||||||
|
}
|
||||||
|
Ok(Some(Some(Self::single(data))))
|
||||||
|
}
|
||||||
|
None => Ok(Some(None)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn single(version: RawMap<'doc, FxBuildHasher>) -> Self {
|
pub fn single(version: RawMap<'doc, FxBuildHasher>) -> Self {
|
||||||
Self { data: version }
|
Self { data: version }
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ use crate::documents::PrimaryKey;
|
|||||||
use crate::progress::{AtomicPayloadStep, Progress};
|
use crate::progress::{AtomicPayloadStep, Progress};
|
||||||
use crate::update::new::document::{DocumentContext, Versions};
|
use crate::update::new::document::{DocumentContext, Versions};
|
||||||
use crate::update::new::indexer::current_edition::sharding::Shards;
|
use crate::update::new::indexer::current_edition::sharding::Shards;
|
||||||
|
use crate::update::new::indexer::update_by_function::obkv_to_rhaimap;
|
||||||
use crate::update::new::steps::IndexingStep;
|
use crate::update::new::steps::IndexingStep;
|
||||||
use crate::update::new::thread_local::MostlySend;
|
use crate::update::new::thread_local::MostlySend;
|
||||||
use crate::update::new::{DocumentIdentifiers, Insertion, Update};
|
use crate::update::new::{DocumentIdentifiers, Insertion, Update};
|
||||||
@@ -176,7 +177,16 @@ impl<'pl> DocumentOperation<'pl> {
|
|||||||
.sort_unstable_by_key(|(_, po)| first_update_pointer(&po.operations).unwrap_or(0));
|
.sort_unstable_by_key(|(_, po)| first_update_pointer(&po.operations).unwrap_or(0));
|
||||||
|
|
||||||
let docids_version_offsets = docids_version_offsets.into_bump_slice();
|
let docids_version_offsets = docids_version_offsets.into_bump_slice();
|
||||||
Ok((DocumentOperationChanges { docids_version_offsets }, operations_stats, primary_key))
|
let engine = rhai::Engine::new();
|
||||||
|
// Make sure to correctly setup the engine and remove all settings
|
||||||
|
let ast = index.execute_after_update(rtxn)?.map(|f| engine.compile(f).unwrap());
|
||||||
|
let fidmap = index.fields_ids_map(rtxn)?;
|
||||||
|
|
||||||
|
Ok((
|
||||||
|
DocumentOperationChanges { docids_version_offsets, engine, ast, fidmap },
|
||||||
|
operations_stats,
|
||||||
|
primary_key,
|
||||||
|
))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -474,7 +484,15 @@ impl<'pl> DocumentChanges<'pl> for DocumentOperationChanges<'pl> {
|
|||||||
'pl: 'doc,
|
'pl: 'doc,
|
||||||
{
|
{
|
||||||
let (external_doc, payload_operations) = item;
|
let (external_doc, payload_operations) = item;
|
||||||
payload_operations.merge(external_doc, &context.doc_alloc)
|
payload_operations.merge(
|
||||||
|
&context.rtxn,
|
||||||
|
context.index,
|
||||||
|
&self.fidmap,
|
||||||
|
&self.engine,
|
||||||
|
self.ast.as_ref(),
|
||||||
|
external_doc,
|
||||||
|
&context.doc_alloc,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn len(&self) -> usize {
|
fn len(&self) -> usize {
|
||||||
@@ -483,6 +501,9 @@ impl<'pl> DocumentChanges<'pl> for DocumentOperationChanges<'pl> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub struct DocumentOperationChanges<'pl> {
|
pub struct DocumentOperationChanges<'pl> {
|
||||||
|
engine: rhai::Engine,
|
||||||
|
ast: Option<rhai::AST>,
|
||||||
|
fidmap: FieldsIdsMap,
|
||||||
docids_version_offsets: &'pl [(&'pl str, PayloadOperations<'pl>)],
|
docids_version_offsets: &'pl [(&'pl str, PayloadOperations<'pl>)],
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -571,10 +592,14 @@ impl<'pl> PayloadOperations<'pl> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Returns only the most recent version of a document based on the updates from the payloads.
|
/// Returns only the most recent version of a document based on the updates from the payloads.
|
||||||
///
|
#[allow(clippy::too_many_arguments)]
|
||||||
/// This function is only meant to be used when doing a replacement and not an update.
|
|
||||||
fn merge<'doc>(
|
fn merge<'doc>(
|
||||||
&self,
|
&self,
|
||||||
|
rtxn: &heed::RoTxn,
|
||||||
|
index: &Index,
|
||||||
|
fidmap: &FieldsIdsMap,
|
||||||
|
engine: &rhai::Engine,
|
||||||
|
ast: Option<&rhai::AST>,
|
||||||
external_doc: &'doc str,
|
external_doc: &'doc str,
|
||||||
doc_alloc: &'doc Bump,
|
doc_alloc: &'doc Bump,
|
||||||
) -> Result<Option<DocumentChange<'doc>>>
|
) -> Result<Option<DocumentChange<'doc>>>
|
||||||
@@ -638,9 +663,33 @@ impl<'pl> PayloadOperations<'pl> {
|
|||||||
Ok(document)
|
Ok(document)
|
||||||
});
|
});
|
||||||
|
|
||||||
let Some(versions) = Versions::multiple(versions)? else { return Ok(None) };
|
let versions = match ast {
|
||||||
|
Some(ast) => {
|
||||||
|
let doc = index
|
||||||
|
.documents
|
||||||
|
.get(rtxn, &self.docid)?
|
||||||
|
.map(|obkv| obkv_to_rhaimap(obkv, fidmap))
|
||||||
|
.transpose()?;
|
||||||
|
match Versions::multiple_with_edits(doc, versions, engine, ast, doc_alloc)?
|
||||||
|
{
|
||||||
|
Some(Some(versions)) => Some(versions),
|
||||||
|
Some(None) if self.is_new => return Ok(None),
|
||||||
|
Some(None) => {
|
||||||
|
return Ok(Some(DocumentChange::Deletion(
|
||||||
|
DocumentIdentifiers::create(self.docid, external_doc),
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
None => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => Versions::multiple(versions)?,
|
||||||
|
};
|
||||||
|
|
||||||
if self.is_new {
|
let Some(versions) = versions else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
|
||||||
|
if self.is_new || ast.is_some() {
|
||||||
Ok(Some(DocumentChange::Insertion(Insertion::create(
|
Ok(Some(DocumentChange::Insertion(Insertion::create(
|
||||||
self.docid,
|
self.docid,
|
||||||
external_doc,
|
external_doc,
|
||||||
|
|||||||
@@ -187,7 +187,7 @@ impl<'index> DocumentChanges<'index> for UpdateByFunctionChanges<'index> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn obkv_to_rhaimap(obkv: &KvReaderFieldId, fields_ids_map: &FieldsIdsMap) -> Result<rhai::Map> {
|
pub fn obkv_to_rhaimap(obkv: &KvReaderFieldId, fields_ids_map: &FieldsIdsMap) -> Result<rhai::Map> {
|
||||||
let all_keys = obkv.iter().map(|(k, _v)| k).collect::<Vec<_>>();
|
let all_keys = obkv.iter().map(|(k, _v)| k).collect::<Vec<_>>();
|
||||||
let map: Result<rhai::Map> = all_keys
|
let map: Result<rhai::Map> = all_keys
|
||||||
.iter()
|
.iter()
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ use std::cell::RefCell;
|
|||||||
use std::collections::BTreeSet;
|
use std::collections::BTreeSet;
|
||||||
use std::io::{BufReader, BufWriter, Read, Seek, Write};
|
use std::io::{BufReader, BufWriter, Read, Seek, Write};
|
||||||
use std::iter;
|
use std::iter;
|
||||||
|
use std::num::NonZeroUsize;
|
||||||
|
|
||||||
use hashbrown::HashMap;
|
use hashbrown::HashMap;
|
||||||
use heed::types::{Bytes, DecodeIgnore, Str};
|
use heed::types::{Bytes, DecodeIgnore, Str};
|
||||||
@@ -345,7 +346,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
|
|||||||
indexes.push(PrefixIntegerEntry {
|
indexes.push(PrefixIntegerEntry {
|
||||||
prefix,
|
prefix,
|
||||||
pos,
|
pos,
|
||||||
serialized_length: Some(buffer.len()),
|
serialized_length: NonZeroUsize::new(buffer.len()),
|
||||||
});
|
});
|
||||||
file.write_all(&buffer)?;
|
file.write_all(&buffer)?;
|
||||||
}
|
}
|
||||||
@@ -371,7 +372,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
|
|||||||
key_buffer.extend_from_slice(&pos.to_be_bytes());
|
key_buffer.extend_from_slice(&pos.to_be_bytes());
|
||||||
match serialized_length {
|
match serialized_length {
|
||||||
Some(serialized_length) => {
|
Some(serialized_length) => {
|
||||||
buffer.resize(serialized_length, 0);
|
buffer.resize(serialized_length.get(), 0);
|
||||||
file.read_exact(&mut buffer)?;
|
file.read_exact(&mut buffer)?;
|
||||||
self.prefix_database.remap_data_type::<Bytes>().put(
|
self.prefix_database.remap_data_type::<Bytes>().put(
|
||||||
wtxn,
|
wtxn,
|
||||||
@@ -426,7 +427,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
|
|||||||
index.push(PrefixIntegerEntry {
|
index.push(PrefixIntegerEntry {
|
||||||
prefix,
|
prefix,
|
||||||
pos,
|
pos,
|
||||||
serialized_length: Some(buffer.len()),
|
serialized_length: NonZeroUsize::new(buffer.len()),
|
||||||
});
|
});
|
||||||
file.write_all(buffer)?;
|
file.write_all(buffer)?;
|
||||||
}
|
}
|
||||||
@@ -452,7 +453,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
|
|||||||
key_buffer.extend_from_slice(&pos.to_be_bytes());
|
key_buffer.extend_from_slice(&pos.to_be_bytes());
|
||||||
match serialized_length {
|
match serialized_length {
|
||||||
Some(serialized_length) => {
|
Some(serialized_length) => {
|
||||||
buffer.resize(serialized_length, 0);
|
buffer.resize(serialized_length.get(), 0);
|
||||||
file.read_exact(&mut buffer)?;
|
file.read_exact(&mut buffer)?;
|
||||||
self.prefix_database.remap_data_type::<Bytes>().put(
|
self.prefix_database.remap_data_type::<Bytes>().put(
|
||||||
wtxn,
|
wtxn,
|
||||||
@@ -475,7 +476,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
|
|||||||
struct PrefixIntegerEntry<'a> {
|
struct PrefixIntegerEntry<'a> {
|
||||||
prefix: &'a str,
|
prefix: &'a str,
|
||||||
pos: u16,
|
pos: u16,
|
||||||
serialized_length: Option<usize>,
|
serialized_length: Option<NonZeroUsize>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// TODO doc
|
/// TODO doc
|
||||||
|
|||||||
@@ -197,6 +197,7 @@ pub struct Settings<'a, 't, 'i> {
|
|||||||
facet_search: Setting<bool>,
|
facet_search: Setting<bool>,
|
||||||
chat: Setting<ChatSettings>,
|
chat: Setting<ChatSettings>,
|
||||||
vector_store: Setting<VectorStoreBackend>,
|
vector_store: Setting<VectorStoreBackend>,
|
||||||
|
execute_after_update: Setting<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a, 't, 'i> Settings<'a, 't, 'i> {
|
impl<'a, 't, 'i> Settings<'a, 't, 'i> {
|
||||||
@@ -237,6 +238,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
|
|||||||
facet_search: Setting::NotSet,
|
facet_search: Setting::NotSet,
|
||||||
chat: Setting::NotSet,
|
chat: Setting::NotSet,
|
||||||
vector_store: Setting::NotSet,
|
vector_store: Setting::NotSet,
|
||||||
|
execute_after_update: Setting::NotSet,
|
||||||
indexer_config,
|
indexer_config,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -483,6 +485,14 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
|
|||||||
self.vector_store = Setting::Reset;
|
self.vector_store = Setting::Reset;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn set_execute_after_update(&mut self, value: String) {
|
||||||
|
self.execute_after_update = Setting::Set(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn reset_execute_after_update(&mut self) {
|
||||||
|
self.execute_after_update = Setting::Reset;
|
||||||
|
}
|
||||||
|
|
||||||
#[tracing::instrument(
|
#[tracing::instrument(
|
||||||
level = "trace"
|
level = "trace"
|
||||||
skip(self, progress_callback, should_abort, settings_diff, embedder_stats),
|
skip(self, progress_callback, should_abort, settings_diff, embedder_stats),
|
||||||
@@ -1054,6 +1064,18 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
|
|||||||
Ok(changed)
|
Ok(changed)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn update_execute_after_update(&mut self) -> Result<()> {
|
||||||
|
match self.execute_after_update.as_ref() {
|
||||||
|
Setting::Set(new) => {
|
||||||
|
self.index.put_execute_after_update(self.wtxn, &new).map_err(Into::into)
|
||||||
|
}
|
||||||
|
Setting::Reset => {
|
||||||
|
self.index.delete_execute_after_update(self.wtxn).map(drop).map_err(Into::into)
|
||||||
|
}
|
||||||
|
Setting::NotSet => Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn update_embedding_configs(&mut self) -> Result<BTreeMap<String, EmbedderAction>> {
|
fn update_embedding_configs(&mut self) -> Result<BTreeMap<String, EmbedderAction>> {
|
||||||
match std::mem::take(&mut self.embedder_settings) {
|
match std::mem::take(&mut self.embedder_settings) {
|
||||||
Setting::Set(configs) => self.update_embedding_configs_set(configs),
|
Setting::Set(configs) => self.update_embedding_configs_set(configs),
|
||||||
@@ -1464,6 +1486,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
|
|||||||
self.update_proximity_precision()?;
|
self.update_proximity_precision()?;
|
||||||
self.update_prefix_search()?;
|
self.update_prefix_search()?;
|
||||||
self.update_facet_search()?;
|
self.update_facet_search()?;
|
||||||
|
self.update_execute_after_update()?;
|
||||||
self.update_localized_attributes_rules()?;
|
self.update_localized_attributes_rules()?;
|
||||||
self.update_disabled_typos_terms()?;
|
self.update_disabled_typos_terms()?;
|
||||||
self.update_chat_config()?;
|
self.update_chat_config()?;
|
||||||
@@ -1613,6 +1636,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
|
|||||||
disable_on_numbers: Setting::NotSet, // TODO (require force reindexing of searchables)
|
disable_on_numbers: Setting::NotSet, // TODO (require force reindexing of searchables)
|
||||||
chat: Setting::NotSet,
|
chat: Setting::NotSet,
|
||||||
vector_store: Setting::NotSet,
|
vector_store: Setting::NotSet,
|
||||||
|
execute_after_update: Setting::NotSet,
|
||||||
wtxn: _,
|
wtxn: _,
|
||||||
index: _,
|
index: _,
|
||||||
indexer_config: _,
|
indexer_config: _,
|
||||||
|
|||||||
@@ -892,6 +892,7 @@ fn test_correct_settings_init() {
|
|||||||
disable_on_numbers,
|
disable_on_numbers,
|
||||||
chat,
|
chat,
|
||||||
vector_store,
|
vector_store,
|
||||||
|
execute_after_update,
|
||||||
} = settings;
|
} = settings;
|
||||||
assert!(matches!(searchable_fields, Setting::NotSet));
|
assert!(matches!(searchable_fields, Setting::NotSet));
|
||||||
assert!(matches!(displayed_fields, Setting::NotSet));
|
assert!(matches!(displayed_fields, Setting::NotSet));
|
||||||
@@ -922,6 +923,7 @@ fn test_correct_settings_init() {
|
|||||||
assert!(matches!(disable_on_numbers, Setting::NotSet));
|
assert!(matches!(disable_on_numbers, Setting::NotSet));
|
||||||
assert!(matches!(chat, Setting::NotSet));
|
assert!(matches!(chat, Setting::NotSet));
|
||||||
assert!(matches!(vector_store, Setting::NotSet));
|
assert!(matches!(vector_store, Setting::NotSet));
|
||||||
|
assert!(matches!(execute_after_update, Setting::NotSet));
|
||||||
})
|
})
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user