Compare commits

..

13 Commits

Author SHA1 Message Date
Clément Renault
f4225164fa Merge pull request #6073 from meilisearch/update-version-v1.31.0
Update version for the next release (v1.31.0) in Cargo.toml
2025-12-22 13:33:28 +00:00
Kerollmops
ee9080280e Update version for the next release (v1.31.0) in Cargo.toml 2025-12-22 13:12:00 +00:00
Many the fish
92354d0e03 Merge pull request #6065 from meilisearch/tmp-release-v1.30.1
Bring changes from v1.30.1 into main
2025-12-22 11:27:44 +00:00
Kerollmops
dc7fca4b47 Skip search hugging face on Windows 2025-12-22 12:11:14 +01:00
Kerollmops
a2af104cd0 Fix flaky Ollama tests 2025-12-22 12:11:14 +01:00
Kerollmops
4c666e9774 Fix duplicate import 2025-12-18 18:17:34 +01:00
Kerollmops
7a7e75fc41 Merge remote-tracking branch 'origin/main' into tmp-release-v1.30.1 2025-12-18 17:57:48 +01:00
Clément Renault
7d0633df22 Merge pull request #6064 from meilisearch/hotfix-weightmap-missing-entry 2025-12-18 17:28:33 +01:00
Clément Renault
055c65211f Merge pull request #6059 from meilisearch/exp-806-index-deletion-removed-more-tasks-that-it-should-have-after 2025-12-18 17:21:47 +01:00
ManyTheFish
ae77ca9033 Update Meilisaerch version to v1.30.1 2025-12-18 17:00:41 +01:00
ManyTheFish
2138504df9 Hotfix: log a warning instead of returning an internal error when FieldidsWeightsMap is missing an entry 2025-12-18 16:57:40 +01:00
YoEight
614affd0b1 Fix task attribution during index swap to prevent cross-index task loss 2025-12-18 10:52:17 -05:00
Clémentine
8582975fc5 Make CI test title more human friendly 2025-12-18 16:34:03 +01:00
25 changed files with 224 additions and 354 deletions

34
Cargo.lock generated
View File

@@ -580,7 +580,7 @@ source = "git+https://github.com/meilisearch/bbqueue#e8af4a4bccc8eb36b2b0442c4a9
[[package]]
name = "benchmarks"
version = "1.30.0"
version = "1.31.0"
dependencies = [
"anyhow",
"bumpalo",
@@ -790,7 +790,7 @@ dependencies = [
[[package]]
name = "build-info"
version = "1.30.0"
version = "1.31.0"
dependencies = [
"anyhow",
"time",
@@ -1786,7 +1786,7 @@ dependencies = [
[[package]]
name = "dump"
version = "1.30.0"
version = "1.31.0"
dependencies = [
"anyhow",
"big_s",
@@ -2018,7 +2018,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "file-store"
version = "1.30.0"
version = "1.31.0"
dependencies = [
"tempfile",
"thiserror 2.0.17",
@@ -2040,7 +2040,7 @@ dependencies = [
[[package]]
name = "filter-parser"
version = "1.30.0"
version = "1.31.0"
dependencies = [
"insta",
"levenshtein_automata",
@@ -2068,7 +2068,7 @@ dependencies = [
[[package]]
name = "flatten-serde-json"
version = "1.30.0"
version = "1.31.0"
dependencies = [
"criterion",
"serde_json",
@@ -2231,7 +2231,7 @@ dependencies = [
[[package]]
name = "fuzzers"
version = "1.30.0"
version = "1.31.0"
dependencies = [
"arbitrary",
"bumpalo",
@@ -3185,7 +3185,7 @@ dependencies = [
[[package]]
name = "index-scheduler"
version = "1.30.0"
version = "1.31.0"
dependencies = [
"anyhow",
"backoff",
@@ -3449,7 +3449,7 @@ dependencies = [
[[package]]
name = "json-depth-checker"
version = "1.30.0"
version = "1.31.0"
dependencies = [
"criterion",
"serde_json",
@@ -3939,7 +3939,7 @@ checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0"
[[package]]
name = "meili-snap"
version = "1.30.0"
version = "1.31.0"
dependencies = [
"insta",
"md5 0.8.0",
@@ -3950,7 +3950,7 @@ dependencies = [
[[package]]
name = "meilisearch"
version = "1.30.0"
version = "1.31.0"
dependencies = [
"actix-cors",
"actix-http",
@@ -4048,7 +4048,7 @@ dependencies = [
[[package]]
name = "meilisearch-auth"
version = "1.30.0"
version = "1.31.0"
dependencies = [
"base64 0.22.1",
"enum-iterator",
@@ -4067,7 +4067,7 @@ dependencies = [
[[package]]
name = "meilisearch-types"
version = "1.30.0"
version = "1.31.0"
dependencies = [
"actix-web",
"anyhow",
@@ -4105,7 +4105,7 @@ dependencies = [
[[package]]
name = "meilitool"
version = "1.30.0"
version = "1.31.0"
dependencies = [
"anyhow",
"clap",
@@ -4139,7 +4139,7 @@ dependencies = [
[[package]]
name = "milli"
version = "1.30.0"
version = "1.31.0"
dependencies = [
"arroy",
"bbqueue",
@@ -4718,7 +4718,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "permissive-json-pointer"
version = "1.30.0"
version = "1.31.0"
dependencies = [
"big_s",
"serde_json",
@@ -7758,7 +7758,7 @@ dependencies = [
[[package]]
name = "xtask"
version = "1.30.0"
version = "1.31.0"
dependencies = [
"anyhow",
"build-info",

View File

@@ -23,7 +23,7 @@ members = [
]
[workspace.package]
version = "1.30.0"
version = "1.31.0"
authors = [
"Quentin de Quelen <quentin@dequelen.me>",
"Clément Renault <clement@meilisearch.com>",

View File

@@ -346,7 +346,6 @@ pub(crate) mod test {
prefix_search: Setting::NotSet,
chat: Setting::NotSet,
vector_store: Setting::NotSet,
execute_after_update: Setting::NotSet,
_kind: std::marker::PhantomData,
};
settings.check()

View File

@@ -423,7 +423,6 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
prefix_search: v6::Setting::NotSet,
chat: v6::Setting::NotSet,
vector_store: v6::Setting::NotSet,
execute_after_update: v6::Setting::NotSet,
_kind: std::marker::PhantomData,
}
}

View File

@@ -306,18 +306,6 @@ fn create_or_open_index(
) -> Result<Index> {
let options = EnvOpenOptions::new();
let mut options = options.read_txn_without_tls();
let map_size = match std::env::var("MEILI_MAX_INDEX_SIZE") {
Ok(max_size) => {
let max_size = max_size.parse().unwrap();
map_size.min(max_size)
}
Err(VarError::NotPresent) => map_size,
Err(VarError::NotUnicode(e)) => {
panic!("Non unicode max index size in `MEILI_MAX_INDEX_SIZE`: {e:?}")
}
};
options.map_size(clamp_to_page_size(map_size));
// You can find more details about this experimental

View File

@@ -662,13 +662,8 @@ impl IndexScheduler {
// 2. Get the task set for index = name that appeared before the index swap task
let mut index_lhs_task_ids = self.queue.tasks.index_tasks(wtxn, lhs)?;
index_lhs_task_ids.remove_range(task_id..);
let index_rhs_task_ids = if rename {
let mut index_rhs_task_ids = self.queue.tasks.index_tasks(wtxn, rhs)?;
index_rhs_task_ids.remove_range(task_id..);
index_rhs_task_ids
} else {
RoaringBitmap::new()
};
let mut index_rhs_task_ids = self.queue.tasks.index_tasks(wtxn, rhs)?;
index_rhs_task_ids.remove_range(task_id..);
// 3. before_name -> new_name in the task's KindWithContent
progress.update_progress(InnerSwappingTwoIndexes::UpdateTheTasks);

View File

@@ -7,9 +7,9 @@ source: crates/index-scheduler/src/scheduler/test.rs
----------------------------------------------------------------------
### All Tasks:
0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }}
1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }}
1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }}
2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }}
3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }}
3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }}
4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }}
5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }}
----------------------------------------------------------------------
@@ -22,10 +22,10 @@ succeeded [0,1,2,3,4,]
"indexSwap" [4,5,]
----------------------------------------------------------------------
### Index Tasks:
a [4,5,]
b [0,1,4,]
c [4,5,]
d [2,3,4,]
a [1,4,5,]
b [0,4,]
c [3,4,5,]
d [2,4,]
----------------------------------------------------------------------
### Index Mapper:
a: { number_of_documents: 0, field_distribution: {} }

View File

@@ -7,9 +7,9 @@ source: crates/index-scheduler/src/scheduler/test.rs
----------------------------------------------------------------------
### All Tasks:
0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }}
1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }}
1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }}
2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }}
3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }}
3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }}
4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }}
5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }}
----------------------------------------------------------------------
@@ -22,10 +22,10 @@ succeeded [0,1,2,3,4,5,]
"indexSwap" [4,5,]
----------------------------------------------------------------------
### Index Tasks:
a [5,]
b [0,1,4,]
c [4,5,]
d [2,3,4,]
a [3,4,5,]
b [0,4,]
c [1,4,5,]
d [2,4,]
----------------------------------------------------------------------
### Index Mapper:
a: { number_of_documents: 0, field_distribution: {} }

View File

@@ -7,9 +7,9 @@ source: crates/index-scheduler/src/scheduler/test.rs
----------------------------------------------------------------------
### All Tasks:
0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }}
1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }}
1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }}
2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }}
3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }}
3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }}
4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }}
5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }}
6 {uid: 6, batch_uid: 6, status: succeeded, details: { swaps: [] }, kind: IndexSwap { swaps: [] }}
@@ -23,10 +23,10 @@ succeeded [0,1,2,3,4,5,6,]
"indexSwap" [4,5,6,]
----------------------------------------------------------------------
### Index Tasks:
a [5,]
b [0,1,4,]
c [4,5,]
d [2,3,4,]
a [3,4,5,]
b [0,4,]
c [1,4,5,]
d [2,4,]
----------------------------------------------------------------------
### Index Mapper:
a: { number_of_documents: 0, field_distribution: {} }

View File

@@ -5,7 +5,7 @@ use crate::test_utils::Breakpoint::*;
use crate::test_utils::{
index_creation_task, read_json, replace_document_import_task, sample_documents,
};
use crate::IndexScheduler;
use crate::{IndexScheduler, Query};
use big_s::S;
use meili_snap::{json_string, snapshot};
use meilisearch_auth::AuthFilter;
@@ -404,6 +404,103 @@ fn swap_indexes() {
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "third_empty_swap_processed");
}
#[test]
fn swap_indexes_with_correct_task_allocations() {
let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);
let to_enqueue = [index_creation_task("a", "id"), index_creation_task("b", "id")];
for task in to_enqueue {
let _ = index_scheduler.register(task, None, false).unwrap();
index_scheduler.assert_internally_consistent();
}
handle.advance_n_successful_batches(2);
let (file0, count0) = sample_documents(&index_scheduler, 1, 1);
let (file1, count1) = sample_documents(&index_scheduler, 2, 2);
let (file2, count2) = sample_documents(&index_scheduler, 3, 3);
let (file3, count3) = sample_documents(&index_scheduler, 4, 4);
file0.persist().unwrap();
file1.persist().unwrap();
file2.persist().unwrap();
file3.persist().unwrap();
index_scheduler
.register(replace_document_import_task("a", Some("id"), 1, count0), None, false)
.unwrap();
index_scheduler
.register(replace_document_import_task("a", Some("id"), 2, count1), None, false)
.unwrap();
index_scheduler
.register(replace_document_import_task("b", Some("id"), 3, count2), None, false)
.unwrap();
index_scheduler
.register(replace_document_import_task("b", Some("id"), 4, count3), None, false)
.unwrap();
handle.advance_n_successful_batches(2);
let (a_tasks, _) = index_scheduler
.get_tasks_from_authorized_indexes(
&Query { index_uids: Some(vec!["a".to_string()]), ..Default::default() },
&AuthFilter::default(),
)
.unwrap();
assert_eq!(a_tasks.len(), 3);
let (b_tasks, _) = index_scheduler
.get_tasks_from_authorized_indexes(
&Query { index_uids: Some(vec!["b".to_string()]), ..Default::default() },
&AuthFilter::default(),
)
.unwrap();
assert_eq!(b_tasks.len(), 3);
index_scheduler
.register(
KindWithContent::IndexSwap {
swaps: vec![IndexSwap { indexes: ("a".to_owned(), "b".to_owned()), rename: false }],
},
None,
false,
)
.unwrap();
handle.advance_one_successful_batch();
let (a_after_tasks, _) = index_scheduler
.get_tasks_from_authorized_indexes(
&Query { index_uids: Some(vec!["a".to_string()]), ..Default::default() },
&AuthFilter::default(),
)
.unwrap();
let (b_after_tasks, _) = index_scheduler
.get_tasks_from_authorized_indexes(
&Query { index_uids: Some(vec!["b".to_string()]), ..Default::default() },
&AuthFilter::default(),
)
.unwrap();
assert_eq!(a_after_tasks.len(), 3);
assert_eq!(a_after_tasks.len(), b_after_tasks.len());
for (a, b) in a_tasks.iter().zip(b_after_tasks.iter()) {
assert_eq!(a.uid, b.uid);
}
for (b, a) in b_tasks.iter().zip(a_after_tasks.iter()) {
assert_eq!(b.uid, a.uid);
}
}
#[test]
fn swap_indexes_errors() {
let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);

View File

@@ -328,7 +328,6 @@ InvalidSettingsDisplayedAttributes , InvalidRequest , BAD_REQU
InvalidSettingsDistinctAttribute , InvalidRequest , BAD_REQUEST ;
InvalidSettingsProximityPrecision , InvalidRequest , BAD_REQUEST ;
InvalidSettingsFacetSearch , InvalidRequest , BAD_REQUEST ;
InvalidSettingsexecuteAfterUpdate , InvalidRequest , BAD_REQUEST ;
InvalidSettingsPrefixSearch , InvalidRequest , BAD_REQUEST ;
InvalidSettingsFaceting , InvalidRequest , BAD_REQUEST ;
InvalidSettingsFilterableAttributes , InvalidRequest , BAD_REQUEST ;

View File

@@ -326,12 +326,6 @@ pub struct Settings<T> {
#[schema(value_type = Option<VectorStoreBackend>)]
pub vector_store: Setting<VectorStoreBackend>,
/// Function to execute after an update
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
#[deserr(default, error = DeserrJsonError<InvalidSettingsexecuteAfterUpdate>)]
#[schema(value_type = Option<String>, example = json!("doc.likes += 1"))]
pub execute_after_update: Setting<String>,
#[serde(skip)]
#[deserr(skip)]
pub _kind: PhantomData<T>,
@@ -401,7 +395,6 @@ impl Settings<Checked> {
prefix_search: Setting::Reset,
chat: Setting::Reset,
vector_store: Setting::Reset,
execute_after_update: Setting::Reset,
_kind: PhantomData,
}
}
@@ -430,7 +423,6 @@ impl Settings<Checked> {
prefix_search,
chat,
vector_store,
execute_after_update,
_kind,
} = self;
@@ -457,7 +449,6 @@ impl Settings<Checked> {
prefix_search,
vector_store,
chat,
execute_after_update,
_kind: PhantomData,
}
}
@@ -510,7 +501,6 @@ impl Settings<Unchecked> {
prefix_search: self.prefix_search,
chat: self.chat,
vector_store: self.vector_store,
execute_after_update: self.execute_after_update,
_kind: PhantomData,
}
}
@@ -592,10 +582,6 @@ impl Settings<Unchecked> {
prefix_search: other.prefix_search.or(self.prefix_search),
chat: other.chat.clone().or(self.chat.clone()),
vector_store: other.vector_store.or(self.vector_store),
execute_after_update: other
.execute_after_update
.clone()
.or(self.execute_after_update.clone()),
_kind: PhantomData,
}
}
@@ -636,7 +622,6 @@ pub fn apply_settings_to_builder(
prefix_search,
chat,
vector_store,
execute_after_update,
_kind,
} = settings;
@@ -860,14 +845,6 @@ pub fn apply_settings_to_builder(
Setting::Reset => builder.reset_vector_store(),
Setting::NotSet => (),
}
match execute_after_update {
Setting::Set(execute_after_update) => {
builder.set_execute_after_update(execute_after_update.clone())
}
Setting::Reset => builder.reset_execute_after_update(),
Setting::NotSet => (),
}
}
pub enum SecretPolicy {
@@ -967,13 +944,13 @@ pub fn settings(
.collect();
let vector_store = index.get_vector_store(rtxn)?;
let embedders = Setting::Set(embedders);
let search_cutoff_ms = index.search_cutoff(rtxn)?;
let localized_attributes_rules = index.localized_attributes_rules(rtxn)?;
let prefix_search = index.prefix_search(rtxn)?.map(PrefixSearchSettings::from);
let facet_search = index.facet_search(rtxn)?;
let chat = index.chat_config(rtxn).map(ChatSettings::from)?;
let execute_after_update = index.execute_after_update(rtxn)?;
let mut settings = Settings {
displayed_attributes: match displayed_attributes {
@@ -1018,10 +995,6 @@ pub fn settings(
Some(vector_store) => Setting::Set(vector_store),
None => Setting::Reset,
},
execute_after_update: match execute_after_update {
Some(function) => Setting::Set(function.to_string()),
None => Setting::NotSet,
},
_kind: PhantomData,
};
@@ -1252,7 +1225,6 @@ pub(crate) mod test {
prefix_search: Setting::NotSet,
chat: Setting::NotSet,
vector_store: Setting::NotSet,
execute_after_update: Setting::NotSet,
_kind: PhantomData::<Unchecked>,
};
@@ -1286,7 +1258,7 @@ pub(crate) mod test {
prefix_search: Setting::NotSet,
chat: Setting::NotSet,
vector_store: Setting::NotSet,
execute_after_update: Setting::NotSet,
_kind: PhantomData::<Unchecked>,
};

View File

@@ -465,17 +465,6 @@ make_setting_routes!(
camelcase_attr: "facetSearch",
analytics: FacetSearchAnalytics
},
{
route: "/execute-after-update",
update_verb: put,
value_type: String,
err_type: meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::error::deserr_codes::InvalidSettingsexecuteAfterUpdate,
>,
attr: execute_after_update,
camelcase_attr: "executeAfterUpdate",
analytics: ExecuteAfterUpdateAnalytics
},
{
route: "/prefix-search",
update_verb: put,
@@ -596,9 +585,6 @@ pub async fn update_all(
new_settings.non_separator_tokens.as_ref().set(),
),
facet_search: FacetSearchAnalytics::new(new_settings.facet_search.as_ref().set()),
execute_after_update: ExecuteAfterUpdateAnalytics::new(
new_settings.execute_after_update.as_ref().set(),
),
prefix_search: PrefixSearchAnalytics::new(new_settings.prefix_search.as_ref().set()),
chat: ChatAnalytics::new(new_settings.chat.as_ref().set()),
vector_store: VectorStoreAnalytics::new(new_settings.vector_store.as_ref().set()),

View File

@@ -42,7 +42,6 @@ pub struct SettingsAnalytics {
pub prefix_search: PrefixSearchAnalytics,
pub chat: ChatAnalytics,
pub vector_store: VectorStoreAnalytics,
pub execute_after_update: ExecuteAfterUpdateAnalytics,
}
impl Aggregate for SettingsAnalytics {
@@ -198,9 +197,6 @@ impl Aggregate for SettingsAnalytics {
set: new.facet_search.set | self.facet_search.set,
value: new.facet_search.value.or(self.facet_search.value),
},
execute_after_update: ExecuteAfterUpdateAnalytics {
set: new.execute_after_update.set | self.execute_after_update.set,
},
prefix_search: PrefixSearchAnalytics {
set: new.prefix_search.set | self.prefix_search.set,
value: new.prefix_search.value.or(self.prefix_search.value),
@@ -673,21 +669,6 @@ impl FacetSearchAnalytics {
}
}
#[derive(Serialize, Default)]
pub struct ExecuteAfterUpdateAnalytics {
pub set: bool,
}
impl ExecuteAfterUpdateAnalytics {
pub fn new(distinct: Option<&String>) -> Self {
Self { set: distinct.is_some() }
}
pub fn into_settings(self) -> SettingsAnalytics {
SettingsAnalytics { execute_after_update: self, ..Default::default() }
}
}
#[derive(Serialize, Default)]
pub struct PrefixSearchAnalytics {
pub set: bool,

View File

@@ -452,6 +452,7 @@ async fn limit_offset() {
}
#[actix_rt::test]
#[cfg(not(windows))]
async fn simple_search_hf() {
let server = Server::new_shared();
let index = index_with_documents_hf(server, &SIMPLE_SEARCH_DOCUMENTS).await;

View File

@@ -103,7 +103,7 @@ async fn swap_indexes() {
{
"uid": 1,
"batchUid": 1,
"indexUid": "b",
"indexUid": "a",
"status": "succeeded",
"type": "documentAdditionOrUpdate",
"canceledBy": null,
@@ -266,7 +266,7 @@ async fn swap_indexes() {
{
"uid": 4,
"batchUid": 4,
"indexUid": "d",
"indexUid": "c",
"status": "succeeded",
"type": "documentAdditionOrUpdate",
"canceledBy": null,
@@ -341,7 +341,7 @@ async fn swap_indexes() {
{
"uid": 0,
"batchUid": 0,
"indexUid": "b",
"indexUid": "a",
"status": "succeeded",
"type": "documentAdditionOrUpdate",
"canceledBy": null,

View File

@@ -274,19 +274,19 @@ async fn test_both_apis() {
"birthyear": 2011,
"breed": "Beagle"
},
{
"id": 3,
"name": "Max",
"gender": "M",
"birthyear": 1995,
"breed": "Labrador Retriever"
},
{
"id": 2,
"name": "Vénus",
"gender": "F",
"birthyear": 2003,
"breed": "Jack Russel Terrier"
},
{
"id": 3,
"name": "Max",
"gender": "M",
"birthyear": 1995,
"breed": "Labrador Retriever"
}
]
"###);
@@ -314,19 +314,19 @@ async fn test_both_apis() {
"birthyear": 2011,
"breed": "Beagle"
},
{
"id": 3,
"name": "Max",
"gender": "M",
"birthyear": 1995,
"breed": "Labrador Retriever"
},
{
"id": 2,
"name": "Vénus",
"gender": "F",
"birthyear": 2003,
"breed": "Jack Russel Terrier"
},
{
"id": 3,
"name": "Max",
"gender": "M",
"birthyear": 1995,
"breed": "Labrador Retriever"
}
]
"###);
@@ -354,19 +354,19 @@ async fn test_both_apis() {
"birthyear": 2011,
"breed": "Beagle"
},
{
"id": 0,
"name": "kefir",
"gender": "M",
"birthyear": 2023,
"breed": "Patou"
},
{
"id": 2,
"name": "Vénus",
"gender": "F",
"birthyear": 2003,
"breed": "Jack Russel Terrier"
},
{
"id": 0,
"name": "kefir",
"gender": "M",
"birthyear": 2023,
"breed": "Patou"
}
]
"###);
@@ -394,19 +394,19 @@ async fn test_both_apis() {
"birthyear": 2011,
"breed": "Beagle"
},
{
"id": 0,
"name": "kefir",
"gender": "M",
"birthyear": 2023,
"breed": "Patou"
},
{
"id": 2,
"name": "Vénus",
"gender": "F",
"birthyear": 2003,
"breed": "Jack Russel Terrier"
},
{
"id": 0,
"name": "kefir",
"gender": "M",
"birthyear": 2023,
"breed": "Patou"
}
]
"###);
@@ -420,13 +420,6 @@ async fn test_both_apis() {
snapshot!(code, @"200 OK");
snapshot!(json_string!(response["hits"]), @r###"
[
{
"id": 0,
"name": "kefir",
"gender": "M",
"birthyear": 2023,
"breed": "Patou"
},
{
"id": 1,
"name": "Intel",
@@ -435,11 +428,11 @@ async fn test_both_apis() {
"breed": "Beagle"
},
{
"id": 3,
"name": "Max",
"id": 0,
"name": "kefir",
"gender": "M",
"birthyear": 1995,
"breed": "Labrador Retriever"
"birthyear": 2023,
"breed": "Patou"
},
{
"id": 2,
@@ -447,6 +440,13 @@ async fn test_both_apis() {
"gender": "F",
"birthyear": 2003,
"breed": "Jack Russel Terrier"
},
{
"id": 3,
"name": "Max",
"gender": "M",
"birthyear": 1995,
"breed": "Labrador Retriever"
}
]
"###);
@@ -460,13 +460,6 @@ async fn test_both_apis() {
snapshot!(code, @"200 OK");
snapshot!(json_string!(response["hits"]), @r###"
[
{
"id": 0,
"name": "kefir",
"gender": "M",
"birthyear": 2023,
"breed": "Patou"
},
{
"id": 1,
"name": "Intel",
@@ -475,11 +468,11 @@ async fn test_both_apis() {
"breed": "Beagle"
},
{
"id": 3,
"name": "Max",
"id": 0,
"name": "kefir",
"gender": "M",
"birthyear": 1995,
"breed": "Labrador Retriever"
"birthyear": 2023,
"breed": "Patou"
},
{
"id": 2,
@@ -487,6 +480,13 @@ async fn test_both_apis() {
"gender": "F",
"birthyear": 2003,
"breed": "Jack Russel Terrier"
},
{
"id": 3,
"name": "Max",
"gender": "M",
"birthyear": 1995,
"breed": "Labrador Retriever"
}
]
"###);

View File

@@ -84,7 +84,6 @@ pub mod main_key {
pub const SEARCH_CUTOFF: &str = "search_cutoff";
pub const LOCALIZED_ATTRIBUTES_RULES: &str = "localized_attributes_rules";
pub const FACET_SEARCH: &str = "facet_search";
pub const EXECUTE_AFTER_UPDATE: &str = "execute-after-update";
pub const PREFIX_SEARCH: &str = "prefix_search";
pub const DOCUMENTS_STATS: &str = "documents_stats";
pub const DISABLED_TYPOS_TERMS: &str = "disabled_typos_terms";
@@ -1769,22 +1768,6 @@ impl Index {
self.main.remap_key_type::<Str>().delete(txn, main_key::CHAT)
}
pub fn execute_after_update<'t>(&self, txn: &'t RoTxn<'_>) -> heed::Result<Option<&'t str>> {
self.main.remap_types::<Str, Str>().get(txn, main_key::EXECUTE_AFTER_UPDATE)
}
pub(crate) fn put_execute_after_update(
&self,
txn: &mut RwTxn<'_>,
val: &str,
) -> heed::Result<()> {
self.main.remap_types::<Str, Str>().put(txn, main_key::EXECUTE_AFTER_UPDATE, &val)
}
pub(crate) fn delete_execute_after_update(&self, txn: &mut RwTxn<'_>) -> heed::Result<bool> {
self.main.remap_key_type::<Str>().delete(txn, main_key::EXECUTE_AFTER_UPDATE)
}
pub fn localized_attributes_rules(
&self,
rtxn: &RoTxn<'_>,

View File

@@ -79,16 +79,27 @@ impl RankingRuleGraphTrait for FidGraph {
let mut edges = vec![];
for fid in all_fields.iter().copied() {
let weight = weights_map
.weight(fid)
.ok_or(InternalError::FieldidsWeightsMapMissingEntry { key: fid })?;
if weight > current_max_weight {
current_max_weight = weight;
match weights_map.weight(fid) {
Some(weight) => {
if weight > current_max_weight {
current_max_weight = weight;
}
edges.push((
weight as u32 * term.term_ids.len() as u32,
conditions_interner
.insert(FidCondition { term: term.clone(), fid: Some(fid) }),
));
}
None => {
// Hotfix: this is a temporary solution to handle the case where the weight is not found in the weights map.
// This is due to a database corruption in word_fid_docids database.
tracing::warn!(
"{:?}",
InternalError::FieldidsWeightsMapMissingEntry { key: fid }
);
}
}
edges.push((
weight as u32 * term.term_ids.len() as u32,
conditions_interner.insert(FidCondition { term: term.clone(), fid: Some(fid) }),
));
}
// always lookup the max_fid if we don't already and add an artificial condition for max scoring

View File

@@ -470,71 +470,6 @@ impl<'doc> Versions<'doc> {
Ok(Some(Self::single(data)))
}
pub fn multiple_with_edits(
doc: Option<rhai::Map>,
mut versions: impl Iterator<Item = Result<RawMap<'doc, FxBuildHasher>>>,
engine: &rhai::Engine,
edit_function: &rhai::AST,
doc_alloc: &'doc bumpalo::Bump,
) -> Result<Option<Option<Self>>> {
let Some(data) = versions.next() else { return Ok(None) };
let mut doc = doc.unwrap_or_default();
let mut data = data?;
for version in versions {
let version = version?;
for (field, value) in version {
data.insert(field, value);
}
let mut scope = rhai::Scope::new();
data.iter().for_each(|(k, v)| {
doc.insert(k.into(), serde_json::from_str(v.get()).unwrap());
});
scope.push("doc", doc.clone());
let _ = engine.eval_ast_with_scope::<rhai::Dynamic>(&mut scope, edit_function).unwrap();
data = RawMap::with_hasher_in(FxBuildHasher, doc_alloc);
match scope.get_value::<rhai::Map>("doc") {
Some(map) => {
for (key, value) in map {
let mut vec = bumpalo::collections::Vec::new_in(doc_alloc);
serde_json::to_writer(&mut vec, &value).unwrap();
let key = doc_alloc.alloc_str(key.as_str());
let raw_value = serde_json::from_slice(vec.into_bump_slice()).unwrap();
data.insert(key, raw_value);
}
}
// In case the deletes the document and it's not the last change
// we simply set the document to an empty one and await the next change.
None => (),
}
}
// We must also run the code after the last change
let mut scope = rhai::Scope::new();
data.iter().for_each(|(k, v)| {
doc.insert(k.into(), serde_json::from_str(v.get()).unwrap());
});
scope.push("doc", doc);
let _ = engine.eval_ast_with_scope::<rhai::Dynamic>(&mut scope, edit_function).unwrap();
data = RawMap::with_hasher_in(FxBuildHasher, doc_alloc);
match scope.get_value::<rhai::Map>("doc") {
Some(map) => {
for (key, value) in map {
let mut vec = bumpalo::collections::Vec::new_in(doc_alloc);
serde_json::to_writer(&mut vec, &value).unwrap();
let key = doc_alloc.alloc_str(key.as_str());
let raw_value = serde_json::from_slice(vec.into_bump_slice()).unwrap();
data.insert(key, raw_value);
}
Ok(Some(Some(Self::single(data))))
}
None => Ok(Some(None)),
}
}
pub fn single(version: RawMap<'doc, FxBuildHasher>) -> Self {
Self { data: version }
}

View File

@@ -18,7 +18,6 @@ use crate::documents::PrimaryKey;
use crate::progress::{AtomicPayloadStep, Progress};
use crate::update::new::document::{DocumentContext, Versions};
use crate::update::new::indexer::current_edition::sharding::Shards;
use crate::update::new::indexer::update_by_function::obkv_to_rhaimap;
use crate::update::new::steps::IndexingStep;
use crate::update::new::thread_local::MostlySend;
use crate::update::new::{DocumentIdentifiers, Insertion, Update};
@@ -177,16 +176,7 @@ impl<'pl> DocumentOperation<'pl> {
.sort_unstable_by_key(|(_, po)| first_update_pointer(&po.operations).unwrap_or(0));
let docids_version_offsets = docids_version_offsets.into_bump_slice();
let engine = rhai::Engine::new();
// Make sure to correctly setup the engine and remove all settings
let ast = index.execute_after_update(rtxn)?.map(|f| engine.compile(f).unwrap());
let fidmap = index.fields_ids_map(rtxn)?;
Ok((
DocumentOperationChanges { docids_version_offsets, engine, ast, fidmap },
operations_stats,
primary_key,
))
Ok((DocumentOperationChanges { docids_version_offsets }, operations_stats, primary_key))
}
}
@@ -484,15 +474,7 @@ impl<'pl> DocumentChanges<'pl> for DocumentOperationChanges<'pl> {
'pl: 'doc,
{
let (external_doc, payload_operations) = item;
payload_operations.merge(
&context.rtxn,
context.index,
&self.fidmap,
&self.engine,
self.ast.as_ref(),
external_doc,
&context.doc_alloc,
)
payload_operations.merge(external_doc, &context.doc_alloc)
}
fn len(&self) -> usize {
@@ -501,9 +483,6 @@ impl<'pl> DocumentChanges<'pl> for DocumentOperationChanges<'pl> {
}
pub struct DocumentOperationChanges<'pl> {
engine: rhai::Engine,
ast: Option<rhai::AST>,
fidmap: FieldsIdsMap,
docids_version_offsets: &'pl [(&'pl str, PayloadOperations<'pl>)],
}
@@ -592,14 +571,10 @@ impl<'pl> PayloadOperations<'pl> {
}
/// Returns only the most recent version of a document based on the updates from the payloads.
#[allow(clippy::too_many_arguments)]
///
/// This function is only meant to be used when doing a replacement and not an update.
fn merge<'doc>(
&self,
rtxn: &heed::RoTxn,
index: &Index,
fidmap: &FieldsIdsMap,
engine: &rhai::Engine,
ast: Option<&rhai::AST>,
external_doc: &'doc str,
doc_alloc: &'doc Bump,
) -> Result<Option<DocumentChange<'doc>>>
@@ -663,33 +638,9 @@ impl<'pl> PayloadOperations<'pl> {
Ok(document)
});
let versions = match ast {
Some(ast) => {
let doc = index
.documents
.get(rtxn, &self.docid)?
.map(|obkv| obkv_to_rhaimap(obkv, fidmap))
.transpose()?;
match Versions::multiple_with_edits(doc, versions, engine, ast, doc_alloc)?
{
Some(Some(versions)) => Some(versions),
Some(None) if self.is_new => return Ok(None),
Some(None) => {
return Ok(Some(DocumentChange::Deletion(
DocumentIdentifiers::create(self.docid, external_doc),
)));
}
None => None,
}
}
None => Versions::multiple(versions)?,
};
let Some(versions) = Versions::multiple(versions)? else { return Ok(None) };
let Some(versions) = versions else {
return Ok(None);
};
if self.is_new || ast.is_some() {
if self.is_new {
Ok(Some(DocumentChange::Insertion(Insertion::create(
self.docid,
external_doc,

View File

@@ -187,7 +187,7 @@ impl<'index> DocumentChanges<'index> for UpdateByFunctionChanges<'index> {
}
}
pub fn obkv_to_rhaimap(obkv: &KvReaderFieldId, fields_ids_map: &FieldsIdsMap) -> Result<rhai::Map> {
fn obkv_to_rhaimap(obkv: &KvReaderFieldId, fields_ids_map: &FieldsIdsMap) -> Result<rhai::Map> {
let all_keys = obkv.iter().map(|(k, _v)| k).collect::<Vec<_>>();
let map: Result<rhai::Map> = all_keys
.iter()

View File

@@ -2,7 +2,6 @@ use std::cell::RefCell;
use std::collections::BTreeSet;
use std::io::{BufReader, BufWriter, Read, Seek, Write};
use std::iter;
use std::num::NonZeroUsize;
use hashbrown::HashMap;
use heed::types::{Bytes, DecodeIgnore, Str};
@@ -346,7 +345,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
indexes.push(PrefixIntegerEntry {
prefix,
pos,
serialized_length: NonZeroUsize::new(buffer.len()),
serialized_length: Some(buffer.len()),
});
file.write_all(&buffer)?;
}
@@ -372,7 +371,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
key_buffer.extend_from_slice(&pos.to_be_bytes());
match serialized_length {
Some(serialized_length) => {
buffer.resize(serialized_length.get(), 0);
buffer.resize(serialized_length, 0);
file.read_exact(&mut buffer)?;
self.prefix_database.remap_data_type::<Bytes>().put(
wtxn,
@@ -427,7 +426,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
index.push(PrefixIntegerEntry {
prefix,
pos,
serialized_length: NonZeroUsize::new(buffer.len()),
serialized_length: Some(buffer.len()),
});
file.write_all(buffer)?;
}
@@ -453,7 +452,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
key_buffer.extend_from_slice(&pos.to_be_bytes());
match serialized_length {
Some(serialized_length) => {
buffer.resize(serialized_length.get(), 0);
buffer.resize(serialized_length, 0);
file.read_exact(&mut buffer)?;
self.prefix_database.remap_data_type::<Bytes>().put(
wtxn,
@@ -476,7 +475,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
struct PrefixIntegerEntry<'a> {
prefix: &'a str,
pos: u16,
serialized_length: Option<NonZeroUsize>,
serialized_length: Option<usize>,
}
/// TODO doc

View File

@@ -197,7 +197,6 @@ pub struct Settings<'a, 't, 'i> {
facet_search: Setting<bool>,
chat: Setting<ChatSettings>,
vector_store: Setting<VectorStoreBackend>,
execute_after_update: Setting<String>,
}
impl<'a, 't, 'i> Settings<'a, 't, 'i> {
@@ -238,7 +237,6 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
facet_search: Setting::NotSet,
chat: Setting::NotSet,
vector_store: Setting::NotSet,
execute_after_update: Setting::NotSet,
indexer_config,
}
}
@@ -485,14 +483,6 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
self.vector_store = Setting::Reset;
}
pub fn set_execute_after_update(&mut self, value: String) {
self.execute_after_update = Setting::Set(value);
}
pub fn reset_execute_after_update(&mut self) {
self.execute_after_update = Setting::Reset;
}
#[tracing::instrument(
level = "trace"
skip(self, progress_callback, should_abort, settings_diff, embedder_stats),
@@ -1064,18 +1054,6 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
Ok(changed)
}
fn update_execute_after_update(&mut self) -> Result<()> {
match self.execute_after_update.as_ref() {
Setting::Set(new) => {
self.index.put_execute_after_update(self.wtxn, &new).map_err(Into::into)
}
Setting::Reset => {
self.index.delete_execute_after_update(self.wtxn).map(drop).map_err(Into::into)
}
Setting::NotSet => Ok(()),
}
}
fn update_embedding_configs(&mut self) -> Result<BTreeMap<String, EmbedderAction>> {
match std::mem::take(&mut self.embedder_settings) {
Setting::Set(configs) => self.update_embedding_configs_set(configs),
@@ -1486,7 +1464,6 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
self.update_proximity_precision()?;
self.update_prefix_search()?;
self.update_facet_search()?;
self.update_execute_after_update()?;
self.update_localized_attributes_rules()?;
self.update_disabled_typos_terms()?;
self.update_chat_config()?;
@@ -1636,7 +1613,6 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
disable_on_numbers: Setting::NotSet, // TODO (require force reindexing of searchables)
chat: Setting::NotSet,
vector_store: Setting::NotSet,
execute_after_update: Setting::NotSet,
wtxn: _,
index: _,
indexer_config: _,

View File

@@ -892,7 +892,6 @@ fn test_correct_settings_init() {
disable_on_numbers,
chat,
vector_store,
execute_after_update,
} = settings;
assert!(matches!(searchable_fields, Setting::NotSet));
assert!(matches!(displayed_fields, Setting::NotSet));
@@ -923,7 +922,6 @@ fn test_correct_settings_init() {
assert!(matches!(disable_on_numbers, Setting::NotSet));
assert!(matches!(chat, Setting::NotSet));
assert!(matches!(vector_store, Setting::NotSet));
assert!(matches!(execute_after_update, Setting::NotSet));
})
.unwrap();
}