mirror of
https://github.com/meilisearch/meilisearch.git
synced 2025-12-23 12:57:17 +00:00
Compare commits
13 Commits
post-updat
...
v1.31.0
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f4225164fa | ||
|
|
ee9080280e | ||
|
|
92354d0e03 | ||
|
|
dc7fca4b47 | ||
|
|
a2af104cd0 | ||
|
|
4c666e9774 | ||
|
|
7a7e75fc41 | ||
|
|
7d0633df22 | ||
|
|
055c65211f | ||
|
|
ae77ca9033 | ||
|
|
2138504df9 | ||
|
|
614affd0b1 | ||
|
|
8582975fc5 |
34
Cargo.lock
generated
34
Cargo.lock
generated
@@ -580,7 +580,7 @@ source = "git+https://github.com/meilisearch/bbqueue#e8af4a4bccc8eb36b2b0442c4a9
|
||||
|
||||
[[package]]
|
||||
name = "benchmarks"
|
||||
version = "1.30.0"
|
||||
version = "1.31.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bumpalo",
|
||||
@@ -790,7 +790,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "build-info"
|
||||
version = "1.30.0"
|
||||
version = "1.31.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"time",
|
||||
@@ -1786,7 +1786,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dump"
|
||||
version = "1.30.0"
|
||||
version = "1.31.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"big_s",
|
||||
@@ -2018,7 +2018,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
|
||||
|
||||
[[package]]
|
||||
name = "file-store"
|
||||
version = "1.30.0"
|
||||
version = "1.31.0"
|
||||
dependencies = [
|
||||
"tempfile",
|
||||
"thiserror 2.0.17",
|
||||
@@ -2040,7 +2040,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filter-parser"
|
||||
version = "1.30.0"
|
||||
version = "1.31.0"
|
||||
dependencies = [
|
||||
"insta",
|
||||
"levenshtein_automata",
|
||||
@@ -2068,7 +2068,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "flatten-serde-json"
|
||||
version = "1.30.0"
|
||||
version = "1.31.0"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"serde_json",
|
||||
@@ -2231,7 +2231,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "fuzzers"
|
||||
version = "1.30.0"
|
||||
version = "1.31.0"
|
||||
dependencies = [
|
||||
"arbitrary",
|
||||
"bumpalo",
|
||||
@@ -3185,7 +3185,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "index-scheduler"
|
||||
version = "1.30.0"
|
||||
version = "1.31.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"backoff",
|
||||
@@ -3449,7 +3449,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "json-depth-checker"
|
||||
version = "1.30.0"
|
||||
version = "1.31.0"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"serde_json",
|
||||
@@ -3939,7 +3939,7 @@ checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0"
|
||||
|
||||
[[package]]
|
||||
name = "meili-snap"
|
||||
version = "1.30.0"
|
||||
version = "1.31.0"
|
||||
dependencies = [
|
||||
"insta",
|
||||
"md5 0.8.0",
|
||||
@@ -3950,7 +3950,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "meilisearch"
|
||||
version = "1.30.0"
|
||||
version = "1.31.0"
|
||||
dependencies = [
|
||||
"actix-cors",
|
||||
"actix-http",
|
||||
@@ -4048,7 +4048,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "meilisearch-auth"
|
||||
version = "1.30.0"
|
||||
version = "1.31.0"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"enum-iterator",
|
||||
@@ -4067,7 +4067,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "meilisearch-types"
|
||||
version = "1.30.0"
|
||||
version = "1.31.0"
|
||||
dependencies = [
|
||||
"actix-web",
|
||||
"anyhow",
|
||||
@@ -4105,7 +4105,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "meilitool"
|
||||
version = "1.30.0"
|
||||
version = "1.31.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
@@ -4139,7 +4139,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "milli"
|
||||
version = "1.30.0"
|
||||
version = "1.31.0"
|
||||
dependencies = [
|
||||
"arroy",
|
||||
"bbqueue",
|
||||
@@ -4718,7 +4718,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
||||
|
||||
[[package]]
|
||||
name = "permissive-json-pointer"
|
||||
version = "1.30.0"
|
||||
version = "1.31.0"
|
||||
dependencies = [
|
||||
"big_s",
|
||||
"serde_json",
|
||||
@@ -7758,7 +7758,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "xtask"
|
||||
version = "1.30.0"
|
||||
version = "1.31.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"build-info",
|
||||
|
||||
@@ -23,7 +23,7 @@ members = [
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "1.30.0"
|
||||
version = "1.31.0"
|
||||
authors = [
|
||||
"Quentin de Quelen <quentin@dequelen.me>",
|
||||
"Clément Renault <clement@meilisearch.com>",
|
||||
|
||||
@@ -346,7 +346,6 @@ pub(crate) mod test {
|
||||
prefix_search: Setting::NotSet,
|
||||
chat: Setting::NotSet,
|
||||
vector_store: Setting::NotSet,
|
||||
execute_after_update: Setting::NotSet,
|
||||
_kind: std::marker::PhantomData,
|
||||
};
|
||||
settings.check()
|
||||
|
||||
@@ -423,7 +423,6 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
|
||||
prefix_search: v6::Setting::NotSet,
|
||||
chat: v6::Setting::NotSet,
|
||||
vector_store: v6::Setting::NotSet,
|
||||
execute_after_update: v6::Setting::NotSet,
|
||||
_kind: std::marker::PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -306,18 +306,6 @@ fn create_or_open_index(
|
||||
) -> Result<Index> {
|
||||
let options = EnvOpenOptions::new();
|
||||
let mut options = options.read_txn_without_tls();
|
||||
|
||||
let map_size = match std::env::var("MEILI_MAX_INDEX_SIZE") {
|
||||
Ok(max_size) => {
|
||||
let max_size = max_size.parse().unwrap();
|
||||
map_size.min(max_size)
|
||||
}
|
||||
Err(VarError::NotPresent) => map_size,
|
||||
Err(VarError::NotUnicode(e)) => {
|
||||
panic!("Non unicode max index size in `MEILI_MAX_INDEX_SIZE`: {e:?}")
|
||||
}
|
||||
};
|
||||
|
||||
options.map_size(clamp_to_page_size(map_size));
|
||||
|
||||
// You can find more details about this experimental
|
||||
|
||||
@@ -662,13 +662,8 @@ impl IndexScheduler {
|
||||
// 2. Get the task set for index = name that appeared before the index swap task
|
||||
let mut index_lhs_task_ids = self.queue.tasks.index_tasks(wtxn, lhs)?;
|
||||
index_lhs_task_ids.remove_range(task_id..);
|
||||
let index_rhs_task_ids = if rename {
|
||||
let mut index_rhs_task_ids = self.queue.tasks.index_tasks(wtxn, rhs)?;
|
||||
index_rhs_task_ids.remove_range(task_id..);
|
||||
index_rhs_task_ids
|
||||
} else {
|
||||
RoaringBitmap::new()
|
||||
};
|
||||
let mut index_rhs_task_ids = self.queue.tasks.index_tasks(wtxn, rhs)?;
|
||||
index_rhs_task_ids.remove_range(task_id..);
|
||||
|
||||
// 3. before_name -> new_name in the task's KindWithContent
|
||||
progress.update_progress(InnerSwappingTwoIndexes::UpdateTheTasks);
|
||||
|
||||
@@ -7,9 +7,9 @@ source: crates/index-scheduler/src/scheduler/test.rs
|
||||
----------------------------------------------------------------------
|
||||
### All Tasks:
|
||||
0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }}
|
||||
1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }}
|
||||
1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }}
|
||||
2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }}
|
||||
3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }}
|
||||
3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }}
|
||||
4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }}
|
||||
5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }}
|
||||
----------------------------------------------------------------------
|
||||
@@ -22,10 +22,10 @@ succeeded [0,1,2,3,4,]
|
||||
"indexSwap" [4,5,]
|
||||
----------------------------------------------------------------------
|
||||
### Index Tasks:
|
||||
a [4,5,]
|
||||
b [0,1,4,]
|
||||
c [4,5,]
|
||||
d [2,3,4,]
|
||||
a [1,4,5,]
|
||||
b [0,4,]
|
||||
c [3,4,5,]
|
||||
d [2,4,]
|
||||
----------------------------------------------------------------------
|
||||
### Index Mapper:
|
||||
a: { number_of_documents: 0, field_distribution: {} }
|
||||
|
||||
@@ -7,9 +7,9 @@ source: crates/index-scheduler/src/scheduler/test.rs
|
||||
----------------------------------------------------------------------
|
||||
### All Tasks:
|
||||
0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }}
|
||||
1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }}
|
||||
1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }}
|
||||
2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }}
|
||||
3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }}
|
||||
3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }}
|
||||
4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }}
|
||||
5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }}
|
||||
----------------------------------------------------------------------
|
||||
@@ -22,10 +22,10 @@ succeeded [0,1,2,3,4,5,]
|
||||
"indexSwap" [4,5,]
|
||||
----------------------------------------------------------------------
|
||||
### Index Tasks:
|
||||
a [5,]
|
||||
b [0,1,4,]
|
||||
c [4,5,]
|
||||
d [2,3,4,]
|
||||
a [3,4,5,]
|
||||
b [0,4,]
|
||||
c [1,4,5,]
|
||||
d [2,4,]
|
||||
----------------------------------------------------------------------
|
||||
### Index Mapper:
|
||||
a: { number_of_documents: 0, field_distribution: {} }
|
||||
|
||||
@@ -7,9 +7,9 @@ source: crates/index-scheduler/src/scheduler/test.rs
|
||||
----------------------------------------------------------------------
|
||||
### All Tasks:
|
||||
0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }}
|
||||
1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }}
|
||||
1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }}
|
||||
2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }}
|
||||
3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }}
|
||||
3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }}
|
||||
4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }}
|
||||
5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }}
|
||||
6 {uid: 6, batch_uid: 6, status: succeeded, details: { swaps: [] }, kind: IndexSwap { swaps: [] }}
|
||||
@@ -23,10 +23,10 @@ succeeded [0,1,2,3,4,5,6,]
|
||||
"indexSwap" [4,5,6,]
|
||||
----------------------------------------------------------------------
|
||||
### Index Tasks:
|
||||
a [5,]
|
||||
b [0,1,4,]
|
||||
c [4,5,]
|
||||
d [2,3,4,]
|
||||
a [3,4,5,]
|
||||
b [0,4,]
|
||||
c [1,4,5,]
|
||||
d [2,4,]
|
||||
----------------------------------------------------------------------
|
||||
### Index Mapper:
|
||||
a: { number_of_documents: 0, field_distribution: {} }
|
||||
|
||||
@@ -5,7 +5,7 @@ use crate::test_utils::Breakpoint::*;
|
||||
use crate::test_utils::{
|
||||
index_creation_task, read_json, replace_document_import_task, sample_documents,
|
||||
};
|
||||
use crate::IndexScheduler;
|
||||
use crate::{IndexScheduler, Query};
|
||||
use big_s::S;
|
||||
use meili_snap::{json_string, snapshot};
|
||||
use meilisearch_auth::AuthFilter;
|
||||
@@ -404,6 +404,103 @@ fn swap_indexes() {
|
||||
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "third_empty_swap_processed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn swap_indexes_with_correct_task_allocations() {
|
||||
let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);
|
||||
|
||||
let to_enqueue = [index_creation_task("a", "id"), index_creation_task("b", "id")];
|
||||
|
||||
for task in to_enqueue {
|
||||
let _ = index_scheduler.register(task, None, false).unwrap();
|
||||
index_scheduler.assert_internally_consistent();
|
||||
}
|
||||
|
||||
handle.advance_n_successful_batches(2);
|
||||
|
||||
let (file0, count0) = sample_documents(&index_scheduler, 1, 1);
|
||||
let (file1, count1) = sample_documents(&index_scheduler, 2, 2);
|
||||
let (file2, count2) = sample_documents(&index_scheduler, 3, 3);
|
||||
let (file3, count3) = sample_documents(&index_scheduler, 4, 4);
|
||||
|
||||
file0.persist().unwrap();
|
||||
file1.persist().unwrap();
|
||||
file2.persist().unwrap();
|
||||
file3.persist().unwrap();
|
||||
|
||||
index_scheduler
|
||||
.register(replace_document_import_task("a", Some("id"), 1, count0), None, false)
|
||||
.unwrap();
|
||||
|
||||
index_scheduler
|
||||
.register(replace_document_import_task("a", Some("id"), 2, count1), None, false)
|
||||
.unwrap();
|
||||
|
||||
index_scheduler
|
||||
.register(replace_document_import_task("b", Some("id"), 3, count2), None, false)
|
||||
.unwrap();
|
||||
|
||||
index_scheduler
|
||||
.register(replace_document_import_task("b", Some("id"), 4, count3), None, false)
|
||||
.unwrap();
|
||||
|
||||
handle.advance_n_successful_batches(2);
|
||||
|
||||
let (a_tasks, _) = index_scheduler
|
||||
.get_tasks_from_authorized_indexes(
|
||||
&Query { index_uids: Some(vec!["a".to_string()]), ..Default::default() },
|
||||
&AuthFilter::default(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(a_tasks.len(), 3);
|
||||
|
||||
let (b_tasks, _) = index_scheduler
|
||||
.get_tasks_from_authorized_indexes(
|
||||
&Query { index_uids: Some(vec!["b".to_string()]), ..Default::default() },
|
||||
&AuthFilter::default(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(b_tasks.len(), 3);
|
||||
|
||||
index_scheduler
|
||||
.register(
|
||||
KindWithContent::IndexSwap {
|
||||
swaps: vec![IndexSwap { indexes: ("a".to_owned(), "b".to_owned()), rename: false }],
|
||||
},
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
handle.advance_one_successful_batch();
|
||||
|
||||
let (a_after_tasks, _) = index_scheduler
|
||||
.get_tasks_from_authorized_indexes(
|
||||
&Query { index_uids: Some(vec!["a".to_string()]), ..Default::default() },
|
||||
&AuthFilter::default(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let (b_after_tasks, _) = index_scheduler
|
||||
.get_tasks_from_authorized_indexes(
|
||||
&Query { index_uids: Some(vec!["b".to_string()]), ..Default::default() },
|
||||
&AuthFilter::default(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(a_after_tasks.len(), 3);
|
||||
assert_eq!(a_after_tasks.len(), b_after_tasks.len());
|
||||
|
||||
for (a, b) in a_tasks.iter().zip(b_after_tasks.iter()) {
|
||||
assert_eq!(a.uid, b.uid);
|
||||
}
|
||||
|
||||
for (b, a) in b_tasks.iter().zip(a_after_tasks.iter()) {
|
||||
assert_eq!(b.uid, a.uid);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn swap_indexes_errors() {
|
||||
let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);
|
||||
|
||||
@@ -328,7 +328,6 @@ InvalidSettingsDisplayedAttributes , InvalidRequest , BAD_REQU
|
||||
InvalidSettingsDistinctAttribute , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSettingsProximityPrecision , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSettingsFacetSearch , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSettingsexecuteAfterUpdate , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSettingsPrefixSearch , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSettingsFaceting , InvalidRequest , BAD_REQUEST ;
|
||||
InvalidSettingsFilterableAttributes , InvalidRequest , BAD_REQUEST ;
|
||||
|
||||
@@ -326,12 +326,6 @@ pub struct Settings<T> {
|
||||
#[schema(value_type = Option<VectorStoreBackend>)]
|
||||
pub vector_store: Setting<VectorStoreBackend>,
|
||||
|
||||
/// Function to execute after an update
|
||||
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
|
||||
#[deserr(default, error = DeserrJsonError<InvalidSettingsexecuteAfterUpdate>)]
|
||||
#[schema(value_type = Option<String>, example = json!("doc.likes += 1"))]
|
||||
pub execute_after_update: Setting<String>,
|
||||
|
||||
#[serde(skip)]
|
||||
#[deserr(skip)]
|
||||
pub _kind: PhantomData<T>,
|
||||
@@ -401,7 +395,6 @@ impl Settings<Checked> {
|
||||
prefix_search: Setting::Reset,
|
||||
chat: Setting::Reset,
|
||||
vector_store: Setting::Reset,
|
||||
execute_after_update: Setting::Reset,
|
||||
_kind: PhantomData,
|
||||
}
|
||||
}
|
||||
@@ -430,7 +423,6 @@ impl Settings<Checked> {
|
||||
prefix_search,
|
||||
chat,
|
||||
vector_store,
|
||||
execute_after_update,
|
||||
_kind,
|
||||
} = self;
|
||||
|
||||
@@ -457,7 +449,6 @@ impl Settings<Checked> {
|
||||
prefix_search,
|
||||
vector_store,
|
||||
chat,
|
||||
execute_after_update,
|
||||
_kind: PhantomData,
|
||||
}
|
||||
}
|
||||
@@ -510,7 +501,6 @@ impl Settings<Unchecked> {
|
||||
prefix_search: self.prefix_search,
|
||||
chat: self.chat,
|
||||
vector_store: self.vector_store,
|
||||
execute_after_update: self.execute_after_update,
|
||||
_kind: PhantomData,
|
||||
}
|
||||
}
|
||||
@@ -592,10 +582,6 @@ impl Settings<Unchecked> {
|
||||
prefix_search: other.prefix_search.or(self.prefix_search),
|
||||
chat: other.chat.clone().or(self.chat.clone()),
|
||||
vector_store: other.vector_store.or(self.vector_store),
|
||||
execute_after_update: other
|
||||
.execute_after_update
|
||||
.clone()
|
||||
.or(self.execute_after_update.clone()),
|
||||
_kind: PhantomData,
|
||||
}
|
||||
}
|
||||
@@ -636,7 +622,6 @@ pub fn apply_settings_to_builder(
|
||||
prefix_search,
|
||||
chat,
|
||||
vector_store,
|
||||
execute_after_update,
|
||||
_kind,
|
||||
} = settings;
|
||||
|
||||
@@ -860,14 +845,6 @@ pub fn apply_settings_to_builder(
|
||||
Setting::Reset => builder.reset_vector_store(),
|
||||
Setting::NotSet => (),
|
||||
}
|
||||
|
||||
match execute_after_update {
|
||||
Setting::Set(execute_after_update) => {
|
||||
builder.set_execute_after_update(execute_after_update.clone())
|
||||
}
|
||||
Setting::Reset => builder.reset_execute_after_update(),
|
||||
Setting::NotSet => (),
|
||||
}
|
||||
}
|
||||
|
||||
pub enum SecretPolicy {
|
||||
@@ -967,13 +944,13 @@ pub fn settings(
|
||||
.collect();
|
||||
|
||||
let vector_store = index.get_vector_store(rtxn)?;
|
||||
|
||||
let embedders = Setting::Set(embedders);
|
||||
let search_cutoff_ms = index.search_cutoff(rtxn)?;
|
||||
let localized_attributes_rules = index.localized_attributes_rules(rtxn)?;
|
||||
let prefix_search = index.prefix_search(rtxn)?.map(PrefixSearchSettings::from);
|
||||
let facet_search = index.facet_search(rtxn)?;
|
||||
let chat = index.chat_config(rtxn).map(ChatSettings::from)?;
|
||||
let execute_after_update = index.execute_after_update(rtxn)?;
|
||||
|
||||
let mut settings = Settings {
|
||||
displayed_attributes: match displayed_attributes {
|
||||
@@ -1018,10 +995,6 @@ pub fn settings(
|
||||
Some(vector_store) => Setting::Set(vector_store),
|
||||
None => Setting::Reset,
|
||||
},
|
||||
execute_after_update: match execute_after_update {
|
||||
Some(function) => Setting::Set(function.to_string()),
|
||||
None => Setting::NotSet,
|
||||
},
|
||||
_kind: PhantomData,
|
||||
};
|
||||
|
||||
@@ -1252,7 +1225,6 @@ pub(crate) mod test {
|
||||
prefix_search: Setting::NotSet,
|
||||
chat: Setting::NotSet,
|
||||
vector_store: Setting::NotSet,
|
||||
execute_after_update: Setting::NotSet,
|
||||
_kind: PhantomData::<Unchecked>,
|
||||
};
|
||||
|
||||
@@ -1286,7 +1258,7 @@ pub(crate) mod test {
|
||||
prefix_search: Setting::NotSet,
|
||||
chat: Setting::NotSet,
|
||||
vector_store: Setting::NotSet,
|
||||
execute_after_update: Setting::NotSet,
|
||||
|
||||
_kind: PhantomData::<Unchecked>,
|
||||
};
|
||||
|
||||
|
||||
@@ -465,17 +465,6 @@ make_setting_routes!(
|
||||
camelcase_attr: "facetSearch",
|
||||
analytics: FacetSearchAnalytics
|
||||
},
|
||||
{
|
||||
route: "/execute-after-update",
|
||||
update_verb: put,
|
||||
value_type: String,
|
||||
err_type: meilisearch_types::deserr::DeserrJsonError<
|
||||
meilisearch_types::error::deserr_codes::InvalidSettingsexecuteAfterUpdate,
|
||||
>,
|
||||
attr: execute_after_update,
|
||||
camelcase_attr: "executeAfterUpdate",
|
||||
analytics: ExecuteAfterUpdateAnalytics
|
||||
},
|
||||
{
|
||||
route: "/prefix-search",
|
||||
update_verb: put,
|
||||
@@ -596,9 +585,6 @@ pub async fn update_all(
|
||||
new_settings.non_separator_tokens.as_ref().set(),
|
||||
),
|
||||
facet_search: FacetSearchAnalytics::new(new_settings.facet_search.as_ref().set()),
|
||||
execute_after_update: ExecuteAfterUpdateAnalytics::new(
|
||||
new_settings.execute_after_update.as_ref().set(),
|
||||
),
|
||||
prefix_search: PrefixSearchAnalytics::new(new_settings.prefix_search.as_ref().set()),
|
||||
chat: ChatAnalytics::new(new_settings.chat.as_ref().set()),
|
||||
vector_store: VectorStoreAnalytics::new(new_settings.vector_store.as_ref().set()),
|
||||
|
||||
@@ -42,7 +42,6 @@ pub struct SettingsAnalytics {
|
||||
pub prefix_search: PrefixSearchAnalytics,
|
||||
pub chat: ChatAnalytics,
|
||||
pub vector_store: VectorStoreAnalytics,
|
||||
pub execute_after_update: ExecuteAfterUpdateAnalytics,
|
||||
}
|
||||
|
||||
impl Aggregate for SettingsAnalytics {
|
||||
@@ -198,9 +197,6 @@ impl Aggregate for SettingsAnalytics {
|
||||
set: new.facet_search.set | self.facet_search.set,
|
||||
value: new.facet_search.value.or(self.facet_search.value),
|
||||
},
|
||||
execute_after_update: ExecuteAfterUpdateAnalytics {
|
||||
set: new.execute_after_update.set | self.execute_after_update.set,
|
||||
},
|
||||
prefix_search: PrefixSearchAnalytics {
|
||||
set: new.prefix_search.set | self.prefix_search.set,
|
||||
value: new.prefix_search.value.or(self.prefix_search.value),
|
||||
@@ -673,21 +669,6 @@ impl FacetSearchAnalytics {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Default)]
|
||||
pub struct ExecuteAfterUpdateAnalytics {
|
||||
pub set: bool,
|
||||
}
|
||||
|
||||
impl ExecuteAfterUpdateAnalytics {
|
||||
pub fn new(distinct: Option<&String>) -> Self {
|
||||
Self { set: distinct.is_some() }
|
||||
}
|
||||
|
||||
pub fn into_settings(self) -> SettingsAnalytics {
|
||||
SettingsAnalytics { execute_after_update: self, ..Default::default() }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Default)]
|
||||
pub struct PrefixSearchAnalytics {
|
||||
pub set: bool,
|
||||
|
||||
@@ -452,6 +452,7 @@ async fn limit_offset() {
|
||||
}
|
||||
|
||||
#[actix_rt::test]
|
||||
#[cfg(not(windows))]
|
||||
async fn simple_search_hf() {
|
||||
let server = Server::new_shared();
|
||||
let index = index_with_documents_hf(server, &SIMPLE_SEARCH_DOCUMENTS).await;
|
||||
|
||||
@@ -103,7 +103,7 @@ async fn swap_indexes() {
|
||||
{
|
||||
"uid": 1,
|
||||
"batchUid": 1,
|
||||
"indexUid": "b",
|
||||
"indexUid": "a",
|
||||
"status": "succeeded",
|
||||
"type": "documentAdditionOrUpdate",
|
||||
"canceledBy": null,
|
||||
@@ -266,7 +266,7 @@ async fn swap_indexes() {
|
||||
{
|
||||
"uid": 4,
|
||||
"batchUid": 4,
|
||||
"indexUid": "d",
|
||||
"indexUid": "c",
|
||||
"status": "succeeded",
|
||||
"type": "documentAdditionOrUpdate",
|
||||
"canceledBy": null,
|
||||
@@ -341,7 +341,7 @@ async fn swap_indexes() {
|
||||
{
|
||||
"uid": 0,
|
||||
"batchUid": 0,
|
||||
"indexUid": "b",
|
||||
"indexUid": "a",
|
||||
"status": "succeeded",
|
||||
"type": "documentAdditionOrUpdate",
|
||||
"canceledBy": null,
|
||||
|
||||
@@ -274,19 +274,19 @@ async fn test_both_apis() {
|
||||
"birthyear": 2011,
|
||||
"breed": "Beagle"
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "Max",
|
||||
"gender": "M",
|
||||
"birthyear": 1995,
|
||||
"breed": "Labrador Retriever"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "Vénus",
|
||||
"gender": "F",
|
||||
"birthyear": 2003,
|
||||
"breed": "Jack Russel Terrier"
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "Max",
|
||||
"gender": "M",
|
||||
"birthyear": 1995,
|
||||
"breed": "Labrador Retriever"
|
||||
}
|
||||
]
|
||||
"###);
|
||||
@@ -314,19 +314,19 @@ async fn test_both_apis() {
|
||||
"birthyear": 2011,
|
||||
"breed": "Beagle"
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "Max",
|
||||
"gender": "M",
|
||||
"birthyear": 1995,
|
||||
"breed": "Labrador Retriever"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "Vénus",
|
||||
"gender": "F",
|
||||
"birthyear": 2003,
|
||||
"breed": "Jack Russel Terrier"
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "Max",
|
||||
"gender": "M",
|
||||
"birthyear": 1995,
|
||||
"breed": "Labrador Retriever"
|
||||
}
|
||||
]
|
||||
"###);
|
||||
@@ -354,19 +354,19 @@ async fn test_both_apis() {
|
||||
"birthyear": 2011,
|
||||
"breed": "Beagle"
|
||||
},
|
||||
{
|
||||
"id": 0,
|
||||
"name": "kefir",
|
||||
"gender": "M",
|
||||
"birthyear": 2023,
|
||||
"breed": "Patou"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "Vénus",
|
||||
"gender": "F",
|
||||
"birthyear": 2003,
|
||||
"breed": "Jack Russel Terrier"
|
||||
},
|
||||
{
|
||||
"id": 0,
|
||||
"name": "kefir",
|
||||
"gender": "M",
|
||||
"birthyear": 2023,
|
||||
"breed": "Patou"
|
||||
}
|
||||
]
|
||||
"###);
|
||||
@@ -394,19 +394,19 @@ async fn test_both_apis() {
|
||||
"birthyear": 2011,
|
||||
"breed": "Beagle"
|
||||
},
|
||||
{
|
||||
"id": 0,
|
||||
"name": "kefir",
|
||||
"gender": "M",
|
||||
"birthyear": 2023,
|
||||
"breed": "Patou"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "Vénus",
|
||||
"gender": "F",
|
||||
"birthyear": 2003,
|
||||
"breed": "Jack Russel Terrier"
|
||||
},
|
||||
{
|
||||
"id": 0,
|
||||
"name": "kefir",
|
||||
"gender": "M",
|
||||
"birthyear": 2023,
|
||||
"breed": "Patou"
|
||||
}
|
||||
]
|
||||
"###);
|
||||
@@ -420,13 +420,6 @@ async fn test_both_apis() {
|
||||
snapshot!(code, @"200 OK");
|
||||
snapshot!(json_string!(response["hits"]), @r###"
|
||||
[
|
||||
{
|
||||
"id": 0,
|
||||
"name": "kefir",
|
||||
"gender": "M",
|
||||
"birthyear": 2023,
|
||||
"breed": "Patou"
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"name": "Intel",
|
||||
@@ -435,11 +428,11 @@ async fn test_both_apis() {
|
||||
"breed": "Beagle"
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "Max",
|
||||
"id": 0,
|
||||
"name": "kefir",
|
||||
"gender": "M",
|
||||
"birthyear": 1995,
|
||||
"breed": "Labrador Retriever"
|
||||
"birthyear": 2023,
|
||||
"breed": "Patou"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
@@ -447,6 +440,13 @@ async fn test_both_apis() {
|
||||
"gender": "F",
|
||||
"birthyear": 2003,
|
||||
"breed": "Jack Russel Terrier"
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "Max",
|
||||
"gender": "M",
|
||||
"birthyear": 1995,
|
||||
"breed": "Labrador Retriever"
|
||||
}
|
||||
]
|
||||
"###);
|
||||
@@ -460,13 +460,6 @@ async fn test_both_apis() {
|
||||
snapshot!(code, @"200 OK");
|
||||
snapshot!(json_string!(response["hits"]), @r###"
|
||||
[
|
||||
{
|
||||
"id": 0,
|
||||
"name": "kefir",
|
||||
"gender": "M",
|
||||
"birthyear": 2023,
|
||||
"breed": "Patou"
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"name": "Intel",
|
||||
@@ -475,11 +468,11 @@ async fn test_both_apis() {
|
||||
"breed": "Beagle"
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "Max",
|
||||
"id": 0,
|
||||
"name": "kefir",
|
||||
"gender": "M",
|
||||
"birthyear": 1995,
|
||||
"breed": "Labrador Retriever"
|
||||
"birthyear": 2023,
|
||||
"breed": "Patou"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
@@ -487,6 +480,13 @@ async fn test_both_apis() {
|
||||
"gender": "F",
|
||||
"birthyear": 2003,
|
||||
"breed": "Jack Russel Terrier"
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "Max",
|
||||
"gender": "M",
|
||||
"birthyear": 1995,
|
||||
"breed": "Labrador Retriever"
|
||||
}
|
||||
]
|
||||
"###);
|
||||
|
||||
@@ -84,7 +84,6 @@ pub mod main_key {
|
||||
pub const SEARCH_CUTOFF: &str = "search_cutoff";
|
||||
pub const LOCALIZED_ATTRIBUTES_RULES: &str = "localized_attributes_rules";
|
||||
pub const FACET_SEARCH: &str = "facet_search";
|
||||
pub const EXECUTE_AFTER_UPDATE: &str = "execute-after-update";
|
||||
pub const PREFIX_SEARCH: &str = "prefix_search";
|
||||
pub const DOCUMENTS_STATS: &str = "documents_stats";
|
||||
pub const DISABLED_TYPOS_TERMS: &str = "disabled_typos_terms";
|
||||
@@ -1769,22 +1768,6 @@ impl Index {
|
||||
self.main.remap_key_type::<Str>().delete(txn, main_key::CHAT)
|
||||
}
|
||||
|
||||
pub fn execute_after_update<'t>(&self, txn: &'t RoTxn<'_>) -> heed::Result<Option<&'t str>> {
|
||||
self.main.remap_types::<Str, Str>().get(txn, main_key::EXECUTE_AFTER_UPDATE)
|
||||
}
|
||||
|
||||
pub(crate) fn put_execute_after_update(
|
||||
&self,
|
||||
txn: &mut RwTxn<'_>,
|
||||
val: &str,
|
||||
) -> heed::Result<()> {
|
||||
self.main.remap_types::<Str, Str>().put(txn, main_key::EXECUTE_AFTER_UPDATE, &val)
|
||||
}
|
||||
|
||||
pub(crate) fn delete_execute_after_update(&self, txn: &mut RwTxn<'_>) -> heed::Result<bool> {
|
||||
self.main.remap_key_type::<Str>().delete(txn, main_key::EXECUTE_AFTER_UPDATE)
|
||||
}
|
||||
|
||||
pub fn localized_attributes_rules(
|
||||
&self,
|
||||
rtxn: &RoTxn<'_>,
|
||||
|
||||
@@ -79,16 +79,27 @@ impl RankingRuleGraphTrait for FidGraph {
|
||||
|
||||
let mut edges = vec![];
|
||||
for fid in all_fields.iter().copied() {
|
||||
let weight = weights_map
|
||||
.weight(fid)
|
||||
.ok_or(InternalError::FieldidsWeightsMapMissingEntry { key: fid })?;
|
||||
if weight > current_max_weight {
|
||||
current_max_weight = weight;
|
||||
match weights_map.weight(fid) {
|
||||
Some(weight) => {
|
||||
if weight > current_max_weight {
|
||||
current_max_weight = weight;
|
||||
}
|
||||
|
||||
edges.push((
|
||||
weight as u32 * term.term_ids.len() as u32,
|
||||
conditions_interner
|
||||
.insert(FidCondition { term: term.clone(), fid: Some(fid) }),
|
||||
));
|
||||
}
|
||||
None => {
|
||||
// Hotfix: this is a temporary solution to handle the case where the weight is not found in the weights map.
|
||||
// This is due to a database corruption in word_fid_docids database.
|
||||
tracing::warn!(
|
||||
"{:?}",
|
||||
InternalError::FieldidsWeightsMapMissingEntry { key: fid }
|
||||
);
|
||||
}
|
||||
}
|
||||
edges.push((
|
||||
weight as u32 * term.term_ids.len() as u32,
|
||||
conditions_interner.insert(FidCondition { term: term.clone(), fid: Some(fid) }),
|
||||
));
|
||||
}
|
||||
|
||||
// always lookup the max_fid if we don't already and add an artificial condition for max scoring
|
||||
|
||||
@@ -470,71 +470,6 @@ impl<'doc> Versions<'doc> {
|
||||
Ok(Some(Self::single(data)))
|
||||
}
|
||||
|
||||
pub fn multiple_with_edits(
|
||||
doc: Option<rhai::Map>,
|
||||
mut versions: impl Iterator<Item = Result<RawMap<'doc, FxBuildHasher>>>,
|
||||
engine: &rhai::Engine,
|
||||
edit_function: &rhai::AST,
|
||||
doc_alloc: &'doc bumpalo::Bump,
|
||||
) -> Result<Option<Option<Self>>> {
|
||||
let Some(data) = versions.next() else { return Ok(None) };
|
||||
|
||||
let mut doc = doc.unwrap_or_default();
|
||||
let mut data = data?;
|
||||
for version in versions {
|
||||
let version = version?;
|
||||
for (field, value) in version {
|
||||
data.insert(field, value);
|
||||
}
|
||||
|
||||
let mut scope = rhai::Scope::new();
|
||||
data.iter().for_each(|(k, v)| {
|
||||
doc.insert(k.into(), serde_json::from_str(v.get()).unwrap());
|
||||
});
|
||||
scope.push("doc", doc.clone());
|
||||
|
||||
let _ = engine.eval_ast_with_scope::<rhai::Dynamic>(&mut scope, edit_function).unwrap();
|
||||
data = RawMap::with_hasher_in(FxBuildHasher, doc_alloc);
|
||||
match scope.get_value::<rhai::Map>("doc") {
|
||||
Some(map) => {
|
||||
for (key, value) in map {
|
||||
let mut vec = bumpalo::collections::Vec::new_in(doc_alloc);
|
||||
serde_json::to_writer(&mut vec, &value).unwrap();
|
||||
let key = doc_alloc.alloc_str(key.as_str());
|
||||
let raw_value = serde_json::from_slice(vec.into_bump_slice()).unwrap();
|
||||
data.insert(key, raw_value);
|
||||
}
|
||||
}
|
||||
// In case the deletes the document and it's not the last change
|
||||
// we simply set the document to an empty one and await the next change.
|
||||
None => (),
|
||||
}
|
||||
}
|
||||
|
||||
// We must also run the code after the last change
|
||||
let mut scope = rhai::Scope::new();
|
||||
data.iter().for_each(|(k, v)| {
|
||||
doc.insert(k.into(), serde_json::from_str(v.get()).unwrap());
|
||||
});
|
||||
scope.push("doc", doc);
|
||||
|
||||
let _ = engine.eval_ast_with_scope::<rhai::Dynamic>(&mut scope, edit_function).unwrap();
|
||||
data = RawMap::with_hasher_in(FxBuildHasher, doc_alloc);
|
||||
match scope.get_value::<rhai::Map>("doc") {
|
||||
Some(map) => {
|
||||
for (key, value) in map {
|
||||
let mut vec = bumpalo::collections::Vec::new_in(doc_alloc);
|
||||
serde_json::to_writer(&mut vec, &value).unwrap();
|
||||
let key = doc_alloc.alloc_str(key.as_str());
|
||||
let raw_value = serde_json::from_slice(vec.into_bump_slice()).unwrap();
|
||||
data.insert(key, raw_value);
|
||||
}
|
||||
Ok(Some(Some(Self::single(data))))
|
||||
}
|
||||
None => Ok(Some(None)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn single(version: RawMap<'doc, FxBuildHasher>) -> Self {
|
||||
Self { data: version }
|
||||
}
|
||||
|
||||
@@ -18,7 +18,6 @@ use crate::documents::PrimaryKey;
|
||||
use crate::progress::{AtomicPayloadStep, Progress};
|
||||
use crate::update::new::document::{DocumentContext, Versions};
|
||||
use crate::update::new::indexer::current_edition::sharding::Shards;
|
||||
use crate::update::new::indexer::update_by_function::obkv_to_rhaimap;
|
||||
use crate::update::new::steps::IndexingStep;
|
||||
use crate::update::new::thread_local::MostlySend;
|
||||
use crate::update::new::{DocumentIdentifiers, Insertion, Update};
|
||||
@@ -177,16 +176,7 @@ impl<'pl> DocumentOperation<'pl> {
|
||||
.sort_unstable_by_key(|(_, po)| first_update_pointer(&po.operations).unwrap_or(0));
|
||||
|
||||
let docids_version_offsets = docids_version_offsets.into_bump_slice();
|
||||
let engine = rhai::Engine::new();
|
||||
// Make sure to correctly setup the engine and remove all settings
|
||||
let ast = index.execute_after_update(rtxn)?.map(|f| engine.compile(f).unwrap());
|
||||
let fidmap = index.fields_ids_map(rtxn)?;
|
||||
|
||||
Ok((
|
||||
DocumentOperationChanges { docids_version_offsets, engine, ast, fidmap },
|
||||
operations_stats,
|
||||
primary_key,
|
||||
))
|
||||
Ok((DocumentOperationChanges { docids_version_offsets }, operations_stats, primary_key))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -484,15 +474,7 @@ impl<'pl> DocumentChanges<'pl> for DocumentOperationChanges<'pl> {
|
||||
'pl: 'doc,
|
||||
{
|
||||
let (external_doc, payload_operations) = item;
|
||||
payload_operations.merge(
|
||||
&context.rtxn,
|
||||
context.index,
|
||||
&self.fidmap,
|
||||
&self.engine,
|
||||
self.ast.as_ref(),
|
||||
external_doc,
|
||||
&context.doc_alloc,
|
||||
)
|
||||
payload_operations.merge(external_doc, &context.doc_alloc)
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
@@ -501,9 +483,6 @@ impl<'pl> DocumentChanges<'pl> for DocumentOperationChanges<'pl> {
|
||||
}
|
||||
|
||||
pub struct DocumentOperationChanges<'pl> {
|
||||
engine: rhai::Engine,
|
||||
ast: Option<rhai::AST>,
|
||||
fidmap: FieldsIdsMap,
|
||||
docids_version_offsets: &'pl [(&'pl str, PayloadOperations<'pl>)],
|
||||
}
|
||||
|
||||
@@ -592,14 +571,10 @@ impl<'pl> PayloadOperations<'pl> {
|
||||
}
|
||||
|
||||
/// Returns only the most recent version of a document based on the updates from the payloads.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
///
|
||||
/// This function is only meant to be used when doing a replacement and not an update.
|
||||
fn merge<'doc>(
|
||||
&self,
|
||||
rtxn: &heed::RoTxn,
|
||||
index: &Index,
|
||||
fidmap: &FieldsIdsMap,
|
||||
engine: &rhai::Engine,
|
||||
ast: Option<&rhai::AST>,
|
||||
external_doc: &'doc str,
|
||||
doc_alloc: &'doc Bump,
|
||||
) -> Result<Option<DocumentChange<'doc>>>
|
||||
@@ -663,33 +638,9 @@ impl<'pl> PayloadOperations<'pl> {
|
||||
Ok(document)
|
||||
});
|
||||
|
||||
let versions = match ast {
|
||||
Some(ast) => {
|
||||
let doc = index
|
||||
.documents
|
||||
.get(rtxn, &self.docid)?
|
||||
.map(|obkv| obkv_to_rhaimap(obkv, fidmap))
|
||||
.transpose()?;
|
||||
match Versions::multiple_with_edits(doc, versions, engine, ast, doc_alloc)?
|
||||
{
|
||||
Some(Some(versions)) => Some(versions),
|
||||
Some(None) if self.is_new => return Ok(None),
|
||||
Some(None) => {
|
||||
return Ok(Some(DocumentChange::Deletion(
|
||||
DocumentIdentifiers::create(self.docid, external_doc),
|
||||
)));
|
||||
}
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
None => Versions::multiple(versions)?,
|
||||
};
|
||||
let Some(versions) = Versions::multiple(versions)? else { return Ok(None) };
|
||||
|
||||
let Some(versions) = versions else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
if self.is_new || ast.is_some() {
|
||||
if self.is_new {
|
||||
Ok(Some(DocumentChange::Insertion(Insertion::create(
|
||||
self.docid,
|
||||
external_doc,
|
||||
|
||||
@@ -187,7 +187,7 @@ impl<'index> DocumentChanges<'index> for UpdateByFunctionChanges<'index> {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn obkv_to_rhaimap(obkv: &KvReaderFieldId, fields_ids_map: &FieldsIdsMap) -> Result<rhai::Map> {
|
||||
fn obkv_to_rhaimap(obkv: &KvReaderFieldId, fields_ids_map: &FieldsIdsMap) -> Result<rhai::Map> {
|
||||
let all_keys = obkv.iter().map(|(k, _v)| k).collect::<Vec<_>>();
|
||||
let map: Result<rhai::Map> = all_keys
|
||||
.iter()
|
||||
|
||||
@@ -2,7 +2,6 @@ use std::cell::RefCell;
|
||||
use std::collections::BTreeSet;
|
||||
use std::io::{BufReader, BufWriter, Read, Seek, Write};
|
||||
use std::iter;
|
||||
use std::num::NonZeroUsize;
|
||||
|
||||
use hashbrown::HashMap;
|
||||
use heed::types::{Bytes, DecodeIgnore, Str};
|
||||
@@ -346,7 +345,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
|
||||
indexes.push(PrefixIntegerEntry {
|
||||
prefix,
|
||||
pos,
|
||||
serialized_length: NonZeroUsize::new(buffer.len()),
|
||||
serialized_length: Some(buffer.len()),
|
||||
});
|
||||
file.write_all(&buffer)?;
|
||||
}
|
||||
@@ -372,7 +371,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
|
||||
key_buffer.extend_from_slice(&pos.to_be_bytes());
|
||||
match serialized_length {
|
||||
Some(serialized_length) => {
|
||||
buffer.resize(serialized_length.get(), 0);
|
||||
buffer.resize(serialized_length, 0);
|
||||
file.read_exact(&mut buffer)?;
|
||||
self.prefix_database.remap_data_type::<Bytes>().put(
|
||||
wtxn,
|
||||
@@ -427,7 +426,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
|
||||
index.push(PrefixIntegerEntry {
|
||||
prefix,
|
||||
pos,
|
||||
serialized_length: NonZeroUsize::new(buffer.len()),
|
||||
serialized_length: Some(buffer.len()),
|
||||
});
|
||||
file.write_all(buffer)?;
|
||||
}
|
||||
@@ -453,7 +452,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
|
||||
key_buffer.extend_from_slice(&pos.to_be_bytes());
|
||||
match serialized_length {
|
||||
Some(serialized_length) => {
|
||||
buffer.resize(serialized_length.get(), 0);
|
||||
buffer.resize(serialized_length, 0);
|
||||
file.read_exact(&mut buffer)?;
|
||||
self.prefix_database.remap_data_type::<Bytes>().put(
|
||||
wtxn,
|
||||
@@ -476,7 +475,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
|
||||
struct PrefixIntegerEntry<'a> {
|
||||
prefix: &'a str,
|
||||
pos: u16,
|
||||
serialized_length: Option<NonZeroUsize>,
|
||||
serialized_length: Option<usize>,
|
||||
}
|
||||
|
||||
/// TODO doc
|
||||
|
||||
@@ -197,7 +197,6 @@ pub struct Settings<'a, 't, 'i> {
|
||||
facet_search: Setting<bool>,
|
||||
chat: Setting<ChatSettings>,
|
||||
vector_store: Setting<VectorStoreBackend>,
|
||||
execute_after_update: Setting<String>,
|
||||
}
|
||||
|
||||
impl<'a, 't, 'i> Settings<'a, 't, 'i> {
|
||||
@@ -238,7 +237,6 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
|
||||
facet_search: Setting::NotSet,
|
||||
chat: Setting::NotSet,
|
||||
vector_store: Setting::NotSet,
|
||||
execute_after_update: Setting::NotSet,
|
||||
indexer_config,
|
||||
}
|
||||
}
|
||||
@@ -485,14 +483,6 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
|
||||
self.vector_store = Setting::Reset;
|
||||
}
|
||||
|
||||
pub fn set_execute_after_update(&mut self, value: String) {
|
||||
self.execute_after_update = Setting::Set(value);
|
||||
}
|
||||
|
||||
pub fn reset_execute_after_update(&mut self) {
|
||||
self.execute_after_update = Setting::Reset;
|
||||
}
|
||||
|
||||
#[tracing::instrument(
|
||||
level = "trace"
|
||||
skip(self, progress_callback, should_abort, settings_diff, embedder_stats),
|
||||
@@ -1064,18 +1054,6 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
|
||||
Ok(changed)
|
||||
}
|
||||
|
||||
fn update_execute_after_update(&mut self) -> Result<()> {
|
||||
match self.execute_after_update.as_ref() {
|
||||
Setting::Set(new) => {
|
||||
self.index.put_execute_after_update(self.wtxn, &new).map_err(Into::into)
|
||||
}
|
||||
Setting::Reset => {
|
||||
self.index.delete_execute_after_update(self.wtxn).map(drop).map_err(Into::into)
|
||||
}
|
||||
Setting::NotSet => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn update_embedding_configs(&mut self) -> Result<BTreeMap<String, EmbedderAction>> {
|
||||
match std::mem::take(&mut self.embedder_settings) {
|
||||
Setting::Set(configs) => self.update_embedding_configs_set(configs),
|
||||
@@ -1486,7 +1464,6 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
|
||||
self.update_proximity_precision()?;
|
||||
self.update_prefix_search()?;
|
||||
self.update_facet_search()?;
|
||||
self.update_execute_after_update()?;
|
||||
self.update_localized_attributes_rules()?;
|
||||
self.update_disabled_typos_terms()?;
|
||||
self.update_chat_config()?;
|
||||
@@ -1636,7 +1613,6 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
|
||||
disable_on_numbers: Setting::NotSet, // TODO (require force reindexing of searchables)
|
||||
chat: Setting::NotSet,
|
||||
vector_store: Setting::NotSet,
|
||||
execute_after_update: Setting::NotSet,
|
||||
wtxn: _,
|
||||
index: _,
|
||||
indexer_config: _,
|
||||
|
||||
@@ -892,7 +892,6 @@ fn test_correct_settings_init() {
|
||||
disable_on_numbers,
|
||||
chat,
|
||||
vector_store,
|
||||
execute_after_update,
|
||||
} = settings;
|
||||
assert!(matches!(searchable_fields, Setting::NotSet));
|
||||
assert!(matches!(displayed_fields, Setting::NotSet));
|
||||
@@ -923,7 +922,6 @@ fn test_correct_settings_init() {
|
||||
assert!(matches!(disable_on_numbers, Setting::NotSet));
|
||||
assert!(matches!(chat, Setting::NotSet));
|
||||
assert!(matches!(vector_store, Setting::NotSet));
|
||||
assert!(matches!(execute_after_update, Setting::NotSet));
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user