snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-10-31 07:56:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add more details about the finalizing progress step

Browse Source
This commit is contained in:
Kerollmops
2025-03-25 16:51:18 +01:00
parent 637bea0370
commit 5820d822c8
12 changed files with 122 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
crates/index-scheduler/src/insta_snapshot.rs
View File

@@ -344,6 +344,7 @@ pub fn snapshot_batch(batch: &Batch) -> String {
let Batch { uid, details, stats, started_at, finished_at, progress: _, enqueued_at } = batch; let Batch { uid, details, stats, started_at, finished_at, progress: _, enqueued_at } = batch;
let stats = BatchStats { let stats = BatchStats {
progress_trace: Default::default(), progress_trace: Default::default(),
internal_database_sizes: Default::default(),
write_channel_congestion: None, write_channel_congestion: None,
..stats.clone() ..stats.clone()
}; };

7
crates/index-scheduler/src/processing.rs
View File

@@ -64,6 +64,13 @@ make_enum_progress! {
} }
} }
make_enum_progress! {
pub enum FinalizingIndexStep {
Committing,
ComputingStats,
}
}
make_enum_progress! { make_enum_progress! {
pub enum TaskCancelationProgress { pub enum TaskCancelationProgress {
RetrievingTasks, RetrievingTasks,

6
crates/index-scheduler/src/scheduler/process_batch.rs
View File

@@ -12,7 +12,7 @@ use roaring::RoaringBitmap;
use super::create_batch::Batch; use super::create_batch::Batch;
use crate::processing::{ use crate::processing::{
AtomicBatchStep, AtomicTaskStep, CreateIndexProgress, DeleteIndexProgress, AtomicBatchStep, AtomicTaskStep, CreateIndexProgress, DeleteIndexProgress, FinalizingIndexStep,
InnerSwappingTwoIndexes, SwappingTheIndexes, TaskCancelationProgress, TaskDeletionProgress, InnerSwappingTwoIndexes, SwappingTheIndexes, TaskCancelationProgress, TaskDeletionProgress,
UpdateIndexProgress, UpdateIndexProgress,
}; };
@@ -151,9 +151,10 @@ impl IndexScheduler {
let mut index_wtxn = index.write_txn()?; let mut index_wtxn = index.write_txn()?;
let pre_commit_dabases_sizes = index.database_sizes(&index_wtxn)?; let pre_commit_dabases_sizes = index.database_sizes(&index_wtxn)?;
let (tasks, congestion) = let (tasks, congestion) =
self.apply_index_operation(&mut index_wtxn, &index, op, progress)?; self.apply_index_operation(&mut index_wtxn, &index, op, &progress)?;
{ {
progress.update_progress(FinalizingIndexStep::Committing);
let span = tracing::trace_span!(target: "indexing::scheduler", "commit"); let span = tracing::trace_span!(target: "indexing::scheduler", "commit");
let _entered = span.enter(); let _entered = span.enter();
@@ -166,6 +167,7 @@ impl IndexScheduler {
// the entire batch. // the entire batch.
let mut post_commit_dabases_sizes = None; let mut post_commit_dabases_sizes = None;
let res = || -> Result<()> { let res = || -> Result<()> {
progress.update_progress(FinalizingIndexStep::ComputingStats);
let index_rtxn = index.read_txn()?; let index_rtxn = index.read_txn()?;
let stats = crate::index_mapper::IndexStats::new(&index, &index_rtxn) let stats = crate::index_mapper::IndexStats::new(&index, &index_rtxn)
.map_err(|e| Error::from_milli(e, Some(index_uid.to_string())))?; .map_err(|e| Error::from_milli(e, Some(index_uid.to_string())))?;

10
crates/index-scheduler/src/scheduler/process_index_operation.rs
View File

@@ -32,7 +32,7 @@ impl IndexScheduler {
index_wtxn: &mut RwTxn<'i>, index_wtxn: &mut RwTxn<'i>,
index: &'i Index, index: &'i Index,
operation: IndexOperation, operation: IndexOperation,
progress: Progress, progress: &Progress,
) -> Result<(Vec<Task>, Option<ChannelCongestion>)> { ) -> Result<(Vec<Task>, Option<ChannelCongestion>)> {
let indexer_alloc = Bump::new(); let indexer_alloc = Bump::new();
let started_processing_at = std::time::Instant::now(); let started_processing_at = std::time::Instant::now();
@@ -186,7 +186,7 @@ impl IndexScheduler {
&document_changes, &document_changes,
embedders, embedders,
&|| must_stop_processing.get(), &|| must_stop_processing.get(),
&progress, progress,
) )
.map_err(|e| Error::from_milli(e, Some(index_uid.clone())))?, .map_err(|e| Error::from_milli(e, Some(index_uid.clone())))?,
); );
@@ -307,7 +307,7 @@ impl IndexScheduler {
&document_changes, &document_changes,
embedders, embedders,
&|| must_stop_processing.get(), &|| must_stop_processing.get(),
&progress, progress,
) )
.map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?, .map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?,
); );
@@ -465,7 +465,7 @@ impl IndexScheduler {
&document_changes, &document_changes,
embedders, embedders,
&|| must_stop_processing.get(), &|| must_stop_processing.get(),
&progress, progress,
) )
.map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?, .map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?,
); );
@@ -520,7 +520,7 @@ impl IndexScheduler {
index_uid: index_uid.clone(), index_uid: index_uid.clone(),
tasks: cleared_tasks, tasks: cleared_tasks,
}, },
progress.clone(), progress,
)?; )?;
let (settings_tasks, _congestion) = self.apply_index_operation( let (settings_tasks, _congestion) = self.apply_index_operation(

126
crates/meilisearch/tests/batches/mod.rs
View File

@@ -281,7 +281,8 @@ async fn test_summarized_document_addition_or_update() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r###" @r###"
{ {
@@ -303,7 +304,8 @@ async fn test_summarized_document_addition_or_update() {
"test": 1 "test": 1
}, },
"progressTrace": "[progressTrace]", "progressTrace": "[progressTrace]",
"writeChannelCongestion": "[writeChannelCongestion]" "writeChannelCongestion": "[writeChannelCongestion]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -322,7 +324,8 @@ async fn test_summarized_document_addition_or_update() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r###" @r###"
{ {
@@ -344,7 +347,8 @@ async fn test_summarized_document_addition_or_update() {
"test": 1 "test": 1
}, },
"progressTrace": "[progressTrace]", "progressTrace": "[progressTrace]",
"writeChannelCongestion": "[writeChannelCongestion]" "writeChannelCongestion": "[writeChannelCongestion]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -367,7 +371,8 @@ async fn test_summarized_delete_documents_by_batch() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r###" @r###"
{ {
@@ -388,7 +393,8 @@ async fn test_summarized_delete_documents_by_batch() {
"indexUids": { "indexUids": {
"test": 1 "test": 1
}, },
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -407,7 +413,8 @@ async fn test_summarized_delete_documents_by_batch() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r###" @r###"
{ {
@@ -428,7 +435,8 @@ async fn test_summarized_delete_documents_by_batch() {
"indexUids": { "indexUids": {
"test": 1 "test": 1
}, },
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -453,7 +461,8 @@ async fn test_summarized_delete_documents_by_filter() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r###" @r###"
{ {
@@ -475,7 +484,8 @@ async fn test_summarized_delete_documents_by_filter() {
"indexUids": { "indexUids": {
"test": 1 "test": 1
}, },
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -495,7 +505,8 @@ async fn test_summarized_delete_documents_by_filter() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r###" @r###"
{ {
@@ -517,7 +528,8 @@ async fn test_summarized_delete_documents_by_filter() {
"indexUids": { "indexUids": {
"test": 1 "test": 1
}, },
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -537,7 +549,8 @@ async fn test_summarized_delete_documents_by_filter() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r#" @r#"
{ {
@@ -559,7 +572,8 @@ async fn test_summarized_delete_documents_by_filter() {
"indexUids": { "indexUids": {
"test": 1 "test": 1
}, },
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -583,7 +597,8 @@ async fn test_summarized_delete_document_by_id() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r#" @r#"
{ {
@@ -604,7 +619,8 @@ async fn test_summarized_delete_document_by_id() {
"indexUids": { "indexUids": {
"test": 1 "test": 1
}, },
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -623,7 +639,8 @@ async fn test_summarized_delete_document_by_id() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r#" @r#"
{ {
@@ -644,7 +661,8 @@ async fn test_summarized_delete_document_by_id() {
"indexUids": { "indexUids": {
"test": 1 "test": 1
}, },
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -679,7 +697,8 @@ async fn test_summarized_settings_update() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r###" @r###"
{ {
@@ -709,7 +728,8 @@ async fn test_summarized_settings_update() {
"indexUids": { "indexUids": {
"test": 1 "test": 1
}, },
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -732,7 +752,8 @@ async fn test_summarized_index_creation() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r###" @r###"
{ {
@@ -750,7 +771,8 @@ async fn test_summarized_index_creation() {
"indexUids": { "indexUids": {
"test": 1 "test": 1
}, },
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -768,7 +790,8 @@ async fn test_summarized_index_creation() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r###" @r###"
{ {
@@ -788,7 +811,8 @@ async fn test_summarized_index_creation() {
"indexUids": { "indexUids": {
"test": 1 "test": 1
}, },
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -921,7 +945,8 @@ async fn test_summarized_index_update() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r###" @r###"
{ {
@@ -939,7 +964,8 @@ async fn test_summarized_index_update() {
"indexUids": { "indexUids": {
"test": 1 "test": 1
}, },
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -957,7 +983,8 @@ async fn test_summarized_index_update() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r###" @r###"
{ {
@@ -977,7 +1004,8 @@ async fn test_summarized_index_update() {
"indexUids": { "indexUids": {
"test": 1 "test": 1
}, },
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -998,7 +1026,8 @@ async fn test_summarized_index_update() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r#" @r#"
{ {
@@ -1016,7 +1045,8 @@ async fn test_summarized_index_update() {
"indexUids": { "indexUids": {
"test": 1 "test": 1
}, },
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -1034,7 +1064,8 @@ async fn test_summarized_index_update() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r###" @r###"
{ {
@@ -1054,7 +1085,8 @@ async fn test_summarized_index_update() {
"indexUids": { "indexUids": {
"test": 1 "test": 1
}, },
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -1080,7 +1112,8 @@ async fn test_summarized_index_swap() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r###" @r###"
{ {
@@ -1105,7 +1138,8 @@ async fn test_summarized_index_swap() {
"indexSwap": 1 "indexSwap": 1
}, },
"indexUids": {}, "indexUids": {},
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -1129,7 +1163,8 @@ async fn test_summarized_index_swap() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r###" @r###"
{ {
@@ -1147,7 +1182,8 @@ async fn test_summarized_index_swap() {
"indexUids": { "indexUids": {
"doggos": 1 "doggos": 1
}, },
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -1173,7 +1209,8 @@ async fn test_summarized_batch_cancelation() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r###" @r###"
{ {
@@ -1193,7 +1230,8 @@ async fn test_summarized_batch_cancelation() {
"taskCancelation": 1 "taskCancelation": 1
}, },
"indexUids": {}, "indexUids": {},
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -1219,7 +1257,8 @@ async fn test_summarized_batch_deletion() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r###" @r###"
{ {
@@ -1239,7 +1278,8 @@ async fn test_summarized_batch_deletion() {
"taskDeletion": 1 "taskDeletion": 1
}, },
"indexUids": {}, "indexUids": {},
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
@@ -1262,7 +1302,8 @@ async fn test_summarized_dump_creation() {
".startedAt" => "[date]", ".startedAt" => "[date]",
".finishedAt" => "[date]", ".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]", ".stats.progressTrace" => "[progressTrace]",
".stats.writeChannelCongestion" => "[writeChannelCongestion]" ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
}, },
@r###" @r###"
{ {
@@ -1280,7 +1321,8 @@ async fn test_summarized_dump_creation() {
"dumpCreation": 1 "dumpCreation": 1
}, },
"indexUids": {}, "indexUids": {},
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",

1
crates/meilisearch/tests/dumps/mod.rs
View File

@@ -2236,6 +2236,7 @@ async fn import_dump_v6_containing_batches_and_enqueued_tasks() {
".results[0].finishedAt" => "[date]", ".results[0].finishedAt" => "[date]",
".results[0].duration" => "[date]", ".results[0].duration" => "[date]",
".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.progressTrace" => "[progressTrace]",
".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]",
".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]",
}), name: "batches"); }), name: "batches");

1
crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v6_containing_batches_and_enqueued_tasks/batches.snap
View File

@@ -22,6 +22,7 @@ source: crates/meilisearch/tests/dumps/mod.rs
"kefir": 1 "kefir": 1
}, },
"progressTrace": "[progressTrace]", "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]",
"writeChannelCongestion": "[writeChannelCongestion]" "writeChannelCongestion": "[writeChannelCongestion]"
}, },
"duration": "[date]", "duration": "[date]",

3
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap
View File

@@ -19,7 +19,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"upgradeDatabase": 1 "upgradeDatabase": 1
}, },
"indexUids": {}, "indexUids": {},
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",

3
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap
View File

@@ -19,7 +19,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"upgradeDatabase": 1 "upgradeDatabase": 1
}, },
"indexUids": {}, "indexUids": {},
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",

3
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap
View File

@@ -19,7 +19,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"upgradeDatabase": 1 "upgradeDatabase": 1
}, },
"indexUids": {}, "indexUids": {},
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",

3
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap
View File

@@ -19,7 +19,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"upgradeDatabase": 1 "upgradeDatabase": 1
}, },
"indexUids": {}, "indexUids": {},
"progressTrace": "[progressTrace]" "progressTrace": "[progressTrace]",
"internalDatabaseSizes": "[internalDatabaseSizes]"
}, },
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",

22
crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
View File

@@ -166,7 +166,7 @@ async fn check_the_index_scheduler(server: &Server) {
let (tasks, _) = server.tasks_filter("limit=1000").await; let (tasks, _) = server.tasks_filter("limit=1000").await;
snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "the_whole_task_queue_once_everything_has_been_processed"); snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "the_whole_task_queue_once_everything_has_been_processed");
let (batches, _) = server.batches_filter("limit=1000").await; let (batches, _) = server.batches_filter("limit=1000").await;
snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "the_whole_batch_queue_once_everything_has_been_processed"); snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "the_whole_batch_queue_once_everything_has_been_processed");
// Tests all the tasks query parameters // Tests all the tasks query parameters
let (tasks, _) = server.tasks_filter("uids=10").await; let (tasks, _) = server.tasks_filter("uids=10").await;
@@ -193,26 +193,26 @@ async fn check_the_index_scheduler(server: &Server) {
// Tests all the batches query parameters // Tests all the batches query parameters
let (batches, _) = server.batches_filter("uids=10").await; let (batches, _) = server.batches_filter("uids=10").await;
snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_uids_equal_10"); snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_uids_equal_10");
let (batches, _) = server.batches_filter("batchUids=10").await; let (batches, _) = server.batches_filter("batchUids=10").await;
snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_batchUids_equal_10"); snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_batchUids_equal_10");
let (batches, _) = server.batches_filter("statuses=canceled").await; let (batches, _) = server.batches_filter("statuses=canceled").await;
snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_statuses_equal_canceled"); snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_statuses_equal_canceled");
// types has already been tested above to retrieve the upgrade database // types has already been tested above to retrieve the upgrade database
let (batches, _) = server.batches_filter("canceledBy=19").await; let (batches, _) = server.batches_filter("canceledBy=19").await;
snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_canceledBy_equal_19"); snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_canceledBy_equal_19");
let (batches, _) = server.batches_filter("beforeEnqueuedAt=2025-01-16T16:47:41Z").await; let (batches, _) = server.batches_filter("beforeEnqueuedAt=2025-01-16T16:47:41Z").await;
snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41"); snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41");
let (batches, _) = server.batches_filter("afterEnqueuedAt=2025-01-16T16:47:41Z").await; let (batches, _) = server.batches_filter("afterEnqueuedAt=2025-01-16T16:47:41Z").await;
snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41"); snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41");
let (batches, _) = server.batches_filter("beforeStartedAt=2025-01-16T16:47:41Z").await; let (batches, _) = server.batches_filter("beforeStartedAt=2025-01-16T16:47:41Z").await;
snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeStartedAt_equal_2025-01-16T16_47_41"); snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeStartedAt_equal_2025-01-16T16_47_41");
let (batches, _) = server.batches_filter("afterStartedAt=2025-01-16T16:47:41Z").await; let (batches, _) = server.batches_filter("afterStartedAt=2025-01-16T16:47:41Z").await;
snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterStartedAt_equal_2025-01-16T16_47_41"); snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterStartedAt_equal_2025-01-16T16_47_41");
let (batches, _) = server.batches_filter("beforeFinishedAt=2025-01-16T16:47:41Z").await; let (batches, _) = server.batches_filter("beforeFinishedAt=2025-01-16T16:47:41Z").await;
snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeFinishedAt_equal_2025-01-16T16_47_41"); snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeFinishedAt_equal_2025-01-16T16_47_41");
let (batches, _) = server.batches_filter("afterFinishedAt=2025-01-16T16:47:41Z").await; let (batches, _) = server.batches_filter("afterFinishedAt=2025-01-16T16:47:41Z").await;
snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41"); snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41");
let (stats, _) = server.stats().await; let (stats, _) = server.stats().await;
assert_json_snapshot!(stats, { assert_json_snapshot!(stats, {