diff --git a/assets/grafana-dashboard.json b/assets/grafana-dashboard.json index 027afcbba..4c1afd2c7 100644 --- a/assets/grafana-dashboard.json +++ b/assets/grafana-dashboard.json @@ -1501,6 +1501,300 @@ "title": "Task queue latency", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 29, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "meilisearch_task_queue_used_size{instance=\"$instance\", job=\"$job\"}", + "interval": "", + "legendFormat": "{{value}} ", + "range": true, + "refId": "A" + } + ], + "title": "Task queue used size in bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 29, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "meilisearch_task_queue_size_until_stop_registering{instance=\"$instance\", job=\"$job\"}", + "interval": "", + "legendFormat": "{{value}} ", + "range": true, + "refId": "A" + } + ], + "title": "Task queue available size until it stop receiving tasks.", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 29, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "meilisearch_task_queue_max_size{instance=\"$instance\", job=\"$job\"}", + "interval": "", + "legendFormat": "{{value}} ", + "range": true, + "refId": "A" + } + ], + "title": "Task queue maximum possible size", + "type": "stat" + }, { "collapsed": true, "datasource": { diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 7fcc42593..a5afdcbf9 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -74,6 +74,8 @@ use crate::utils::clamp_to_page_size; pub(crate) type BEI128 = I128; +const TASK_SCHEDULER_SIZE_THRESHOLD_PERCENT_INT: u64 = 40; + #[derive(Debug)] pub struct IndexSchedulerOptions { /// The path to the version file of Meilisearch. @@ -425,6 +427,17 @@ impl IndexScheduler { Ok(self.env.non_free_pages_size()?) } + /// Return the maximum possible database size + pub fn max_size(&self) -> Result { + Ok(self.env.info().map_size as u64) + } + + /// Return the max size of task allowed until the task queue stop receiving. + pub fn remaining_size_until_task_queue_stop(&self) -> Result { + Ok((self.env.info().map_size as u64 * TASK_SCHEDULER_SIZE_THRESHOLD_PERCENT_INT / 100) + .saturating_sub(self.used_size()?)) + } + /// Return the index corresponding to the name. /// /// * If the index wasn't opened before, the index will be opened. @@ -627,7 +640,8 @@ impl IndexScheduler { ) -> Result { // if the task doesn't delete or cancel anything and 40% of the task queue is full, we must refuse to enqueue the incoming task if !matches!(&kind, KindWithContent::TaskDeletion { tasks, .. } | KindWithContent::TaskCancelation { tasks, .. } if !tasks.is_empty()) - && (self.env.non_free_pages_size()? * 100) / self.env.info().map_size as u64 > 40 + && (self.env.non_free_pages_size()? * 100) / self.env.info().map_size as u64 + > TASK_SCHEDULER_SIZE_THRESHOLD_PERCENT_INT { return Err(Error::NoSpaceLeftInTaskQueue); } diff --git a/crates/meilisearch/src/metrics.rs b/crates/meilisearch/src/metrics.rs index a48554e6c..29c1aeae8 100644 --- a/crates/meilisearch/src/metrics.rs +++ b/crates/meilisearch/src/metrics.rs @@ -68,4 +68,20 @@ lazy_static! { "Meilisearch Task Queue Latency in Seconds", ) .expect("Can't create a metric"); + pub static ref MEILISEARCH_TASK_QUEUE_MAX_SIZE: IntGauge = register_int_gauge!(opts!( + "meilisearch_task_queue_max_size", + "Meilisearch Task Queue Max Size", + )) + .expect("Can't create a metric"); + pub static ref MEILISEARCH_TASK_QUEUE_USED_SIZE: IntGauge = register_int_gauge!(opts!( + "meilisearch_task_queue_used_size", + "Meilisearch Task Queue Used Size" + )) + .expect("Can't create a metric"); + pub static ref MEILISEARCH_TASK_QUEUE_SIZE_UNTIL_STOP_REGISTERING: IntGauge = + register_int_gauge!(opts!( + "meilisearch_task_queue_size_until_stop_registering", + "Meilisearch Task Queue Size Until Stop Registering", + )) + .expect("Can't create a metric"); } diff --git a/crates/meilisearch/src/routes/metrics.rs b/crates/meilisearch/src/routes/metrics.rs index 6e93284c2..31abb93a7 100644 --- a/crates/meilisearch/src/routes/metrics.rs +++ b/crates/meilisearch/src/routes/metrics.rs @@ -169,6 +169,11 @@ pub async fn get_metrics( .map(|task| (OffsetDateTime::now_utc() - task.enqueued_at).as_seconds_f64()) .unwrap_or(0.0); crate::metrics::MEILISEARCH_TASK_QUEUE_LATENCY_SECONDS.set(task_queue_latency_seconds); + crate::metrics::MEILISEARCH_TASK_QUEUE_MAX_SIZE.set(index_scheduler.max_size()? as i64); + crate::metrics::MEILISEARCH_TASK_QUEUE_USED_SIZE.set(index_scheduler.used_size()? as i64); + + crate::metrics::MEILISEARCH_TASK_QUEUE_SIZE_UNTIL_STOP_REGISTERING + .set(index_scheduler.remaining_size_until_task_queue_stop()? as i64); let encoder = TextEncoder::new(); let mut buffer = vec![];