diff --git a/BENCHMARKS.md b/BENCHMARKS.md index 6cabb11b5..d4f8ffb72 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -124,6 +124,7 @@ They are JSON files with the following structure (comments are not actually supp { // Name of the workload. Must be unique to the workload, as it will be used to group results on the dashboard. "name": "hackernews.ndjson_1M,no-threads", + "type": "bench", // Number of consecutive runs of the commands that should be performed. // Each run uses a fresh instance of Meilisearch and a fresh database. // Each run produces its own report file. diff --git a/crates/xtask/src/bench/dashboard.rs b/crates/xtask/src/bench/dashboard.rs index a3600a5ec..573585631 100644 --- a/crates/xtask/src/bench/dashboard.rs +++ b/crates/xtask/src/bench/dashboard.rs @@ -8,7 +8,7 @@ use tracing_trace::processor::span_stats::CallStats; use uuid::Uuid; use super::env_info; -use super::workload::Workload; +use super::workload::BenchWorkload; use crate::common::client::Client; #[derive(Debug, Clone)] @@ -89,7 +89,7 @@ impl DashboardClient { pub async fn create_workload( &self, invocation_uuid: Uuid, - workload: &Workload, + workload: &BenchWorkload, ) -> anyhow::Result { let Self::Client(dashboard_client) = self else { return Ok(Uuid::now_v7()) }; diff --git a/crates/xtask/src/bench/meili_process.rs b/crates/xtask/src/bench/meili_process.rs index 50c690977..079ba2b33 100644 --- a/crates/xtask/src/bench/meili_process.rs +++ b/crates/xtask/src/bench/meili_process.rs @@ -5,7 +5,7 @@ use anyhow::{bail, Context as _}; use tokio::process::Command; use tokio::time; -use super::workload::Workload; +use super::workload::BenchWorkload; use crate::common::assets::Asset; use crate::common::client::Client; @@ -68,7 +68,7 @@ pub async fn build() -> anyhow::Result<()> { pub async fn start( client: &Client, master_key: Option<&str>, - workload: &Workload, + workload: &BenchWorkload, asset_folder: &str, mut command: Command, ) -> anyhow::Result { diff --git a/crates/xtask/src/bench/mod.rs b/crates/xtask/src/bench/mod.rs index ea877a01e..63b83e708 100644 --- a/crates/xtask/src/bench/mod.rs +++ b/crates/xtask/src/bench/mod.rs @@ -6,13 +6,14 @@ mod workload; use crate::common::args::CommonArgs; use crate::common::logs::setup_logs; +use crate::common::workload::Workload; use std::path::PathBuf; -use anyhow::Context; +use anyhow::{bail, Context}; use clap::Parser; -use self::workload::Workload; use crate::common::client::Client; +pub use workload::BenchWorkload; pub fn default_http_addr() -> String { "127.0.0.1:7700".to_string() @@ -137,6 +138,10 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> { .with_context(|| format!("error opening {}", workload_file.display()))?, ) .with_context(|| format!("error parsing {} as JSON", workload_file.display()))?; + + let Workload::Bench(workload) = workload else { + bail!("workload file {} is not a bench workload", workload_file.display()); + }; let workload_name = workload.name.clone(); diff --git a/crates/xtask/src/bench/workload.rs b/crates/xtask/src/bench/workload.rs index 65ad1630a..439aec1e4 100644 --- a/crates/xtask/src/bench/workload.rs +++ b/crates/xtask/src/bench/workload.rs @@ -20,7 +20,7 @@ use crate::common::client::Client; /// A bench workload. /// Not to be confused with [a test workload](crate::test::workload::Workload). #[derive(Deserialize)] -pub struct Workload { +pub struct BenchWorkload { pub name: String, pub run_count: u16, pub extra_cli_args: Vec, @@ -37,7 +37,7 @@ async fn run_commands( logs_client: &Client, meili_client: &Client, workload_uuid: Uuid, - workload: &Workload, + workload: &BenchWorkload, args: &BenchDeriveArgs, run_number: u16, ) -> anyhow::Result>> { @@ -86,7 +86,7 @@ pub async fn execute( meili_client: &Client, invocation_uuid: Uuid, master_key: Option<&str>, - workload: Workload, + workload: BenchWorkload, args: &BenchDeriveArgs, binary_path: Option<&Path>, ) -> anyhow::Result<()> { @@ -134,7 +134,7 @@ async fn execute_run( meili_client: &Client, workload_uuid: Uuid, master_key: Option<&str>, - workload: &Workload, + workload: &BenchWorkload, args: &BenchDeriveArgs, binary_path: Option<&Path>, run_number: u16, diff --git a/crates/xtask/src/common/mod.rs b/crates/xtask/src/common/mod.rs index 3f8feb1a2..c8fbef1c2 100644 --- a/crates/xtask/src/common/mod.rs +++ b/crates/xtask/src/common/mod.rs @@ -2,3 +2,4 @@ pub mod args; pub mod assets; pub mod client; pub mod logs; +pub mod workload; diff --git a/crates/xtask/src/common/workload.rs b/crates/xtask/src/common/workload.rs new file mode 100644 index 000000000..88ba651b2 --- /dev/null +++ b/crates/xtask/src/common/workload.rs @@ -0,0 +1,10 @@ +use serde::Deserialize; + +use crate::{bench::BenchWorkload, test::TestWorkload}; + +#[derive(Deserialize)] +#[serde(tag = "type")] +pub enum Workload { + Bench(BenchWorkload), + Test(TestWorkload), +} diff --git a/crates/xtask/src/test/mod.rs b/crates/xtask/src/test/mod.rs index ce77a8726..aefe34601 100644 --- a/crates/xtask/src/test/mod.rs +++ b/crates/xtask/src/test/mod.rs @@ -4,6 +4,8 @@ use crate::common::{args::CommonArgs, logs::setup_logs}; mod workload; +pub use workload::TestWorkload; + /// Run tests from a workload #[derive(Parser, Debug)] pub struct TestDeriveArgs { diff --git a/crates/xtask/src/test/workload.rs b/crates/xtask/src/test/workload.rs index 1063d6d89..3c88e1af4 100644 --- a/crates/xtask/src/test/workload.rs +++ b/crates/xtask/src/test/workload.rs @@ -1,12 +1,12 @@ -use std::collections::BTreeMap; use serde::Deserialize; +use std::collections::BTreeMap; use crate::common::assets::Asset; /// A test workload. /// Not to be confused with [a bench workload](crate::bench::workload::Workload). #[derive(Deserialize)] -pub struct Workload { +pub struct TestWorkload { pub name: String, pub assets: BTreeMap, } diff --git a/workloads/embeddings-movies-subset-hf.json b/workloads/embeddings-movies-subset-hf.json index 4f6c5be35..32ccb3656 100644 --- a/workloads/embeddings-movies-subset-hf.json +++ b/workloads/embeddings-movies-subset-hf.json @@ -1,5 +1,6 @@ { "name": "movies-subset-hf-embeddings", + "type": "bench", "run_count": 5, "extra_cli_args": [ "--max-indexing-threads=4" diff --git a/workloads/embeddings-settings-add.json b/workloads/embeddings-settings-add.json index 67f9709db..6fd7622bf 100644 --- a/workloads/embeddings-settings-add.json +++ b/workloads/embeddings-settings-add.json @@ -1,5 +1,6 @@ { "name": "settings-add-embeddings-hf", + "type": "bench", "run_count": 5, "extra_cli_args": [ "--max-indexing-threads=4" diff --git a/workloads/hackernews-add-new-documents.json b/workloads/hackernews-add-new-documents.json index 0470a0792..aa9607ccb 100644 --- a/workloads/hackernews-add-new-documents.json +++ b/workloads/hackernews-add-new-documents.json @@ -1,5 +1,6 @@ { "name": "hackernews.add_new_documents", + "type": "bench", "run_count": 3, "extra_cli_args": [], "assets": { diff --git a/workloads/hackernews-ignore-first-100k.json b/workloads/hackernews-ignore-first-100k.json index e65abb98a..112c7f695 100644 --- a/workloads/hackernews-ignore-first-100k.json +++ b/workloads/hackernews-ignore-first-100k.json @@ -1,5 +1,6 @@ { "name": "hackernews.ndjson_1M_ignore_first_100k", + "type": "bench", "run_count": 3, "extra_cli_args": [], "assets": { diff --git a/workloads/hackernews-modify-facet-numbers.json b/workloads/hackernews-modify-facet-numbers.json index 5c6acf626..af2b0a6d1 100644 --- a/workloads/hackernews-modify-facet-numbers.json +++ b/workloads/hackernews-modify-facet-numbers.json @@ -1,5 +1,6 @@ { "name": "hackernews.modify_facet_numbers", + "type": "bench", "run_count": 3, "extra_cli_args": [], "assets": { diff --git a/workloads/hackernews-modify-facet-strings.json b/workloads/hackernews-modify-facet-strings.json index b5d4235a0..225799641 100644 --- a/workloads/hackernews-modify-facet-strings.json +++ b/workloads/hackernews-modify-facet-strings.json @@ -1,5 +1,6 @@ { "name": "hackernews.modify_facet_strings", + "type": "bench", "run_count": 3, "extra_cli_args": [], "assets": { diff --git a/workloads/hackernews-modify-searchables.json b/workloads/hackernews-modify-searchables.json index 248026f19..a6291c716 100644 --- a/workloads/hackernews-modify-searchables.json +++ b/workloads/hackernews-modify-searchables.json @@ -1,5 +1,6 @@ { "name": "hackernews.modify_searchables", + "type": "bench", "run_count": 3, "extra_cli_args": [], "assets": { diff --git a/workloads/hackernews.json b/workloads/hackernews.json index 5762a7309..5726a1440 100644 --- a/workloads/hackernews.json +++ b/workloads/hackernews.json @@ -1,5 +1,6 @@ { "name": "hackernews.ndjson_1M", + "type": "bench", "run_count": 3, "extra_cli_args": [], "assets": { diff --git a/workloads/movies-nothreads.json b/workloads/movies-nothreads.json index d3dd006fa..6e35fec20 100644 --- a/workloads/movies-nothreads.json +++ b/workloads/movies-nothreads.json @@ -1,5 +1,6 @@ { "name": "movies.json,no-threads", + "type": "bench", "run_count": 2, "extra_cli_args": [ "--max-indexing-threads=1" diff --git a/workloads/movies.json b/workloads/movies.json index 445ff3aca..337a9856a 100644 --- a/workloads/movies.json +++ b/workloads/movies.json @@ -1,5 +1,6 @@ { "name": "movies.json", + "type": "bench", "run_count": 10, "extra_cli_args": [], "assets": { diff --git a/workloads/search/embeddings-movies-subset-hf.json b/workloads/search/embeddings-movies-subset-hf.json index 720d41790..5d70c933e 100644 --- a/workloads/search/embeddings-movies-subset-hf.json +++ b/workloads/search/embeddings-movies-subset-hf.json @@ -1,5 +1,6 @@ { "name": "search-movies-subset-hf-embeddings", + "type": "bench", "run_count": 2, "target": "search::=trace", "extra_cli_args": [ diff --git a/workloads/search/filterable-movies.json b/workloads/search/filterable-movies.json index 381d80f19..12ec49aee 100644 --- a/workloads/search/filterable-movies.json +++ b/workloads/search/filterable-movies.json @@ -1,5 +1,6 @@ { "name": "search-filterable-movies.json", + "type": "bench", "run_count": 10, "target": "search::=trace", "extra_cli_args": [], diff --git a/workloads/search/geosort.json b/workloads/search/geosort.json index 00788f70b..b7fedd560 100644 --- a/workloads/search/geosort.json +++ b/workloads/search/geosort.json @@ -1,6 +1,7 @@ { "name": "search-geosort.jsonl_1M", - "run_count": 3, + "type": "bench", + "run_count": 3, "target": "search::=trace", "extra_cli_args": [], "assets": { diff --git a/workloads/search/hackernews.json b/workloads/search/hackernews.json index 8d0a4b9b7..6a1d1c504 100644 --- a/workloads/search/hackernews.json +++ b/workloads/search/hackernews.json @@ -1,5 +1,6 @@ { "name": "search-hackernews.ndjson_1M", + "type": "bench", "run_count": 3, "target": "search::=trace", "extra_cli_args": [], diff --git a/workloads/search/movies.json b/workloads/search/movies.json index 939fe6111..e0b16242c 100644 --- a/workloads/search/movies.json +++ b/workloads/search/movies.json @@ -1,5 +1,6 @@ { "name": "search-movies.json", + "type": "bench", "run_count": 10, "target": "search::=trace", "extra_cli_args": [], diff --git a/workloads/search/sortable-movies.json b/workloads/search/sortable-movies.json index 5174b0ec1..3f8852884 100644 --- a/workloads/search/sortable-movies.json +++ b/workloads/search/sortable-movies.json @@ -1,5 +1,6 @@ { "name": "search-sortable-movies.json", + "type": "bench", "run_count": 10, "target": "search::=trace", "extra_cli_args": [], diff --git a/workloads/settings-add-remove-filters.json b/workloads/settings-add-remove-filters.json index f017ed960..bb35d69a9 100644 --- a/workloads/settings-add-remove-filters.json +++ b/workloads/settings-add-remove-filters.json @@ -1,5 +1,6 @@ { "name": "settings-add-remove-filters.json", + "type": "bench", "run_count": 5, "extra_cli_args": [ "--max-indexing-threads=4" diff --git a/workloads/settings-proximity-precision.json b/workloads/settings-proximity-precision.json index ac6d98da0..8c017f95d 100644 --- a/workloads/settings-proximity-precision.json +++ b/workloads/settings-proximity-precision.json @@ -1,5 +1,6 @@ { "name": "settings-proximity-precision.json", + "type": "bench", "run_count": 5, "extra_cli_args": [ "--max-indexing-threads=4" diff --git a/workloads/settings-remove-add-swap-searchable.json b/workloads/settings-remove-add-swap-searchable.json index 7f70d1ce8..2401abb56 100644 --- a/workloads/settings-remove-add-swap-searchable.json +++ b/workloads/settings-remove-add-swap-searchable.json @@ -1,5 +1,6 @@ { "name": "settings-remove-add-swap-searchable.json", + "type": "bench", "run_count": 5, "extra_cli_args": [ "--max-indexing-threads=4" diff --git a/workloads/settings-typo.json b/workloads/settings-typo.json index e04135877..bcd1e4a95 100644 --- a/workloads/settings-typo.json +++ b/workloads/settings-typo.json @@ -1,5 +1,6 @@ { "name": "settings-typo.json", + "type": "bench", "run_count": 5, "extra_cli_args": [ "--max-indexing-threads=4"