Modify existing workload files to use precommands

- logs to stderr instead of stdout
- prints links to the dashboard when there is a dashboard

workloads/hackernews.json

@@ -54,7 +54,7 @@
       "sha256": "27e25efd0b68b159b8b21350d9af76938710cb29ce0393fa71b41c4f3c630ffe"
     }
   },
-  "commands": [
+  "precommands": [
     {
       "route": "indexes/movies/settings",
       "method": "PATCH",
@@ -78,8 +78,10 @@
           ]
         }
       },
-      "synchronous": "DontWait"
-    },
+      "synchronous": "WaitForTask"
+    }
+  ],
+  "commands": [
     {
       "route": "indexes/movies/documents",
       "method": "POST",

workloads/movies-nothreads.json

@@ -11,7 +11,7 @@
       "sha256": "5b6e4cb660bc20327776e8a33ea197b43d9ec84856710ead1cc87ab24df77de1"
     }
   },
-  "commands": [
+  "precommands": [
     {
       "route": "indexes/movies/settings",
       "method": "PATCH",
@@ -30,8 +30,10 @@
           ]
         }
       },
-      "synchronous": "DontWait"
-    },
+      "synchronous": "WaitForTask"
+    }
+  ],
+  "commands": [
     {
       "route": "indexes/movies/documents",
       "method": "POST",

workloads/movies-subset-hf-embeddings.json

@@ -11,7 +11,7 @@
       "sha256": "d215e395e4240f12f03b8f1f68901eac82d9e7ded5b462cbf4a6b8efde76c6c6"
     }
   },
-  "commands": [
+  "precommands": [
     {
       "route": "experimental-features",
       "method": "PATCH",
@@ -55,7 +55,9 @@
         }
       },
       "synchronous": "WaitForTask"
-    },
+    }
+  ],
+  "commands": [
     {
       "route": "indexes/movies/documents",
       "method": "POST",

workloads/settings-add-embeddings.json

@@ -11,7 +11,7 @@
       "sha256": "d215e395e4240f12f03b8f1f68901eac82d9e7ded5b462cbf4a6b8efde76c6c6"
     }
   },
-  "commands": [
+  "precommands": [
     {
       "route": "experimental-features",
       "method": "PATCH",
@@ -49,7 +49,9 @@
         "asset": "movies-100.json"
       },
       "synchronous": "WaitForTask"
-    },
+    }
+  ],
+  "commands": [
     {
       "route": "indexes/movies/settings",
       "method": "PATCH",

workloads/settings-add-remove-filters.json

@@ -11,7 +11,7 @@
       "sha256": "28c359a0956958af0ba204ec11bad3045a0864a10b4838914fea25a01724f84b"
     }
   },
-  "commands": [
+  "precommands": [
     {
       "route": "indexes/peoples/settings",
       "method": "PATCH",
@@ -59,7 +59,9 @@
         "asset": "150k-people.json"
       },
       "synchronous": "WaitForTask"
-    },
+    }
+  ],
+  "commands": [
     {
       "route": "indexes/peoples/settings",
       "method": "PATCH",

workloads/settings-proximity-precision.json

@@ -11,7 +11,7 @@
       "sha256": "28c359a0956958af0ba204ec11bad3045a0864a10b4838914fea25a01724f84b"
     }
   },
-  "commands": [
+  "precommands": [
     {
       "route": "indexes/peoples/settings",
       "method": "PATCH",
@@ -61,7 +61,9 @@
         "asset": "150k-people.json"
       },
       "synchronous": "WaitForTask"
-    },
+    }
+  ],
+  "commands": [
     {
       "route": "indexes/peoples/settings",
       "method": "PATCH",

workloads/settings-remove-add-swap-searchable.json

@@ -11,7 +11,7 @@
       "sha256": "28c359a0956958af0ba204ec11bad3045a0864a10b4838914fea25a01724f84b"
     }
   },
-  "commands": [
+  "precommands": [
     {
       "route": "indexes/peoples/settings",
       "method": "PATCH",
@@ -61,7 +61,9 @@
         "asset": "150k-people.json"
       },
       "synchronous": "WaitForTask"
-    },
+    }
+  ],
+  "commands": [
     {
       "route": "indexes/peoples/settings",
       "method": "PATCH",

workloads/settings-typo.json

@@ -11,7 +11,7 @@
       "sha256": "28c359a0956958af0ba204ec11bad3045a0864a10b4838914fea25a01724f84b"
     }
   },
-  "commands": [
+  "precommands": [
     {
       "route": "indexes/peoples/settings",
       "method": "PATCH",
@@ -62,14 +62,18 @@
         "asset": "150k-people.json"
       },
       "synchronous": "WaitForTask"
-    },
+    }
+  ],
+  "commands": [
     {
       "route": "indexes/peoples/settings",
       "method": "PATCH",
       "body": {
         "inline": {
           "typoTolerance": {
-            "disableOnAttributes": ["featured_job_organization_name"]
+            "disableOnAttributes": [
+              "featured_job_organization_name"
+            ]
           }
         }
       },
@@ -93,7 +97,22 @@
       "body": {
         "inline": {
           "typoTolerance": {
-            "disableOnWords": ["Ben","Elowitz","Kevin","Flaherty", "Ron", "Dustin", "Owen", "Chris", "Mark", "Matt", "Peter", "Van", "Head", "of"]
+            "disableOnWords": [
+              "Ben",
+              "Elowitz",
+              "Kevin",
+              "Flaherty",
+              "Ron",
+              "Dustin",
+              "Owen",
+              "Chris",
+              "Mark",
+              "Matt",
+              "Peter",
+              "Van",
+              "Head",
+              "of"
+            ]
           }
         }
       },

xtask/src/bench/client.rs

@@ -55,6 +55,10 @@ impl Client {
     pub fn delete(&self, route: &str) -> reqwest::RequestBuilder {
         self.request(reqwest::Method::DELETE, route)
     }
+
+    pub fn base_url(&self) -> Option<&str> {
+        self.base_url.as_deref()
+    }
 }
 
 #[derive(Debug, Clone, Copy, Deserialize)]

xtask/src/bench/dashboard.rs

@@ -18,12 +18,9 @@ pub enum DashboardClient {
 }
 
 impl DashboardClient {
-    pub fn new(dashboard_url: &str, api_key: Option<&str>) -> anyhow::Result<Self> {
-        let dashboard_client = Client::new(
-            Some(format!("{}/api/v1", dashboard_url)),
-            api_key,
-            Some(std::time::Duration::from_secs(60)),
-        )?;
+    pub fn new(dashboard_url: String, api_key: Option<&str>) -> anyhow::Result<Self> {
+        let dashboard_client =
+            Client::new(Some(dashboard_url), api_key, Some(std::time::Duration::from_secs(60)))?;
 
         Ok(Self::Client(dashboard_client))
     }
@@ -36,7 +33,7 @@ impl DashboardClient {
         let Self::Client(dashboard_client) = self else { return Ok(()) };
 
         let response = dashboard_client
-            .put("machine")
+            .put("/api/v1/machine")
             .json(&json!({"hostname": env.hostname}))
             .send()
             .await
@@ -62,7 +59,7 @@ impl DashboardClient {
         let Self::Client(dashboard_client) = self else { return Ok(Uuid::now_v7()) };
 
         let response = dashboard_client
-            .put("invocation")
+            .put("/api/v1/invocation")
             .json(&json!({
                 "commit": {
                     "sha1": build_info.commit_sha1,
@@ -97,7 +94,7 @@ impl DashboardClient {
         let Self::Client(dashboard_client) = self else { return Ok(Uuid::now_v7()) };
 
         let response = dashboard_client
-            .put("workload")
+            .put("/api/v1/workload")
             .json(&json!({
                 "invocation_uuid": invocation_uuid,
                 "name": &workload.name,
@@ -124,7 +121,7 @@ impl DashboardClient {
         let Self::Client(dashboard_client) = self else { return Ok(()) };
 
         let response = dashboard_client
-            .put("run")
+            .put("/api/v1/run")
             .json(&json!({
                 "workload_uuid": workload_uuid,
                 "data": report
@@ -159,7 +156,7 @@ impl DashboardClient {
     pub async fn mark_as_failed(&self, invocation_uuid: Uuid, failure_reason: Option<String>) {
         if let DashboardClient::Client(client) = self {
             let response = client
-                .post("cancel-invocation")
+                .post("/api/v1/cancel-invocation")
                 .json(&json!({
                     "invocation_uuid": invocation_uuid,
                     "failure_reason": failure_reason,
@@ -186,4 +183,28 @@ impl DashboardClient {
 
         tracing::warn!(%invocation_uuid, "marked invocation as failed or canceled");
     }
+
+    /// Result URL in markdown
+    pub(crate) fn result_url(
+        &self,
+        workload_name: &str,
+        build_info: &build_info::BuildInfo,
+        baseline_branch: &str,
+    ) -> String {
+        let Self::Client(client) = self else { return Default::default() };
+        let Some(base_url) = client.base_url() else { return Default::default() };
+
+        let Some(commit_sha1) = build_info.commit_sha1 else { return Default::default() };
+
+        // https://bench.meilisearch.dev/view_spans?commit_sha1=500ddc76b549fb9f1af54b2dd6abfa15960381bb&workload_name=settings-add-remove-filters.json&target_branch=reduce-transform-disk-usage&baseline_branch=main
+        let mut url = format!(
+            "{base_url}/view_spans?commit_sha1={commit_sha1}&workload_name={workload_name}"
+        );
+
+        if let Some(target_branch) = build_info.branch {
+            url += &format!("&target_branch={target_branch}&baseline_branch={baseline_branch}");
+        }
+
+        format!("[{workload_name} compared with {baseline_branch}]({url})")
+    }
 }

xtask/src/bench/mod.rs

@@ -6,6 +6,7 @@ mod env_info;
 mod meili_process;
 mod workload;
 
+use std::io::LineWriter;
 use std::path::PathBuf;
 
 use anyhow::Context;
@@ -90,6 +91,7 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
 
     let subscriber = tracing_subscriber::registry().with(
         tracing_subscriber::fmt::layer()
+            .with_writer(|| LineWriter::new(std::io::stderr()))
             .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE)
             .with_filter(filter),
     );
@@ -110,7 +112,7 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
     let dashboard_client = if args.no_dashboard {
         dashboard::DashboardClient::new_dry()
     } else {
-        dashboard::DashboardClient::new(&args.dashboard_url, args.api_key.as_deref())?
+        dashboard::DashboardClient::new(args.dashboard_url.clone(), args.api_key.as_deref())?
     };
 
     // reporting uses its own client because keeping the stream open to wait for entries
@@ -136,7 +138,7 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
         let commit_message = build_info.commit_msg.context("missing commit message")?.split('\n').next().unwrap();
         let max_workloads = args.workload_file.len();
         let reason: Option<&str> = args.reason.as_deref();
-        let invocation_uuid = dashboard_client.create_invocation( build_info, commit_message, env, max_workloads, reason).await?;
+        let invocation_uuid = dashboard_client.create_invocation(build_info.clone(), commit_message, env, max_workloads, reason).await?;
 
         tracing::info!(workload_count = args.workload_file.len(), "handling workload files");
 
@@ -144,6 +146,7 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
         let workload_runs = tokio::spawn(
             {
                 let dashboard_client = dashboard_client.clone();
+                let mut dashboard_urls = Vec::new();
                 async move {
             for workload_file in args.workload_file.iter() {
                 let workload: Workload = serde_json::from_reader(
@@ -152,6 +155,8 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
                 )
                 .with_context(|| format!("error parsing {} as JSON", workload_file.display()))?;
 
+                let workload_name = workload.name.clone();
+
                 workload::execute(
                     &assets_client,
                     &dashboard_client,
@@ -163,8 +168,23 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
                     &args,
                 )
                 .await?;
+
+                let result_url = dashboard_client.result_url(&workload_name, &build_info, "main");
+
+                if !result_url.is_empty() {
+                dashboard_urls.push(result_url);
+                }
+
+                if let Some(branch) = build_info.branch {
+                    let result_url = dashboard_client.result_url(&workload_name, &build_info, branch);
+
+
+                    if !result_url.is_empty() {
+                    dashboard_urls.push(result_url);
+                    }
+                }
             }
-            Ok::<(), anyhow::Error>(())
+            Ok::<_, anyhow::Error>(dashboard_urls)
         }});
 
         // handle ctrl-c
@@ -176,13 +196,19 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
 
         // wait for the end of the main task, handle result
         match workload_runs.await {
-            Ok(Ok(_)) => {
+            Ok(Ok(urls)) => {
                 tracing::info!("Success");
+                println!("☀️ Benchmark invocation completed, please find the results for your workloads below:");
+                for url in urls {
+                    println!("- {url}");
+                }
                 Ok::<(), anyhow::Error>(())
             }
             Ok(Err(error)) => {
                 tracing::error!(%invocation_uuid, error = %error, "invocation failed, attempting to report the failure to dashboard");
                 dashboard_client.mark_as_failed(invocation_uuid, Some(error.to_string())).await;
+                println!("☔️ Benchmark invocation failed...");
+                println!("{error}");
                 tracing::warn!(%invocation_uuid, "invocation marked as failed following error");
                 Err(error)
             },
@@ -191,10 +217,20 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
                     Ok(panic) => {
                         tracing::error!("invocation panicked, attempting to report the failure to dashboard");
                         dashboard_client.mark_as_failed( invocation_uuid, Some("Panicked".into())).await;
+                        println!("‼️ Benchmark invocation panicked 😱");
+                        let msg = match panic.downcast_ref::<&'static str>() {
+                            Some(s) => *s,
+                            None => match panic.downcast_ref::<String>() {
+                                Some(s) => &s[..],
+                                None => "Box<dyn Any>",
+                            },
+                        };
+                        println!("panicked at {msg}");
                         std::panic::resume_unwind(panic)
                     }
                     Err(_) => {
                         tracing::warn!("task was canceled");
+                        println!("🚫 Benchmark invocation was canceled");
                         Ok(())
                     }
                 }

xtask/src/bench/workload.rs

@@ -22,6 +22,8 @@ pub struct Workload {
     pub run_count: u16,
     pub extra_cli_args: Vec<String>,
     pub assets: BTreeMap<String, Asset>,
+    #[serde(default)]
+    pub precommands: Vec<super::command::Command>,
     pub commands: Vec<super::command::Command>,
 }
 
@@ -37,6 +39,15 @@ async fn run_commands(
     let report_folder = &args.report_folder;
     let workload_name = &workload.name;
 
+    for batch in workload
+        .precommands
+        .as_slice()
+        .split_inclusive(|command| !matches!(command.synchronous, SyncMode::DontWait))
+    {
+        super::command::run_batch(meili_client, batch, &workload.assets, &args.asset_folder)
+            .await?;
+    }
+
     std::fs::create_dir_all(report_folder)
         .with_context(|| format!("could not create report directory at {report_folder}"))?;