Fix asset version issues

crates/xtask/src/common/assets.rs

@@ -166,7 +166,14 @@ fn check_sha256(name: &str, asset: &Asset, mut file: std::fs::File) -> anyhow::R
             }
         }
         None => {
-            tracing::warn!(sha256 = file_hash, "Skipping hash for asset {name} that doesn't have one. Please add it to workload file");
+            let msg = match name.starts_with("meilisearch-v") {
+                true => "Please add it to xtask/src/test/versions.rs",
+                false => "Please add it to workload file",
+            };
+            tracing::warn!(
+                sha256 = file_hash,
+                "Skipping hash for asset {name} that doesn't have one. {msg}"
+            );
             true
         }
     })

crates/xtask/src/common/workload.rs

@@ -4,6 +4,7 @@ use crate::{bench::BenchWorkload, test::TestWorkload};
 
 #[derive(Deserialize)]
 #[serde(tag = "type")]
+#[serde(rename_all = "camelCase")]
 pub enum Workload {
     Bench(BenchWorkload),
     Test(TestWorkload),

crates/xtask/src/test/mod.rs

@@ -2,7 +2,6 @@ use std::time::Duration;
 
 use crate::common::{args::CommonArgs, client::Client, logs::setup_logs, workload::Workload};
 use anyhow::{bail, Context};
-use cargo_metadata::semver::Version;
 use clap::Parser;
 
 mod versions;
@@ -16,8 +15,6 @@ pub struct TestDeriveArgs {
     /// Common arguments shared with other commands
     #[command(flatten)]
     common: CommonArgs,
-
-    initial_version: Version,
 }
 
 pub fn run(args: TestDeriveArgs) -> anyhow::Result<()> {
@@ -58,18 +55,10 @@ async fn run_inner(args: TestDeriveArgs) -> anyhow::Result<()> {
 
         match workload.run(&args, &assets_client, &meili_client).await {
             Ok(_) => {
-                println!(
+                println!("✅ Workload {} completed successfully", workload.name,);
-                    "✅ Workload {} from file {} completed successfully",
-                    workload.name,
-                    workload_file.display()
-                );
             }
             Err(error) => {
-                println!(
+                println!("❌ Workload {} failed: {error}", workload.name,);
-                    "❌ Workload {} from file {} failed: {error}",
-                    workload.name,
-                    workload_file.display()
-                );
                 return Err(error);
             }
         }

crates/xtask/src/test/versions.rs

@@ -6,10 +6,15 @@ use cargo_metadata::semver::Version;
 use serde::Deserialize;
 
 async fn get_sha256(version: &Version, asset_name: &str) -> anyhow::Result<String> {
+    // If version is lower than 1.15 there is no point in trying to get the sha256, GitHub didn't support it
+    if *version < Version::parse("1.15.0")? {
+        anyhow::bail!("version is lower than 1.15, sha256 not available");
+    }
+
     #[derive(Deserialize)]
     struct GithubReleaseAsset {
         name: String,
-        digest: String,
+        digest: Option<String>,
     }
 
     #[derive(Deserialize)]
@@ -19,14 +24,20 @@ async fn get_sha256(version: &Version, asset_name: &str) -> anyhow::Result<Strin
 
     let url =
         format!("https://api.github.com/repos/meilisearch/meilisearch/releases/tags/v{version}");
-    let data: GithubRelease = reqwest::get(url).await?.json().await?;
+    let client = reqwest::Client::builder()
+        .user_agent("Meilisearch bench xtask")
+        .build()
+        .context("failed to build reqwest client")?;
+    let body = client.get(url).send().await?.text().await?;
+    let data: GithubRelease = serde_json::from_str(&body)?;
 
     let digest = data
         .assets
         .into_iter()
         .find(|asset| asset.name.as_str() == asset_name)
         .with_context(|| format!("asset {asset_name} not found in release v{version}"))?
-        .digest;
+        .digest
+        .with_context(|| format!("asset {asset_name} has no digest"))?;
 
     let sha256 =
         digest.strip_prefix("sha256:").map(|s| s.to_string()).context("invalid sha256 format")?;
@@ -79,12 +90,19 @@ async fn add_asset(assets: &mut BTreeMap<String, Asset>, version: &Version) -> a
     let filename = format!("meilisearch-{arch}");
 
     // Try to get the sha256 but it may fail if Github is rate limiting us
-    let sha256 = match get_sha256(version, &filename).await {
+    // We hardcode some values to speed up tests and avoid hitting Github
-        Ok(sha256) => Some(sha256),
+    // Also, versions prior to 1.15 don't have sha256 available anyway
-        Err(err) => {
+    let sha256 = match local_filename.as_str() {
-            eprintln!("⚠️ Warning: could not get sha256 from GitHub: {err}. Proceeding without integrity check.");
+        "meilisearch-1.12.0-macos-apple-silicon" => {
-            None
+            Some(String::from("3b384707a5df9edf66f9157f0ddb70dcd3ac84d4887149169cf93067d06717b7"))
         }
+        _ => match get_sha256(version, &filename).await {
+            Ok(sha256) => Some(sha256),
+            Err(err) => {
+                tracing::warn!("failed to get sha256 for version {version}: {err}");
+                None
+            }
+        },
     };
 
     let url = format!(

crates/xtask/src/test/workload.rs

@@ -48,8 +48,10 @@ enum CommandOrUpgradeVec {
 /// A test workload.
 /// Not to be confused with [a bench workload](crate::bench::workload::Workload).
 #[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
 pub struct TestWorkload {
     pub name: String,
+    pub initial_version: Version,
     pub assets: BTreeMap<String, Asset>,
     pub commands: Vec<CommandOrUpgrade>,
 }
@@ -64,7 +66,7 @@ impl TestWorkload {
         // Group commands between upgrades
         let mut commands_or_upgrade = Vec::new();
         let mut current_commands = Vec::new();
-        let mut all_versions = vec![args.initial_version.clone()];
+        let mut all_versions = vec![self.initial_version.clone()];
         for command_or_upgrade in &self.commands {
             match command_or_upgrade {
                 CommandOrUpgrade::Command(command) => current_commands.push(command.clone()),

workloads/tests/movies.json

@@ -0,0 +1,13 @@
+{
+  "name": "movies",
+  "type": "test",
+  "initialVersion": "1.12.0",
+  "assets": {
+    "movies.json": {
+      "local_location": null,
+      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/movies.json",
+      "sha256": "5b6e4cb660bc20327776e8a33ea197b43d9ec84856710ead1cc87ab24df77de1"
+    }
+  },
+  "commands": []
+}