Api key tests

crates/xtask/src/bench/mod.rs

@@ -28,6 +28,10 @@ pub struct BenchDeriveArgs {
     #[command(flatten)]
     common: CommonArgs,
 
+    /// Meilisearch master keys
+    #[arg(long)]
+    pub master_key: Option<String>,
+
     /// URL of the dashboard.
     #[arg(long, default_value_t = default_dashboard_url())]
     dashboard_url: String,
@@ -84,13 +88,13 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
     // Also we don't want any pesky timeout because we don't know how much time it will take to recover the full trace
     let logs_client = Client::new(
         Some("http://127.0.0.1:7700/logs/stream".into()),
-        args.common.master_key.as_deref(),
+        args.master_key.as_deref(),
         None,
     )?;
 
     let meili_client = Arc::new(Client::new(
         Some("http://127.0.0.1:7700".into()),
-        args.common.master_key.as_deref(),
+        args.master_key.as_deref(),
         Some(std::time::Duration::from_secs(args.common.tasks_queue_timeout_secs)),
     )?);
 
@@ -131,7 +135,7 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
                     &logs_client,
                     &meili_client,
                     invocation_uuid,
-                    args.common.master_key.as_deref(),
+                    args.master_key.as_deref(),
                     workload,
                     &args,
                     args.binary_path.as_deref(),

crates/xtask/src/common/args.rs

@@ -18,10 +18,6 @@ pub struct CommonArgs {
     #[arg(value_name = "WORKLOAD_FILE", last = false)]
     pub workload_file: Vec<PathBuf>,
 
-    /// Meilisearch master keys
-    #[arg(long)]
-    pub master_key: Option<String>,
-
     /// Directory to store the remote assets.
     #[arg(long, default_value_t = default_asset_folder())]
     pub asset_folder: String,

crates/xtask/src/common/command.rs

@@ -13,7 +13,7 @@ use crate::common::assets::{fetch_asset, Asset};
 use crate::common::client::{Client, Method};
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
-#[serde(rename_all = "camelCase")]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
 pub struct Command {
     pub route: String,
     pub method: Method,
@@ -25,8 +25,10 @@ pub struct Command {
     pub expected_response: Option<serde_json::Value>,
     #[serde(default, skip_serializing_if = "HashMap::is_empty")]
     pub register: HashMap<String, String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub api_key_variable: Option<String>,
     #[serde(default)]
-    synchronous: SyncMode,
+    pub synchronous: SyncMode,
 }
 
 #[derive(Default, Clone, Serialize, Deserialize, Debug)]
@@ -46,7 +48,7 @@ impl Body {
     pub fn get(
         self,
         assets: &BTreeMap<String, Asset>,
-        registered: HashMap<String, Value>,
+        registered: &HashMap<String, Value>,
         asset_folder: &str,
     ) -> anyhow::Result<Option<(Vec<u8>, &'static str)>> {
         Ok(match self {
@@ -76,7 +78,7 @@ impl Body {
                 }
 
                 if !registered.is_empty() {
-                    insert_variables(&mut body, &registered);
+                    insert_variables(&mut body, registered);
                 }
 
                 Some((
@@ -105,8 +107,8 @@ impl Display for Command {
     }
 }
 
-#[derive(Default, Debug, Clone, Copy, Serialize, Deserialize)]
+#[derive(Default, Debug, Clone, Copy, Serialize, Deserialize, PartialEq)]
-enum SyncMode {
+pub enum SyncMode {
     DontWait,
     #[default]
     WaitForResponse,
@@ -243,7 +245,7 @@ fn json_eq_ignore(reference: &Value, value: &Value) -> bool {
     }
 }
 
-#[tracing::instrument(skip(client, command, assets, asset_folder), fields(command = %command))]
+#[tracing::instrument(skip(client, command, assets, registered, asset_folder), fields(command = %command))]
 pub async fn run(
     client: &Client,
     command: &Command,
@@ -252,14 +254,44 @@ pub async fn run(
     asset_folder: &str,
     return_value: bool,
 ) -> anyhow::Result<Option<(Value, StatusCode)>> {
+    // Try to replace variables in the route
+    let mut route = &command.route;
+    let mut owned_route;
+    if !registered.is_empty() {
+        while let (Some(pos1), Some(pos2)) = (route.find("{{"), route.rfind("}}")) {
+            if pos2 > pos1 {
+                let name = route[pos1 + 2..pos2].trim();
+                if let Some(replacement) = registered.get(name).and_then(|r| r.as_str()) {
+                    let mut new_route = String::new();
+                    new_route.push_str(&route[..pos1]);
+                    new_route.push_str(replacement);
+                    new_route.push_str(&route[pos2 + 2..]);
+                    owned_route = new_route;
+                    route = &owned_route;
+                    continue;
+                }
+            }
+            break;
+        }
+    }
+
     // memtake the body here to leave an empty body in its place, so that command is not partially moved-out
     let body = command
         .body
         .clone()
-        .get(assets, registered, asset_folder)
+        .get(assets, &registered, asset_folder)
         .with_context(|| format!("while getting body for command {command}"))?;
 
-    let request = client.request(command.method.into(), &command.route);
+    let mut request = client.request(command.method.into(), route);
+
+    // Replace the api key
+    if let Some(var_name) = &command.api_key_variable {
+        if let Some(api_key) = registered.get(var_name).and_then(|v| v.as_str()) {
+            request = request.header("Authorization", format!("Bearer {api_key}"));
+        } else {
+            bail!("could not find API key variable '{var_name}' in registered values");
+        }
+    }
 
     let request = if let Some((body, content_type)) = body {
         request.body(body).header(reqwest::header::CONTENT_TYPE, content_type)
@@ -272,31 +304,35 @@ pub async fn run(
 
     let code = response.status();
 
-    if let Some(expected_status) = command.expected_status {
+    if !return_value {
-        if !return_value && code.as_u16() != expected_status {
+        if let Some(expected_status) = command.expected_status {
-            let response = response
+            if code.as_u16() != expected_status {
-                .text()
+                let response = response
+                    .text()
+                    .await
+                    .context("could not read response body as text")
+                    .context("reading response body when checking expected status")?;
+                bail!("unexpected status code: got {}, expected {expected_status}, response body: '{response}'", code.as_u16());
+            }
+        } else if code.is_client_error() {
+            tracing::error!(%command, %code, "error in workload file");
+            let response: serde_json::Value = response
+                .json()
                 .await
-                .context("could not read response body as text")
+                .context("could not deserialize response as JSON")
-                .context("reading response body when checking expected status")?;
+                .context("parsing error in workload file when sending command")?;
-            bail!("unexpected status code: got {}, expected {expected_status}, response body: '{response}'", code.as_u16());
+            bail!(
+                "error in workload file: server responded with error code {code} and '{response}'"
+            )
+        } else if code.is_server_error() {
+            tracing::error!(%command, %code, "server error");
+            let response: serde_json::Value = response
+                .json()
+                .await
+                .context("could not deserialize response as JSON")
+                .context("parsing server error when sending command")?;
+            bail!("server error: server responded with error code {code} and '{response}'")
         }
-    } else if code.is_client_error() {
-        tracing::error!(%command, %code, "error in workload file");
-        let response: serde_json::Value = response
-            .json()
-            .await
-            .context("could not deserialize response as JSON")
-            .context("parsing error in workload file when sending command")?;
-        bail!("error in workload file: server responded with error code {code} and '{response}'")
-    } else if code.is_server_error() {
-        tracing::error!(%command, %code, "server error");
-        let response: serde_json::Value = response
-            .json()
-            .await
-            .context("could not deserialize response as JSON")
-            .context("parsing server error when sending command")?;
-        bail!("server error: server responded with error code {code} and '{response}'")
     }
 
     if let Some(expected_response) = &command.expected_response {
@@ -357,5 +393,6 @@ pub fn health_command() -> Command {
         synchronous: SyncMode::WaitForResponse,
         expected_status: None,
         expected_response: None,
+        api_key_variable: None,
     }
 }

crates/xtask/src/test/mod.rs

@@ -1,7 +1,9 @@
 use std::{sync::Arc, time::Duration};
 
 use crate::{
-    common::{args::CommonArgs, client::Client, logs::setup_logs, workload::Workload},
+    common::{
+        args::CommonArgs, client::Client, command::SyncMode, logs::setup_logs, workload::Workload,
+    },
     test::workload::CommandOrUpgrade,
 };
 use anyhow::{bail, Context};
@@ -49,7 +51,7 @@ async fn run_inner(args: TestDeriveArgs) -> anyhow::Result<()> {
 
     let meili_client = Arc::new(Client::new(
         Some("http://127.0.0.1:7700".into()),
-        args.common.master_key.as_deref(),
+        Some("masterKey"),
         Some(Duration::from_secs(args.common.tasks_queue_timeout_secs)),
     )?);
 
@@ -68,10 +70,17 @@ async fn run_inner(args: TestDeriveArgs) -> anyhow::Result<()> {
         let has_upgrade =
             workload.commands.iter().any(|c| matches!(c, CommandOrUpgrade::Upgrade { .. }));
 
+        let has_faulty_register = workload.commands.iter().any(|c| {
+            matches!(c, CommandOrUpgrade::Command(cmd) if cmd.synchronous == SyncMode::DontWait && !cmd.register.is_empty())
+        });
+        if has_faulty_register {
+            bail!("workload {} contains commands that register values but are marked as --dont-wait. This is not supported because we cannot guarantee the value will be registered before the next command runs.", workload.name);
+        }
+
         let name = workload.name.clone();
         match workload.run(&args, &assets_client, &meili_client, asset_folder).await {
             Ok(_) => {
-                match args.add_missing_responses || args.update_responses {
+                match args.update_responses {
                     true => println!("🛠️ Workload {name} was updated"),
                     false => println!("✅ Workload {name} passed"),
                 }

crates/xtask/src/test/workload.rs

@@ -25,6 +25,7 @@ use crate::{
 
 #[derive(Serialize, Deserialize, Debug)]
 #[serde(untagged)]
+#[allow(clippy::large_enum_variant)]
 pub enum CommandOrUpgrade {
     Command(Command),
     Upgrade { upgrade: VersionOrLatest },
@@ -71,7 +72,11 @@ fn produce_reference_value(value: &mut Value) {
 pub struct TestWorkload {
     pub name: String,
     pub initial_version: Version,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub master_key: Option<String>,
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
     pub assets: BTreeMap<String, Asset>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub commands: Vec<CommandOrUpgrade>,
 }
 
@@ -116,7 +121,7 @@ impl TestWorkload {
             .binary_path(&args.common.asset_folder)?;
         let mut process = process::start_meili(
             meili_client,
-            args.common.master_key.as_deref(),
+            Some("masterKey"),
             &[],
             &self.name,
             binary_path.as_deref(),
@@ -144,8 +149,8 @@ impl TestWorkload {
                         for (command, (mut response, status)) in commands.into_iter().zip(responses)
                         {
                             if args.update_responses
-                                || (dbg!(args.add_missing_responses)
+                                || (args.add_missing_responses
-                                    && dbg!(command.expected_response.is_none()))
+                                    && command.expected_response.is_none())
                             {
                                 produce_reference_value(&mut response);
                                 command.expected_response = Some(response);
@@ -159,7 +164,7 @@ impl TestWorkload {
                     let binary_path = version.binary_path(&args.common.asset_folder)?;
                     process = process::start_meili(
                         meili_client,
-                        args.common.master_key.as_deref(),
+                        Some("masterKey"),
                         &[String::from("--experimental-dumpless-upgrade")],
                         &self.name,
                         binary_path.as_deref(),

workloads/tests/api-keys.json

@@ -0,0 +1,221 @@
+{
+  "type": "test",
+  "name": "api-keys",
+  "initialVersion": "1.12.0",
+  "commands": [
+    {
+      "route": "keys",
+      "method": "POST",
+      "body": {
+        "inline": {
+          "actions": [
+            "search",
+            "documents.add"
+          ],
+          "description": "Test API Key",
+          "expiresAt": null,
+          "indexes": [
+            "movies"
+          ]
+        }
+      },
+      "expectedStatus": 201,
+      "expectedResponse": {
+        "actions": [
+          "search",
+          "documents.add"
+        ],
+        "createdAt": "[timestamp]",
+        "description": "Test API Key",
+        "expiresAt": null,
+        "indexes": [
+          "movies"
+        ],
+        "key": "c6f64630bad2996b1f675007c8800168e14adf5d6a7bb1a400a6d2b158050eaf",
+        "name": null,
+        "uid": "[uuid]",
+        "updatedAt": "[timestamp]"
+      },
+      "register": {
+        "key": "/key"
+      },
+      "synchronous": "WaitForResponse"
+    },
+    {
+      "route": "keys/{{ key }}",
+      "method": "GET",
+      "body": null,
+      "expectedStatus": 200,
+      "expectedResponse": {
+        "actions": [
+          "search",
+          "documents.add"
+        ],
+        "createdAt": "[timestamp]",
+        "description": "Test API Key",
+        "expiresAt": null,
+        "indexes": [
+          "movies"
+        ],
+        "key": "c6f64630bad2996b1f675007c8800168e14adf5d6a7bb1a400a6d2b158050eaf",
+        "name": null,
+        "uid": "[uuid]",
+        "updatedAt": "[timestamp]"
+      },
+      "synchronous": "WaitForResponse"
+    },
+    {
+      "route": "/indexes",
+      "method": "POST",
+      "body": {
+        "inline": {
+          "primaryKey": "id",
+          "uid": "movies"
+        }
+      },
+      "expectedStatus": 202,
+      "expectedResponse": {
+        "enqueuedAt": "[timestamp]",
+        "indexUid": "movies",
+        "status": "enqueued",
+        "taskUid": 0,
+        "type": "indexCreation"
+      },
+      "synchronous": "WaitForResponse"
+    },
+    {
+      "route": "indexes/movies/documents",
+      "method": "POST",
+      "body": {
+        "inline": {
+          "id": 287947,
+          "overview": "A boy is given the ability to become an adult superhero in times of need with a single magic word.",
+          "poster": "https://image.tmdb.org/t/p/w1280/xnopI5Xtky18MPhK40cZAGAOVeV.jpg",
+          "release_date": "2019-03-23",
+          "title": "Shazam"
+        }
+      },
+      "expectedStatus": 202,
+      "expectedResponse": {
+        "enqueuedAt": "[timestamp]",
+        "indexUid": "movies",
+        "status": "enqueued",
+        "taskUid": 1,
+        "type": "documentAdditionOrUpdate"
+      },
+      "apiKeyVariable": "key",
+      "synchronous": "WaitForResponse"
+    },
+    {
+      "route": "indexes/movies/search?q=shazam",
+      "method": "GET",
+      "body": null,
+      "expectedStatus": 200,
+      "expectedResponse": {
+        "estimatedTotalHits": 0,
+        "hits": [],
+        "limit": 20,
+        "offset": 0,
+        "processingTimeMs": "[duration]",
+        "query": "shazam"
+      },
+      "apiKeyVariable": "key",
+      "synchronous": "WaitForResponse"
+    },
+    {
+      "upgrade": "latest"
+    },
+    {
+      "route": "indexes/movies/search?q=shazam",
+      "method": "GET",
+      "body": null,
+      "expectedStatus": 200,
+      "expectedResponse": {
+        "estimatedTotalHits": 1,
+        "hits": [
+          {
+            "id": 287947,
+            "overview": "A boy is given the ability to become an adult superhero in times of need with a single magic word.",
+            "poster": "https://image.tmdb.org/t/p/w1280/xnopI5Xtky18MPhK40cZAGAOVeV.jpg",
+            "release_date": "2019-03-23",
+            "title": "Shazam"
+          }
+        ],
+        "limit": 20,
+        "offset": 0,
+        "processingTimeMs": "[duration]",
+        "query": "shazam"
+      },
+      "apiKeyVariable": "key",
+      "synchronous": "WaitForResponse"
+    },
+    {
+      "route": "indexes/movies/documents/287947",
+      "method": "DELETE",
+      "body": null,
+      "expectedStatus": 403,
+      "expectedResponse": {
+        "code": "invalid_api_key",
+        "link": "https://docs.meilisearch.com/errors#invalid_api_key",
+        "message": "The provided API key is invalid.",
+        "type": "auth"
+      },
+      "apiKeyVariable": "key",
+      "synchronous": "WaitForResponse"
+    },
+    {
+      "route": "indexes/movies/documents",
+      "method": "POST",
+      "body": {
+        "inline": {
+          "id": 287948,
+          "overview": "Shazam turns evil and the world is in danger.",
+          "poster": "https://image.tmdb.org/t/p/w1280/xnopI5Xtky18MPhK40cZAGAOVeV.jpg",
+          "release_date": "2032-03-23",
+          "title": "Shazam 2"
+        }
+      },
+      "expectedStatus": 202,
+      "expectedResponse": {
+        "enqueuedAt": "[timestamp]",
+        "indexUid": "movies",
+        "status": "enqueued",
+        "taskUid": 3,
+        "type": "documentAdditionOrUpdate"
+      },
+      "apiKeyVariable": "key",
+      "synchronous": "WaitForTask"
+    },
+    {
+      "route": "indexes/movies/search?q=shaza",
+      "method": "GET",
+      "body": null,
+      "expectedStatus": 200,
+      "expectedResponse": {
+        "estimatedTotalHits": 2,
+        "hits": [
+          {
+            "id": 287947,
+            "overview": "A boy is given the ability to become an adult superhero in times of need with a single magic word.",
+            "poster": "https://image.tmdb.org/t/p/w1280/xnopI5Xtky18MPhK40cZAGAOVeV.jpg",
+            "release_date": "2019-03-23",
+            "title": "Shazam"
+          },
+          {
+            "id": 287948,
+            "overview": "Shazam turns evil and the world is in danger.",
+            "poster": "https://image.tmdb.org/t/p/w1280/xnopI5Xtky18MPhK40cZAGAOVeV.jpg",
+            "release_date": "2032-03-23",
+            "title": "Shazam 2"
+          }
+        ],
+        "limit": 20,
+        "offset": 0,
+        "processingTimeMs": "[duration]",
+        "query": "shaza"
+      },
+      "apiKeyVariable": "key",
+      "synchronous": "WaitForResponse"
+    }
+  ]
+}