From 39e796ee0350ec166c3a8a5df3b04199b5054af0 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 26 Aug 2025 13:39:08 +0200 Subject: [PATCH] Add redaction system --- Cargo.lock | 54 ++++++++++++++++++++++++++++++ crates/xtask/Cargo.toml | 1 + crates/xtask/src/common/command.rs | 42 ++++++++++++++++++++++- crates/xtask/src/test/workload.rs | 28 +++++++++++++++- workloads/tests/movies.json | 4 +-- 5 files changed, 125 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 79fb2086b..67f30b636 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -350,6 +350,21 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78200ac3468a57d333cd0ea5dd398e25111194dcacd49208afca95c629a6311d" +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anes" version = "0.1.6" @@ -1106,6 +1121,20 @@ dependencies = [ "whatlang", ] +[[package]] +name = "chrono" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + [[package]] name = "ciborium" version = "0.2.2" @@ -2851,6 +2880,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.0.0" @@ -7298,6 +7351,7 @@ dependencies = [ "anyhow", "build-info", "cargo_metadata", + "chrono", "clap", "futures-core", "futures-util", diff --git a/crates/xtask/Cargo.toml b/crates/xtask/Cargo.toml index 76751c2ec..fce4a7bed 100644 --- a/crates/xtask/Cargo.toml +++ b/crates/xtask/Cargo.toml @@ -43,3 +43,4 @@ tracing-subscriber = "0.3.19" tracing-trace = { version = "0.1.0", path = "../tracing-trace" } uuid = { version = "1.17.0", features = ["v7", "serde"] } similar-asserts = "1.7.0" +chrono = "0.4" diff --git a/crates/xtask/src/common/command.rs b/crates/xtask/src/common/command.rs index 301bdda57..1ec978134 100644 --- a/crates/xtask/src/common/command.rs +++ b/crates/xtask/src/common/command.rs @@ -158,6 +158,46 @@ async fn wait_for_tasks(client: &Client) -> anyhow::Result<()> { Ok(()) } +fn json_eq_ignore(reference: &Value, value: &Value) -> bool { + match reference { + Value::Null | Value::Bool(_) | Value::Number(_) => reference == value, + Value::String(s) => (s.starts_with('[') && s.ends_with(']')) || reference == value, + Value::Array(values) => match value { + Value::Array(other_values) => { + if values.len() != other_values.len() { + return false; + } + for (value, other_value) in values.iter().zip(other_values.iter()) { + if !json_eq_ignore(value, other_value) { + return false; + } + } + true + } + _ => false, + }, + Value::Object(map) => match value { + Value::Object(other_map) => { + if map.len() != other_map.len() { + return false; + } + for (key, value) in map.iter() { + match other_map.get(key) { + Some(other_value) => { + if !json_eq_ignore(value, other_value) { + return false; + } + } + None => return false, + } + } + true + } + _ => false, + }, + } +} + #[tracing::instrument(skip(client, command, assets, asset_folder), fields(command = %command))] pub async fn run( client: &Client, @@ -222,7 +262,7 @@ pub async fn run( if return_value { return Ok(Some((response, code))); } - if &response != expected_response { + if !json_eq_ignore(expected_response, &response) { let expected_pretty = serde_json::to_string_pretty(expected_response) .context("serializing expected response as pretty JSON")?; let response_pretty = serde_json::to_string_pretty(&response) diff --git a/crates/xtask/src/test/workload.rs b/crates/xtask/src/test/workload.rs index dbdba3631..7249d0084 100644 --- a/crates/xtask/src/test/workload.rs +++ b/crates/xtask/src/test/workload.rs @@ -1,6 +1,8 @@ use anyhow::Context; use cargo_metadata::semver::Version; +use chrono::DateTime; use serde::{Deserialize, Serialize}; +use serde_json::Value; use std::{collections::BTreeMap, sync::Arc}; use crate::{ @@ -29,6 +31,29 @@ enum CommandOrUpgradeVec<'a> { Upgrade(VersionOrLatest), } +fn produce_reference_value(value: &mut Value) { + match value { + Value::Null | Value::Bool(_) | Value::Number(_) => (), + Value::String(string) => { + if DateTime::parse_from_rfc3339(string.as_str()).is_ok() { + *string = String::from("[timestamp]"); + } else if uuid::Uuid::parse_str(string).is_ok() { + *string = String::from("[uuid]"); + } + } + Value::Array(values) => { + for value in values { + produce_reference_value(value); + } + } + Value::Object(map) => { + for (_key, value) in map.iter_mut() { + produce_reference_value(value); + } + }, + } +} + /// A test workload. /// Not to be confused with [a bench workload](crate::bench::workload::Workload). #[derive(Serialize, Deserialize)] @@ -104,11 +129,12 @@ impl TestWorkload { .await?; if return_responses { assert_eq!(responses.len(), cloned.len()); - for (command, (response, status)) in commands.into_iter().zip(responses) { + for (command, (mut response, status)) in commands.into_iter().zip(responses) { if args.update_responses || (dbg!(args.add_missing_responses) && dbg!(command.expected_response.is_none())) { + produce_reference_value(&mut response); command.expected_response = Some(response); command.expected_status = Some(status.as_u16()); } diff --git a/workloads/tests/movies.json b/workloads/tests/movies.json index c5b214984..a2668a7b9 100644 --- a/workloads/tests/movies.json +++ b/workloads/tests/movies.json @@ -30,7 +30,7 @@ }, "expectedStatus": 202, "expectedResponse": { - "enqueuedAt": "2025-08-26T09:31:44.665991Z", + "enqueuedAt": "[timestamp]", "indexUid": "movies", "status": "enqueued", "taskUid": 0, @@ -46,7 +46,7 @@ }, "expectedStatus": 202, "expectedResponse": { - "enqueuedAt": "2025-08-26T09:31:44.732206Z", + "enqueuedAt": "[timestamp]", "indexUid": "movies", "status": "enqueued", "taskUid": 1,