//! Tests for a candidate version, before the broker installs it. //! //! A test is a JSON file in `source/tests/*.json`, one scenario per file: //! //! ```json //! { //! "description": "add, then list", //! "steps": [ //! {"send": {"type": "add", "text": "milk"}, "expect": {"type": "added"}}, //! {"send": {"type": "list"}, "from": "someone", "expect": [{"type": "notes"}], "to": "someone"} //! ] //! } //! ``` //! //! Each step sends `send` (optional, from `from` or "test"), then reads one //! message for each entry of `expect` (an object or a list). An expected body //! matches if it is a subset of the actual body. With `to`, each message must //! also go to that recipient. //! //! Each file runs in a fresh instance on a copy of the actor's `/data` and //! `/public` (empty folders for a new actor), without network. A version //! without test files passes. use std::path::Path; use std::time::Duration; use anyhow::{Result, anyhow, bail}; use serde_json::{Value, json}; use wasmtime::component::Component; use crate::broker::{Broker, SHUTDOWN_DEADLINE, shutdown_line, truncate}; use crate::runtime::Instance; use crate::world; const EXPECT_TIMEOUT: Duration = Duration::from_secs(10); /// `expected` matches `actual` if every key of an expected object exists in /// the actual object with a matching value. Other values must be equal. pub fn subset_match(expected: &Value, actual: &Value) -> bool { match (expected, actual) { (Value::Object(e), Value::Object(a)) => e.iter().all(|(k, v)| a.get(k).is_some_and(|av| subset_match(v, av))), (Value::Array(e), Value::Array(a)) => e.len() == a.len() && e.iter().zip(a).all(|(x, y)| subset_match(x, y)), _ => expected == actual, } } struct Scratch { dir: std::path::PathBuf, } impl Drop for Scratch { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.dir); } } fn stderr_tail(path: &Path) -> String { let text = std::fs::read_to_string(path).unwrap_or_default(); let start = text.len().saturating_sub(2000); let mut s = start; while !text.is_char_boundary(s) { s += 1; } text[s..].to_string() } /// The version under test, which starts from a copy of the actor's state. struct Candidate<'a> { broker: &'a Broker, id: &'a str, version: u64, component: &'a Component, } /// Run every test file of `/tests/`. pub async fn test(broker: &Broker, id: &str, version: u64, component: &Component, source: Option<&Path>) -> Result { let candidate = Candidate { broker, id, version, component }; let Some(source) = source else { return Ok(json!({"passed": []})) }; let mut files: Vec<_> = std::fs::read_dir(source.join("tests")) .map(|rd| rd.filter_map(|e| Some(e.ok()?.path())).filter(|p| p.extension().is_some_and(|x| x == "json")).collect()) .unwrap_or_default(); files.sort(); let mut passed = Vec::new(); for file in &files { let name = file.file_name().unwrap().to_string_lossy().to_string(); candidate.run_file(file).await.map_err(|e| anyhow!("test {name} failed: {e:#}"))?; passed.push(name); } Ok(json!({"passed": passed})) } impl Candidate<'_> { /// Start an instance on a fresh copy of the state, without network. fn start(&self) -> Result<(Instance, Scratch)> { static N: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); let n = N.fetch_add(1, std::sync::atomic::Ordering::Relaxed); let dir = self.broker.world.scratch_dir().join(format!("{}-{}-{n}", self.id, std::process::id())); let scratch = Scratch { dir: dir.clone() }; let world = &self.broker.world; for (from, name) in [(world.data_dir(self.id), "data"), (world.public_dir(self.id), "public")] { std::fs::create_dir_all(dir.join(name))?; if from.is_dir() { world::copy_tree(&from, &dir.join(name))?; } } let cfg = self.broker.instance_config(self.id, self.version, Some(&dir)); Ok((self.broker.runtime.start(self.component, cfg)?, scratch)) } async fn run_file(&self, file: &Path) -> Result<()> { let test: Value = serde_json::from_str(&std::fs::read_to_string(file)?).map_err(|e| anyhow!("invalid JSON: {e}"))?; let steps = test["steps"].as_array().ok_or_else(|| anyhow!("missing \"steps\" list"))?; let (mut inst, scratch) = self.start()?; let stderr = scratch.dir.join("stderr.log"); // Lines that the candidate wrote while a step was sent. let mut early = std::collections::VecDeque::new(); for (n, step) in steps.iter().enumerate() { let n = n + 1; if let Some(body) = step.get("send") { let from = step["from"].as_str().unwrap_or("test"); let env = json!({"id": format!("test/{n}"), "from": from, "body": body}); inst.send_line(&env.to_string(), |line| early.push_back(line)) .await .map_err(|e| anyhow!("step {n}: candidate does not read stdin: {e}"))?; } let expected = match step.get("expect") { None => vec![], Some(Value::Array(list)) => list.clone(), Some(one) => vec![one.clone()], }; for expected in expected { let next = async { if early.is_empty() { inst.stdout.recv().await } else { early.pop_front() } }; let line = match tokio::time::timeout(EXPECT_TIMEOUT, next).await { Err(_) => bail!("step {n}: no message within {EXPECT_TIMEOUT:?}; expected {expected}"), Ok(None) => { let exit = inst.wait().await; bail!("step {n}: candidate ended ({exit}); expected {expected}. stderr:\n{}", stderr_tail(&stderr)) } Ok(Some(l)) => l, }; let msg: Value = serde_json::from_str(&line) .map_err(|_| anyhow!("step {n}: candidate wrote a non-JSON line: {}", truncate(&line, 300)))?; if !subset_match(&expected, &msg["body"]) { bail!("step {n}: expected body {expected}, got {}", truncate(&msg["body"].to_string(), 1000)); } if let Some(to) = step.get("to") && msg.get("to") != Some(to) { bail!("step {n}: expected recipient {to}, got {}", msg.get("to").unwrap_or(&Value::Null)); } } } inst.stop(&shutdown_line(), SHUTDOWN_DEADLINE, |_| {}).await; Ok(()) } }