sausage core part 1: a process per call, routing, tree, stop and kill, locks, journal, boot

Checkpoint. Removes the old host. A quality rework follows (modules,
journal writer thread, hash cache, finer tests).

cli/src/cargo.rs

@@ -0,0 +1,308 @@
+//! Rust build helpers for the native cargo plugin.
+
+use std::path::{Path, PathBuf};
+use std::time::Instant;
+
+use anyhow::{Context, Result, anyhow, bail};
+use serde_json::Value;
+use tokio::process::Command;
+
+use include_dir::{Dir, include_dir};
+
+use crate::world::{self, World};
+
+pub const ID: &str = "cargo";
+pub const TARGET: &str = "wasm32-wasip2";
+
+/// The source of the `sausage-actor` SDK.
+static SDK_SRC: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/../sdk/src");
+static SDK_MACROS: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/../sdk/macros/src");
+
+#[derive(Clone, Default, Debug)]
+pub struct TestResults {
+    pub passed: u64,
+    pub failed: u64,
+    pub output: String,
+}
+
+pub struct BuildOutput {
+    /// Host path of the published `.wasm` file.
+    pub wasm: PathBuf,
+    pub seconds: f64,
+    pub warnings: String,
+    pub tests: TestResults,
+}
+
+#[derive(Debug)]
+pub struct BuildFailure {
+    pub reason: String,
+    pub tests: TestResults,
+}
+
+impl std::fmt::Display for BuildFailure {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        if self.tests.failed > 0 && !self.tests.output.is_empty() {
+            write!(f, "{}:\n{}", self.reason, self.tests.output)
+        } else {
+            f.write_str(&self.reason)
+        }
+    }
+}
+
+impl std::error::Error for BuildFailure {}
+
+impl From<anyhow::Error> for BuildFailure {
+    fn from(error: anyhow::Error) -> Self {
+        Self {
+            reason: format!("{error:#}"),
+            tests: TestResults::default(),
+        }
+    }
+}
+
+/// All builds share one target folder, so they run one at a time.
+pub struct Cargo {
+    world: World,
+}
+
+impl Cargo {
+    /// Create the plugin and install the SDK of this host into `public/sdk/`.
+    pub fn new(world: World) -> Result<Self> {
+        let sdk = world.plugin_dir(ID).join("sdk");
+        write_dir(&SDK_SRC, &sdk.join("src"))?;
+        write_dir(&SDK_MACROS, &sdk.join("macros/src"))?;
+        world::write_atomic(
+            &sdk.join("Cargo.toml"),
+            include_bytes!("../../sdk/Cargo.toml"),
+        )?;
+        world::write_atomic(
+            &sdk.join("macros/Cargo.toml"),
+            include_bytes!("../../sdk/macros/Cargo.toml"),
+        )?;
+        Ok(Self { world })
+    }
+
+    /// Run native `cargo test` unless opted out, then build and publish the
+    /// crate's `wasm32-wasip2` binary under `/plugin/cargo/public/builds/<n>/`.
+    pub async fn build(
+        &self,
+        source: &Path,
+        skip_tests: bool,
+    ) -> std::result::Result<BuildOutput, BuildFailure> {
+        if !source.join("Cargo.toml").is_file() {
+            return Err(BuildFailure {
+                reason: format!("{} has no Cargo.toml", source.display()),
+                tests: TestResults::default(),
+            });
+        }
+        let started = Instant::now();
+        let data = self.world.state_dir(ID);
+        let build_dir = data.join("work");
+        world::replace_source(source, &build_dir)?;
+        let manifest = build_dir.join("Cargo.toml");
+        let original = std::fs::read_to_string(&manifest).context("read build manifest")?;
+        let prepared = prepare_manifest(&original, Some(&self.world.plugin_dir(ID).join("sdk")))?;
+        std::fs::write(&manifest, prepared).context("write prepared manifest")?;
+
+        let target_dir = data.join("target");
+        let tests = if skip_tests {
+            TestResults {
+                output: "skipped (skip_tests=true)".into(),
+                ..TestResults::default()
+            }
+        } else {
+            match run_tests(&build_dir, &target_dir).await {
+                Ok(tests) => tests,
+                Err(error) => {
+                    let output = format!("cannot run cargo tests: {error:#}");
+                    return Err(BuildFailure {
+                        reason: output.clone(),
+                        tests: TestResults {
+                            failed: 1,
+                            output,
+                            ..TestResults::default()
+                        },
+                    });
+                }
+            }
+        };
+        if tests.failed > 0 {
+            return Err(BuildFailure {
+                reason: "unit tests failed".into(),
+                tests,
+            });
+        }
+
+        let build_result: Result<BuildOutput> = async {
+            let mut cmd = Command::new("cargo");
+            cmd.arg("build")
+                .arg("--release")
+                .arg("--target")
+                .arg(TARGET)
+                .arg("--message-format=json-render-diagnostics")
+                .arg("--target-dir")
+                .arg(&target_dir)
+                .current_dir(&build_dir)
+                .kill_on_drop(true);
+            let out = cmd
+                .output()
+                .await
+                .context("cannot run cargo (is it on PATH, with the wasm32-wasip2 target?)")?;
+            let stderr = String::from_utf8_lossy(&out.stderr).to_string();
+            if !out.status.success() {
+                bail!("{}", strip_progress(&stderr));
+            }
+            let mut artifact = None;
+            for line in String::from_utf8_lossy(&out.stdout).lines() {
+                if let Ok(v) = serde_json::from_str::<Value>(line)
+                    && v["reason"] == "compiler-artifact"
+                    && let Some(exe) = v["executable"].as_str()
+                {
+                    artifact = Some(PathBuf::from(exe));
+                }
+            }
+            let artifact = artifact.ok_or_else(|| {
+                anyhow!("cargo built no binary. Does the crate have a src/main.rs?")
+            })?;
+
+            let n = next_build_number(&self.world.plugin_dir(ID).join("builds"))?;
+            let dest_dir = self.world.plugin_dir(ID).join("builds").join(n.to_string());
+            let name = artifact.file_name().context("artifact has no file name")?;
+            let dest = dest_dir.join(name);
+            world::write_atomic(&dest, &std::fs::read(&artifact)?)?;
+            Ok(BuildOutput {
+                wasm: dest,
+                seconds: started.elapsed().as_secs_f64(),
+                warnings: strip_progress(&stderr),
+                tests: tests.clone(),
+            })
+        }
+        .await;
+        build_result.map_err(|error| BuildFailure {
+            reason: format!("{error:#}"),
+            tests,
+        })
+    }
+}
+
+/// Run tests for the host triple: WASI crates' pure-logic tests can use the
+/// normal Rust test harness without needing a WASI test runner.
+async fn run_tests(build_dir: &Path, target_dir: &Path) -> Result<TestResults> {
+    let mut cmd = Command::new("cargo");
+    cmd.arg("test")
+        .arg("--target-dir")
+        .arg(target_dir)
+        .current_dir(build_dir)
+        .kill_on_drop(true);
+    let out = cmd.output().await.context("cannot run cargo test")?;
+    let stdout = String::from_utf8_lossy(&out.stdout).to_string();
+    let stderr = String::from_utf8_lossy(&out.stderr).to_string();
+    let combined = match (stdout.is_empty(), stderr.is_empty()) {
+        (true, true) => String::new(),
+        (false, true) => stdout,
+        (true, false) => stderr,
+        (false, false) => format!("{stdout}\n{stderr}"),
+    };
+    let output = strip_progress(&combined);
+    let passed = test_count(&output, "passed");
+    let mut failed = test_count(&output, "failed");
+    // Compilation failures and runner errors have no `test result` summary,
+    // but still make the default test phase fail.
+    if !out.status.success() && failed == 0 {
+        failed = 1;
+    }
+    Ok(TestResults {
+        passed,
+        failed,
+        output,
+    })
+}
+
+/// Count the `passed` or `failed` totals in cargo's per-target test summaries.
+fn test_count(output: &str, field: &str) -> u64 {
+    output
+        .lines()
+        .filter_map(|line| line.split_once("test result:").map(|(_, rest)| rest))
+        .flat_map(|summary| summary.split(';'))
+        .filter_map(|part| {
+            let words: Vec<_> = part.split_whitespace().collect();
+            let index = words.iter().position(|word| *word == field)?;
+            words.get(index.checked_sub(1)?)?.parse::<u64>().ok()
+        })
+        .sum()
+}
+
+/// Point a `sausage-actor` dependency at `sdk` (keeping its features), and
+/// add an empty `[workspace]`, so the crate never joins a parent workspace.
+fn prepare_manifest(text: &str, sdk: Option<&Path>) -> Result<String> {
+    let mut doc: toml_edit::DocumentMut = text.parse()?;
+    if let Some(sdk) = sdk {
+        for section in ["dependencies", "dev-dependencies", "build-dependencies"] {
+            let Some(dep) = doc
+                .get_mut(section)
+                .and_then(|t| t.get_mut("sausage-actor"))
+            else {
+                continue;
+            };
+            let mut table = dep.as_inline_table().cloned().unwrap_or_default();
+            for key in ["version", "path", "git", "branch", "tag", "rev", "registry"] {
+                table.remove(key);
+            }
+            table.insert("path", sdk.display().to_string().into());
+            *dep = toml_edit::value(table);
+        }
+    }
+    if doc.get("workspace").is_none() {
+        doc["workspace"] = toml_edit::table();
+    }
+    Ok(doc.to_string())
+}
+
+/// Write the files of `dir` below `to`. Skip files that did not change, so
+/// that cargo does not rebuild the SDK at every start.
+fn write_dir(dir: &Dir<'_>, to: &Path) -> Result<()> {
+    for file in dir.files() {
+        let dest = to.join(file.path());
+        let contents = match file.path().file_name() {
+            Some(n) if n == "Cargo.toml" => {
+                prepare_manifest(file.contents_utf8().context("Cargo.toml")?, None)?.into_bytes()
+            }
+            _ => file.contents().to_vec(),
+        };
+        if std::fs::read(&dest).ok() != Some(contents.clone()) {
+            std::fs::create_dir_all(dest.parent().context("no parent")?)?;
+            std::fs::write(&dest, contents)?;
+        }
+    }
+    for sub in dir.dirs() {
+        write_dir(sub, to)?;
+    }
+    Ok(())
+}
+
+fn next_build_number(dir: &Path) -> Result<u64> {
+    std::fs::create_dir_all(dir)?;
+    let max = std::fs::read_dir(dir)?
+        .filter_map(|e| e.ok()?.file_name().to_str()?.parse::<u64>().ok())
+        .max()
+        .unwrap_or(0);
+    Ok(max + 1)
+}
+
+/// Remove cargo's "Compiling ..." progress lines. Keep diagnostics.
+fn strip_progress(stderr: &str) -> String {
+    stderr
+        .lines()
+        .filter(|l| {
+            let t = l.trim_start();
+            !(t.starts_with("Compiling ")
+                || t.starts_with("Downloaded ")
+                || t.starts_with("Downloading ")
+                || t.starts_with("Updating ")
+                || t.starts_with("Locking ")
+                || t.starts_with("Adding ")
+                || t.starts_with("Blocking "))
+        })
+        .collect::<Vec<_>>()
+        .join("\n")
+}