Sausage: host, broker, commit-on-close filesystem, cargo actor, console, agent

Imported from the spike branch, with:
- a library crate (Host API) for the CLI and the tests
- JSON test scenarios for candidates
- commit on close only for new or truncated files
- a Nix flake: nix run -- init, nix run -- run

cli/src/cargo_actor.rs

@@ -0,0 +1,169 @@
+//! The native `cargo` actor: builds Rust source folders into `.wasm` files.
+//!
+//! Request:  `{"type": "build", "source": "/actor/<id>/public/<folder>"}`
+//! Replies:  `{"type": "built", "wasm": "/actor/cargo/public/builds/<n>/<name>.wasm", ...}`
+//!           `{"type": "build_failed", "errors": "..."}`
+//!
+//! The broker knows nothing about Rust. This actor is one way to get a
+//! `.wasm` file. Other actors can add other languages.
+
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use anyhow::{Context, Result, anyhow, bail};
+use serde_json::{Value, json};
+use tokio::process::Command;
+use tokio::sync::{Mutex, mpsc};
+
+use crate::broker::{Broker, Envelope, truncate};
+use crate::world::{self, World};
+
+pub const ID: &str = "cargo";
+pub const TARGET: &str = "wasm32-wasip2";
+const BUILD_TIMEOUT: Duration = Duration::from_secs(900);
+
+pub struct BuildOutput {
+    /// Host path of the published `.wasm` file.
+    pub wasm: PathBuf,
+    pub seconds: f64,
+    pub warnings: String,
+}
+
+pub struct Cargo {
+    world: World,
+    /// One build at a time: all builds share one target folder.
+    lock: Mutex<()>,
+}
+
+impl Cargo {
+    pub fn new(world: World) -> Self {
+        Self { world, lock: Mutex::new(()) }
+    }
+
+    /// Build the crate in `source` (a host path) and publish the `.wasm` file
+    /// under `/actor/cargo/public/builds/<n>/`.
+    pub async fn build(&self, source: &Path) -> Result<BuildOutput> {
+        if !source.join("Cargo.toml").is_file() {
+            bail!("{} has no Cargo.toml", source.display());
+        }
+        let _guard = self.lock.lock().await;
+        let started = Instant::now();
+        let data = self.world.data_dir(ID);
+        let key = format!("{:016x}", fxhash(source.to_string_lossy().as_bytes()));
+        let build_dir = data.join("work").join(&key);
+        world::replace_tree(source, &build_dir)?;
+        // The crate must not join a parent workspace by accident.
+        let manifest = build_dir.join("Cargo.toml");
+        let text = std::fs::read_to_string(&manifest)?;
+        if !text.contains("[workspace]") {
+            std::fs::write(&manifest, format!("{text}\n[workspace]\n"))?;
+        }
+
+        let mut cmd = Command::new("cargo");
+        cmd.arg("build")
+            .arg("--release")
+            .arg("--target")
+            .arg(TARGET)
+            .arg("--message-format=json-render-diagnostics")
+            .arg("--target-dir")
+            .arg(data.join("target"))
+            .current_dir(&build_dir)
+            .kill_on_drop(true);
+        let out = tokio::time::timeout(BUILD_TIMEOUT, cmd.output())
+            .await
+            .map_err(|_| anyhow!("cargo build took longer than {BUILD_TIMEOUT:?}"))?
+            .context("cannot run cargo (is it on PATH, with the wasm32-wasip2 target?)")?;
+        let stderr = String::from_utf8_lossy(&out.stderr).to_string();
+        if !out.status.success() {
+            bail!("{}", truncate(&strip_progress(&stderr), 20_000));
+        }
+        let mut artifact = None;
+        for line in String::from_utf8_lossy(&out.stdout).lines() {
+            if let Ok(v) = serde_json::from_str::<Value>(line)
+                && v["reason"] == "compiler-artifact"
+                && let Some(exe) = v["executable"].as_str()
+            {
+                artifact = Some(PathBuf::from(exe));
+            }
+        }
+        let artifact = artifact.ok_or_else(|| anyhow!("cargo built no binary. Does the crate have a src/main.rs?"))?;
+
+        let n = next_build_number(&self.world.public_dir(ID).join("builds"))?;
+        let dest_dir = self.world.public_dir(ID).join("builds").join(n.to_string());
+        let name = artifact.file_name().context("artifact has no file name")?;
+        let dest = dest_dir.join(name);
+        world::write_atomic(&dest, &std::fs::read(&artifact)?)?;
+        Ok(BuildOutput { wasm: dest, seconds: started.elapsed().as_secs_f64(), warnings: strip_progress(&stderr) })
+    }
+
+    pub async fn run_actor(self: Arc<Self>, broker: Arc<Broker>, mut mailbox: mpsc::Receiver<Envelope>) {
+        while let Some(env) = mailbox.recv().await {
+            let this = self.clone();
+            let broker = broker.clone();
+            tokio::spawn(async move {
+                let body = this.handle(&broker, &env).await.unwrap_or_else(|e| {
+                    json!({"type": "build_failed", "source": env.body.get("source"), "errors": format!("{e:#}")})
+                });
+                broker.send(ID, Some(&env.from), None, Some(&env.id), body);
+            });
+        }
+    }
+
+    async fn handle(&self, broker: &Broker, env: &Envelope) -> Result<Value> {
+        match env.body.get("type").and_then(Value::as_str) {
+            Some("build") => {
+                let source_s = env.body.get("source").and_then(Value::as_str).ok_or_else(|| anyhow!("missing \"source\""))?;
+                let source = broker.world.resolve(&env.from, source_s)?;
+                let out = self.build(&source).await?;
+                let wasm = self.world.global_path(&out.wasm).unwrap_or_else(|| out.wasm.display().to_string());
+                Ok(json!({
+                    "type": "built",
+                    "wasm": wasm,
+                    "source": source_s,
+                    "built_by": ID,
+                    "seconds": (out.seconds * 10.0).round() / 10.0,
+                    "warnings": truncate(&out.warnings, 4000),
+                }))
+            }
+            Some("shutdown") => Ok(json!({"type": "ignored"})),
+            other => bail!("unknown request {other:?}. Known: build"),
+        }
+    }
+}
+
+fn next_build_number(dir: &Path) -> Result<u64> {
+    std::fs::create_dir_all(dir)?;
+    let max = std::fs::read_dir(dir)?
+        .filter_map(|e| e.ok()?.file_name().to_str()?.parse::<u64>().ok())
+        .max()
+        .unwrap_or(0);
+    Ok(max + 1)
+}
+
+/// Remove cargo's "Compiling ..." progress lines. Keep diagnostics.
+fn strip_progress(stderr: &str) -> String {
+    stderr
+        .lines()
+        .filter(|l| {
+            let t = l.trim_start();
+            !(t.starts_with("Compiling ")
+                || t.starts_with("Downloaded ")
+                || t.starts_with("Downloading ")
+                || t.starts_with("Updating ")
+                || t.starts_with("Locking ")
+                || t.starts_with("Adding ")
+                || t.starts_with("Blocking "))
+        })
+        .collect::<Vec<_>>()
+        .join("\n")
+}
+
+fn fxhash(bytes: &[u8]) -> u64 {
+    let mut h: u64 = 0xcbf29ce484222325;
+    for b in bytes {
+        h ^= *b as u64;
+        h = h.wrapping_mul(0x100000001b3);
+    }
+    h
+}