diff --git a/host/src/builder/README.md b/host/src/builder/README.md index bf3595e..2c460ce 100644 --- a/host/src/builder/README.md +++ b/host/src/builder/README.md @@ -8,18 +8,27 @@ For `test` and `run`, arguments after `--` belong to the executable, not the hos Other commands retain their `--` arguments, such as `fmt -- --check` and Clippy lint options. The host finds the Cargo workspace above the requested directory and copies its sources through the shared VFS. -It excludes `target` and `.git`. Each workspace owns a source copy, target directory, registry, and kache cache. +It excludes `target` and `.git`. + +Each workspace owns a source copy, target directory, registry, and kache cache. These directories live under `/host/builds/`. + Cargo fetches dependencies with network access. The command then runs offline inside bubblewrap namespaces. Only the source copy, target, and private kache cache permit writes. The registry and Nix store remain read-only. HOME and the host network are absent. The PID namespace removes compiler descendants and the private kache daemon after Cargo exits. The builder copies changed and new source files back through the VFS. +It preserves other source files, including concurrent edits in the world. It publishes WASI executables into the world's workspace target directory. After `cargo clean`, it removes the published artifacts that Cargo removed from the native target. -Every target build links the shared cwd-init object, including its numeric exit-code interface. -The command deadline defaults to 30 minutes. A closed response cancels the native command and its process group. +Every target build uses `wasm32-wasip2` and links the shared cwd-init object, including its numeric exit-code interface. +The host rejects other build targets and `--target-dir` overrides. + +The command deadline defaults to 30 minutes. +The host kills timed-out commands before it sends their timeout message. +A closed response cancels the native command and its process group. + The host retains at most 20 GiB of build directories and evicts the oldest workspace first. The Nix wrapper supplies these tools on PATH: diff --git a/host/src/builder/files.rs b/host/src/builder/files.rs index be61326..7e3a2ef 100644 --- a/host/src/builder/files.rs +++ b/host/src/builder/files.rs @@ -1,7 +1,10 @@ //! Workspace discovery and source/artifact copies through the world's shared VFS. use anyhow::{Context, Result}; use sausage_vfs::{Layer, Open}; -use std::path::Path; +use std::{collections::HashMap, path::Path}; + +/// Source hashes before the native command changes its copy. +pub type Snapshot = HashMap; pub async fn read(root: &dyn Layer, path: &str) -> Result> { let file = root @@ -37,8 +40,9 @@ pub async fn workspace(root: &dyn Layer, cwd: &str) -> Result { } nearest.context("no Cargo.toml above cwd") } -pub async fn copy_out(root: &dyn Layer, source: &str, destination: &Path) -> Result<()> { +pub async fn copy_out(root: &dyn Layer, source: &str, destination: &Path) -> Result { std::fs::create_dir_all(destination)?; + let mut before = Snapshot::new(); for entry in root.list(source).await? { if matches!(entry.name.as_str(), "target" | ".git") { continue; @@ -46,12 +50,14 @@ pub async fn copy_out(root: &dyn Layer, source: &str, destination: &Path) -> Res let from = sausage_vfs::resolve(source, &entry.name); let to = destination.join(&entry.name); if entry.directory { - Box::pin(copy_out(root, &from, &to)).await?; + before.extend(Box::pin(copy_out(root, &from, &to)).await?); } else { - std::fs::write(to, read(root, &from).await?)?; + let bytes = read(root, &from).await?; + before.insert(from, openssl::sha::sha256(&bytes)); + std::fs::write(to, bytes)?; } } - Ok(()) + Ok(before) } pub async fn mkdir(root: &dyn Layer, path: &str) -> Result<()> { let mut current = String::new(); @@ -91,11 +97,12 @@ pub async fn write(root: &dyn Layer, path: &str, bytes: Vec) -> Result<()> { } Ok(()) } +/// A snapshot selects changed sources. Without a snapshot, publish only WASM artifacts. pub async fn copy_back( root: &dyn Layer, source: &Path, destination: &str, - artifacts: bool, + before: Option<&Snapshot>, ) -> Result<()> { for entry in std::fs::read_dir(source)? { let entry = entry?; @@ -105,9 +112,12 @@ pub async fn copy_back( } let to = sausage_vfs::resolve(destination, &name); if entry.file_type()?.is_dir() { - Box::pin(copy_back(root, &entry.path(), &to, artifacts)).await?; - } else if !artifacts || entry.path().extension().is_some_and(|e| e == "wasm") { - write(root, &to, std::fs::read(entry.path())?).await?; + Box::pin(copy_back(root, &entry.path(), &to, before)).await?; + } else if before.is_some() || entry.path().extension().is_some_and(|e| e == "wasm") { + let bytes = std::fs::read(entry.path())?; + if before.is_none_or(|before| before.get(&to) != Some(&openssl::sha::sha256(&bytes))) { + write(root, &to, bytes).await?; + } } } Ok(()) diff --git a/host/src/builder/mod.rs b/host/src/builder/mod.rs index bb017aa..a241b49 100644 --- a/host/src/builder/mod.rs +++ b/host/src/builder/mod.rs @@ -59,6 +59,25 @@ impl Builder { .contains(&action), "unsupported Cargo command" ); + let compiles = ["build", "check", "clippy", "test", "run"].contains(&action); + let mut args = request.argv.iter().take_while(|arg| arg.as_str() != "--"); + while let Some(arg) = args.next() { + ensure!( + arg != "--target-dir" && !arg.starts_with("--target-dir="), + "the builder controls the target directory" + ); + let target = if arg == "--target" { + args.next().map(String::as_str) + } else { + arg.strip_prefix("--target=") + }; + if compiles && let Some(target) = target { + ensure!( + target == "wasm32-wasip2", + "the build target is always wasm32-wasip2" + ); + } + } let seconds = request.timeout_secs.unwrap_or(1800); ensure!(seconds > 0, "timeout must be positive"); let deadline = Instant::now() @@ -81,7 +100,7 @@ impl Builder { if source.exists() { std::fs::remove_dir_all(&source)?; } - files::copy_out(self.root.as_ref(), &workspace, &source).await?; + let before = files::copy_out(self.root.as_ref(), &workspace, &source).await?; for path in [&target, ®istry, &cache, &target.join("tmp")] { std::fs::create_dir_all(path)?; } @@ -147,7 +166,7 @@ impl Builder { startup.arg(tools.join("build.nu")); let (startup_code, _) = run(startup, tx, deadline).await?; if startup_code != 0 { - files::copy_back(self.root.as_ref(), &source, &workspace, false).await?; + files::copy_back(self.root.as_ref(), &source, &workspace, Some(&before)).await?; self.evict(20 * 1024 * 1024 * 1024)?; tx.send(Event::Exit { code: startup_code, @@ -238,7 +257,7 @@ impl Builder { } let split = args.iter().position(|a| a == "--").unwrap_or(args.len()); cmd.args(&args[..split]); - if ["build", "check", "clippy", "test", "run"].contains(&action) { + if compiles { cmd.args([ "--target", "wasm32-wasip2", @@ -257,7 +276,7 @@ impl Builder { self.root.as_ref(), &artifacts, &format!("{world_target}/wasm32-wasip2"), - true, + None, ) .await?; } @@ -284,7 +303,7 @@ impl Builder { executables.dedup(); } } - files::copy_back(self.root.as_ref(), &source, &workspace, false).await?; + files::copy_back(self.root.as_ref(), &source, &workspace, Some(&before)).await?; self.evict(20 * 1024 * 1024 * 1024)?; tx.send(Event::Exit { code, executables }).await?; Ok(()) @@ -410,14 +429,18 @@ async fn run( )?; Ok::<_, anyhow::Error>((status.code().unwrap_or(1), out)) }; - let result = tokio::select! { - result = collect => result, + let (result, timed_out) = tokio::select! { + result = collect => (result, false), _ = tx.closed() => anyhow::bail!("caller exited"), - _ = tokio::time::sleep_until(deadline) => { - tx.send(Event::Stderr { bytes: b"cargo: command timed out\n".to_vec() }).await?; - Ok((124, Vec::new())) - } + _ = tokio::time::sleep_until(deadline) => (Ok((124, Vec::new())), true), }; + // Kill the command before a full output channel can delay its timeout message. drop(group); + if timed_out { + tx.send(Event::Stderr { + bytes: b"cargo: command timed out\n".to_vec(), + }) + .await?; + } result } diff --git a/host/src/builder/tests.rs b/host/src/builder/tests.rs index c0a18f7..cc27d26 100644 --- a/host/src/builder/tests.rs +++ b/host/src/builder/tests.rs @@ -33,6 +33,41 @@ async fn discovers_workspace_and_skips_target() -> Result<()> { Ok(()) } #[tokio::test] +async fn copies_only_changed_and_new_sources() -> Result<()> { + let temp = tempfile::tempdir()?; + let world = crate::world::World::open(temp.path())?; + for name in ["unchanged.rs", "changed.rs"] { + files::write( + world.root.as_ref(), + &format!("/project/{name}"), + b"initial".to_vec(), + ) + .await?; + } + let copy = temp.path().join("copy"); + let before = files::copy_out(world.root.as_ref(), "/project", ©).await?; + files::write( + world.root.as_ref(), + "/project/unchanged.rs", + b"concurrent edit".to_vec(), + ) + .await?; + std::fs::write(copy.join("changed.rs"), b"changed by command")?; + std::fs::write(copy.join("Cargo.lock"), b"created by command")?; + files::copy_back(world.root.as_ref(), ©, "/project", Some(&before)).await?; + for (name, expected) in [ + ("unchanged.rs", "concurrent edit"), + ("changed.rs", "changed by command"), + ("Cargo.lock", "created by command"), + ] { + assert_eq!( + files::read(world.root.as_ref(), &format!("/project/{name}")).await?, + expected.as_bytes() + ); + } + Ok(()) +} +#[tokio::test] async fn clean_removes_only_deleted_artifacts() -> Result<()> { let temp = tempfile::tempdir()?; let world = crate::world::World::open(temp.path())?; @@ -69,6 +104,58 @@ fn relative_host_directory_uses_absolute_namespace_paths() -> Result<()> { assert!(builder.directory.is_absolute()); Ok(()) } +#[tokio::test] +async fn target_options_preserve_the_platform_target_and_cache() -> Result<()> { + let temp = tempfile::tempdir()?; + let world = crate::world::World::open(temp.path())?; + let builder = Builder::new(world.root, &world.host_dir)?; + let (tx, _rx) = mpsc::channel(1); + for options in [ + vec!["--target", "aarch64-unknown-linux-gnu"], + vec!["--target=aarch64-unknown-linux-gnu"], + vec!["--target-dir", "/other"], + vec!["--target-dir=/other"], + ] { + let error = builder + .execute( + Request { + argv: std::iter::once("build") + .chain(options) + .map(str::to_string) + .collect(), + cwd: "/".into(), + timeout_secs: None, + }, + &tx, + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("target"), "{error}"); + } + for (action, options) in [ + ("run", vec!["--target", "wasm32-wasip2"]), + ("run", vec!["--target=wasm32-wasip2"]), + ("run", vec!["--", "--target-dir", "/argument"]), + ("tree", vec!["--target", "all"]), + ] { + let error = builder + .execute( + Request { + argv: std::iter::once(action) + .chain(options) + .map(str::to_string) + .collect(), + cwd: "/".into(), + timeout_secs: None, + }, + &tx, + ) + .await + .unwrap_err(); + assert_eq!(error.to_string(), "no Cargo.toml above cwd"); + } + Ok(()) +} #[test] fn test_filter_is_not_a_build_argument() { let args = ["test", "-p", "example", "filter", "--release"].map(str::to_string); @@ -96,6 +183,36 @@ fn disk_cap_removes_workspaces() -> Result<()> { assert_eq!(files::size(&builder.directory)?, 32); Ok(()) } +#[cfg(target_os = "linux")] +#[tokio::test] +async fn timeout_kills_before_output_can_drain() -> Result<()> { + use std::os::fd::{FromRawFd, OwnedFd}; + let (tx, mut rx) = mpsc::channel(1); + let output = tx.clone(); + let mut cmd = Command::new("sh"); + cmd.args(["-c", "echo $$; exec sleep 1000"]); + let task = + tokio::spawn( + async move { run(cmd, &output, Instant::now() + Duration::from_secs(1)).await }, + ); + let Some(Event::Stdout { bytes }) = + tokio::time::timeout(Duration::from_secs(2), rx.recv()).await? + else { + anyhow::bail!("missing native PID"); + }; + let pid: u32 = std::str::from_utf8(&bytes)?.trim().parse()?; + let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) }; + ensure!(fd >= 0, "pidfd_open: {}", std::io::Error::last_os_error()); + let exited = tokio::io::unix::AsyncFd::new(unsafe { OwnedFd::from_raw_fd(fd as i32) })?; + // Fill the output channel, so the timeout message cannot enter it. + tx.send(Event::Stdout { bytes: Vec::new() }).await?; + let _ready = tokio::time::timeout(Duration::from_secs(2), exited.readable()).await??; + assert!(matches!(rx.recv().await, Some(Event::Stdout { .. }))); + assert!(matches!(rx.recv().await, Some(Event::Stderr { bytes }) + if bytes.as_slice() == b"cargo: command timed out\n")); + assert_eq!(task.await??.0, 124); + Ok(()) +} #[tokio::test] async fn closed_receiver_cancels_command() -> Result<()> { let (tx, mut rx) = mpsc::channel(1);