surma/sausage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cargo.build: a persistent build cache shared by all worlds

Browse Source 
- The cargo target folder is <cache>/cargo-target, next to Wasmtime's
  cache, which moves to <cache>/wasmtime: Wasmtime's cleanup worker
  deletes unknown entries in its folder, also Cargo's artifacts.
- Builds use the SDK at <cache>/sdk/<content hash>/, so cargo reuses
  the compiled SDK in every world. /plugins/cargo/sdk/ stays for agents.
- Tests run in the release profile and share compiled crates with the
  release build.
- The cargo and cli tests use a persistent cache in CARGO_TARGET_TMPDIR:
  warm, the cargo suite takes 11 s (was 235 s) and the cli suite 6 s
  (was 149 s).
- nix flake check runs ignored stress tests. AGENTS.md: rules for fast
  tests.
This commit is contained in:
surma
2026-10-23 08:05:15 +01:00
parent a090e1fd57
commit 93f092e02b
11 changed files with 155 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/src/cargo.rs
+56 -18
View File
@@ -3,6 +3,7 @@ use crate::{Host, world::World};
use anyhow::{Context, Result, bail};
use include_dir::{Dir, include_dir};
use serde_json::{Value, json};
use sha2::{Digest, Sha256};
use std::{
path::{Path, PathBuf},
sync::Arc,
@@ -58,10 +59,12 @@ pub struct Cargo {
world: World,
builds: Arc<tokio::sync::Mutex<()>>,
runner: PathBuf,
target: PathBuf,
sdk: PathBuf,
}
impl Cargo {
/// Installs changed SDK files and selects the current sausage executable as runner.
pub fn new(world: World) -> Result<Self> {
pub fn new(world: World, cache: &Path) -> Result<Self> {
let mut runner = std::env::current_exe()?;
if runner
.parent()
@@ -69,28 +72,29 @@ impl Cargo {
{
runner = runner.parent().unwrap().parent().unwrap().join("sausage");
}
Self::with_runner(world, runner)
Self::with_runner(world, runner, cache)
}
/// Installs the SDK and accepts an explicit sausage binary, including for init integrations.
pub fn with_runner(world: World, runner: PathBuf) -> Result<Self> {
pub fn with_runner(world: World, runner: PathBuf, cache: &Path) -> Result<Self> {
let sdk = world.plugin_dir(ID).join("sdk");
write_dir(&SDK, &sdk.join("src"))?;
write_dir(&MACROS, &sdk.join("macros/src"))?;
write_dir(&INTERFACE, &sdk.join("interface/src"))?;
let mut files = std::collections::BTreeMap::new();
collect_files(&SDK, Path::new("src"), &mut files);
collect_files(&MACROS, Path::new("macros/src"), &mut files);
collect_files(&INTERFACE, Path::new("interface/src"), &mut files);
for (manifest, destination, interface_path) in [
(
include_str!("../../sdk/Cargo.toml"),
sdk.join("Cargo.toml"),
PathBuf::from("Cargo.toml"),
Some("interface"),
),
(
include_str!("../../sdk/macros/Cargo.toml"),
sdk.join("macros/Cargo.toml"),
PathBuf::from("macros/Cargo.toml"),
Some("../interface"),
),
(
include_str!("../../interface/Cargo.toml"),
sdk.join("interface/Cargo.toml"),
PathBuf::from("interface/Cargo.toml"),
None,
),
] {
@@ -108,9 +112,40 @@ impl Cargo {
}
}
}
write_changed(&destination, document.to_string().as_bytes())?;
files.insert(destination, document.to_string().into_bytes());
}
let mut hash = Sha256::new();
for (path, bytes) in &files {
let name = path.to_str().unwrap().as_bytes();
hash.update((name.len() as u64).to_le_bytes());
hash.update(name);
hash.update((bytes.len() as u64).to_le_bytes());
hash.update(bytes);
write_changed(&sdk.join(path), bytes)?;
}
let cached = std::path::absolute(cache)?
.join("sdk")
.join(format!("{:x}", hash.finalize()));
if !cached.is_dir() {
let parent = cached.parent().unwrap();
std::fs::create_dir_all(parent)
.with_context(|| format!("create {}", parent.display()))?;
let staging = tempfile::Builder::new()
.prefix("install-")
.tempdir_in(parent)?;
for (path, bytes) in &files {
write_changed(&staging.path().join(path), bytes)?;
}
// Another builder can publish the same complete, content-addressed SDK first.
if let Err(error) = std::fs::rename(staging.path(), &cached)
&& !cached.is_dir()
{
return Err(error).with_context(|| format!("install {}", cached.display()));
}
}
Ok(Self {
sdk: cached,
target: std::path::absolute(cache)?.join("cargo-target"),
world,
runner,
builds: Arc::new(tokio::sync::Mutex::new(())),
@@ -136,12 +171,12 @@ impl Cargo {
pub async fn build(&self, source: &Path) -> std::result::Result<BuildOutput, BuildFailure> {
let build_guard = self.builds.clone().lock_owned().await;
let work = self.world.state_dir(ID).join("work");
let target = self.world.state_dir(ID).join("target");
let target = self.target.clone();
let temporary = self.world.state_dir(ID).join("tmp");
let cleanup = temporary.clone();
let source = source.to_path_buf();
let copy = work.clone();
let sdk = self.world.plugin_dir(ID).join("sdk");
let sdk = self.sdk.clone();
let build_guard = tokio::task::spawn_blocking(move || -> Result<_> {
if cleanup.exists() {
std::fs::remove_dir_all(&cleanup)
@@ -168,7 +203,7 @@ impl Cargo {
.await
.map_err(anyhow::Error::from)??;
let mut test = Command::new("cargo");
test.args(["test", "--target", TARGET, "--target-dir"])
test.args(["test", "--release", "--target", TARGET, "--target-dir"])
.arg(&target)
.current_dir(&work)
.env("TMPDIR", &temporary)
@@ -372,15 +407,18 @@ fn write_changed(path: &Path, bytes: &[u8]) -> Result<()> {
}
Ok(())
}
/// Installs one embedded source tree without touching unchanged files.
fn write_dir(directory: &Dir<'_>, destination: &Path) -> Result<()> {
/// Collects embedded source files for deterministic hashing and installation.
fn collect_files(
directory: &Dir<'_>,
destination: &Path,
files: &mut std::collections::BTreeMap<PathBuf, Vec<u8>>,
) {
for file in directory.files() {
write_changed(&destination.join(file.path()), file.contents())?;
files.insert(destination.join(file.path()), file.contents().to_vec());
}
for directory in directory.dirs() {
write_dir(directory, destination)?;
collect_files(directory, destination, files);
}
Ok(())
}
/// Counts only final libtest summaries, never individual status lines.
fn test_count(output: &str, field: &str) -> u64 {