Merge v3-t98: precompiled seed tools from Nix as a read-only cache layer, lazy otherwise (T98)

docs/RUNNING.md

@@ -32,6 +32,11 @@ For `init`, you can override it with `--userland BUNDLE` or the environment vari
 A bundle contains `bin/`, `src/`, and the seeded `etc/` tree. It is not a world folder.
 Closing stdin only detaches the optional terminal console. The host and chat service continue.
 
+The package sets `SAUSAGE_PRECOMPILED` to its read-only cache of compiled tools.
+The host checks `--cache` or `~/.cache/sausage` first, then this folder, then compiles missing tools into the writable cache.
+If Wasmtime rejects an artifact, the host logs one line and compiles the program lazily.
+For development or tests, run `sausage precompile BUNDLE OUT` and set `SAUSAGE_PRECOMPILED=OUT`.
+
 The source-tree bundle builder is:
 
 ```sh

flake.nix

@@ -143,6 +143,9 @@
           cargoArtifacts = hostArtifacts;
           doNotRemoveReferencesToVendorDir = true;
         });
+        precompiled = pkgs.runCommand "sausage-precompiled" {} ''
+          ${sausageUnwrapped}/bin/sausage precompile ${bundle} $out
+        '';
         # Keep the wrapper separate from crane's removal of toolchain references.
         sausage = pkgs.symlinkJoin {
           name = "sausage-${sausageUnwrapped.version}";
@@ -150,6 +153,7 @@
           nativeBuildInputs = [ pkgs.makeWrapper ];
           postBuild = ''
             wrapProgram $out/bin/sausage --set-default SAUSAGE_USERLAND ${bundle} \
+              --set-default SAUSAGE_PRECOMPILED ${precompiled} \
               --prefix PATH : ${runtimePath} \
               --set-default CC_wasm32_wasip2 ${wasiCcEnv.CC_wasm32_wasip2} \
               --set-default AR_wasm32_wasip2 ${wasiCcEnv.AR_wasm32_wasip2} \
@@ -170,6 +174,7 @@
           SAUSAGE_TEST_NO_BUILDER_SANDBOX = "1";
           SAUSAGE_NIX_CHECK = "1";
           SAUSAGE_USERLAND = bundle;
+          SAUSAGE_PRECOMPILED = precompiled;
           preCheck = ''
             export SAUSAGE_CACHE="$TMPDIR/sausage-cache"
             export CARGO_TARGET_DIR="$PWD/target"
@@ -181,8 +186,10 @@
         });
         checks.shell = pkgs.runCommand "sausage-shell" {} ''
           export SAUSAGE_CACHE="$TMPDIR/sausage-cache"
+          ${sausage}/bin/sausage init "$TMPDIR/world"
           ${sausage}/bin/sausage sh "$TMPDIR/world" -c \
             'set -e; echo hi | tr a-z A-Z; git --version; rg --version; awk "BEGIN {print 42}"; echo payload > /payload; tar -cf /test.tar /payload; tar -tf /test.tar; qjs -e "console.log(42)"; cargo --help; undo list' > output
+          test ! -e "$SAUSAGE_CACHE"
           cat output
           grep -qx HI output
           grep -q 'git version' output

host/src/main.rs

@@ -14,6 +14,8 @@ struct Cli {
 
 #[derive(Subcommand)]
 enum Command {
+    /// Compile the bundled tools into a reusable read-only cache.
+    Precompile { bundle: PathBuf, out: PathBuf },
     /// Create a world with default tools and /etc/sausage/config.yaml.
     Init {
         world: PathBuf,
@@ -44,6 +46,11 @@ async fn main() -> anyhow::Result<()> {
         sausage::process::configure_cache(cache)?;
     }
     match cli.command {
+        Command::Precompile { bundle, out } => {
+            let count = sausage::process::precompile(&bundle, &out)?;
+            println!("Precompiled programs: {count}");
+            Ok(())
+        }
         Command::Init { world, userland } => {
             let bundle = userland
                 .or_else(|| std::env::var_os("SAUSAGE_USERLAND").map(PathBuf::from))

host/src/process/cache.rs

@@ -1,7 +1,9 @@
-//! Compiles programs on first start and shares cached results across worlds in one host process.
+//! Shares lazy compilation across worlds, with writable and read-only precompiled artifact layers.
 use super::{Compiled, State, adapter, bindings};
+use sha2::{Digest, Sha256};
 use std::{
-    collections::HashMap,
+    collections::{HashMap, HashSet, hash_map::DefaultHasher},
+    hash::{Hash, Hasher},
     path::{Path, PathBuf},
     sync::{Arc, Mutex, OnceLock},
 };
@@ -44,8 +46,47 @@ pub(super) fn shared() -> anyhow::Result<Arc<Compiler>> {
     }
     Ok(shared.as_ref().unwrap().clone())
 }
+/// Compile each distinct bundled program into the runtime's artifact format and keys.
+pub fn precompile(bundle: &Path, out: &Path) -> anyhow::Result<usize> {
+    let compiler = Compiler::with_precompiled(out, None)?;
+    std::fs::create_dir_all(out)?;
+    let mut paths = std::fs::read_dir(bundle.join("bin"))?
+        .map(|entry| entry.map(|entry| entry.path()))
+        .collect::<Result<Vec<_>, _>>()?;
+    paths.sort();
+    let mut keys = HashSet::new();
+    for path in paths {
+        if path
+            .extension()
+            .is_some_and(|extension| extension == "wasm")
+        {
+            let bytes = std::fs::read(&path)?;
+            let key = compiler.key(&bytes);
+            if keys.insert(key.clone()) {
+                write_artifact(out, &key, &compiler.artifact(&bytes)?)?;
+            }
+        }
+    }
+    Ok(keys.len())
+}
+
+// Publish complete artifacts, including when separate hosts share the writable cache.
+fn write_artifact(directory: &Path, key: &str, bytes: &[u8]) -> std::io::Result<()> {
+    std::fs::create_dir_all(directory)?;
+    let temporary = directory.join(format!("{}.tmp", uuid::Uuid::new_v4()));
+    let result = std::fs::write(&temporary, bytes)
+        .and_then(|()| std::fs::rename(&temporary, directory.join(key)));
+    if result.is_err() {
+        let _ = std::fs::remove_file(temporary);
+    }
+    result
+}
+
 pub(super) struct Compiler {
     pub engine: Engine,
+    directory: PathBuf,
+    precompiled: Option<PathBuf>,
+    fingerprint: u64,
     programs: Mutex<HashMap<Vec<u8>, Compilation>>,
     #[cfg(test)]
     before_compile: Mutex<Option<Box<dyn FnOnce() + Send>>>,
@@ -54,16 +95,30 @@ pub(super) struct Compiler {
 }
 impl Compiler {
     pub fn new(directory: &Path) -> anyhow::Result<Arc<Self>> {
+        Self::with_precompiled(
+            directory,
+            std::env::var_os("SAUSAGE_PRECOMPILED").map(PathBuf::from),
+        )
+    }
+    fn with_precompiled(
+        directory: &Path,
+        precompiled: Option<PathBuf>,
+    ) -> anyhow::Result<Arc<Self>> {
         let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
         let mut config = wasmtime::Config::new();
         config.epoch_interruption(true);
         // Keep Cranelift's default optimization: compiles are cached per content, and guests run
         // many times. OptLevel::None saved only 12.7% of compile time (T15) but slows every run.
-        let mut cache = wasmtime::CacheConfig::new();
-        cache.with_directory(directory);
-        config.cache(Some(wasmtime::Cache::new(cache)?));
+        let engine = Engine::new(&config)?;
+        let mut fingerprint = DefaultHasher::new();
+        engine
+            .precompile_compatibility_hash()
+            .hash(&mut fingerprint);
         Ok(Arc::new(Self {
-            engine: Engine::new(&config)?,
+            engine,
+            directory: directory.to_owned(),
+            precompiled,
+            fingerprint: fingerprint.finish(),
             programs: Mutex::new(HashMap::new()),
             #[cfg(test)]
             before_compile: Mutex::new(None),
@@ -95,7 +150,36 @@ impl Compiler {
             result.changed().await?;
         }
     }
+    fn key(&self, bytes: &[u8]) -> String {
+        format!(
+            "{:016x}-{}.cwasm",
+            self.fingerprint,
+            hex::encode(Sha256::digest(bytes))
+        )
+    }
     fn compile(&self, bytes: &[u8]) -> anyhow::Result<Compiled> {
+        let key = self.key(bytes);
+        for directory in std::iter::once(&self.directory).chain(self.precompiled.iter()) {
+            let path = directory.join(&key);
+            let result = match std::fs::read(&path) {
+                Ok(artifact) => self.link(bytes, &artifact),
+                Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
+                Err(error) => Err(error.into()),
+            };
+            match result {
+                Ok(compiled) => return Ok(compiled),
+                Err(error) => eprintln!(
+                    "Ignore precompiled cache {}: {}",
+                    path.display(),
+                    error.to_string().replace('\n', " ")
+                ),
+            }
+        }
+        let artifact = self.artifact(bytes)?;
+        write_artifact(&self.directory, &key, &artifact)?;
+        self.link(bytes, &artifact)
+    }
+    fn artifact(&self, bytes: &[u8]) -> wasmtime::Result<Vec<u8>> {
         #[cfg(test)]
         {
             self.compilations
@@ -105,14 +189,24 @@ impl Compiler {
             }
         }
         if bytes.get(..8) == Some(b"\0asm\x01\0\0\0") {
-            let module = wasmtime::Module::new(&self.engine, bytes)?;
+            self.engine.precompile_module(bytes)
+        } else {
+            self.engine.precompile_component(bytes)
+        }
+    }
+    fn link(&self, bytes: &[u8], artifact: &[u8]) -> anyhow::Result<Compiled> {
+        // SAFETY: artifacts come from this host's compiler or its trusted host/Nix cache, never
+        // from the world. Wasmtime checks engine compatibility before it loads their code.
+        if bytes.get(..8) == Some(b"\0asm\x01\0\0\0") {
+            let module = unsafe { wasmtime::Module::deserialize(&self.engine, artifact)? };
             let mut linker = wasmtime::Linker::new(&self.engine);
             sausage_vfs::preview1::add_to_linker(&mut linker, |s: &mut State| {
                 s.preview1.as_mut().unwrap()
             })?;
             Ok(Compiled::Module(linker.instantiate_pre(&module)?))
         } else {
-            let component = Component::new(&self.engine, bytes)?;
+            // SAFETY: the artifact has the same trusted provenance described above.
+            let component = unsafe { Component::deserialize(&self.engine, artifact)? };
             let mut linker = Linker::new(&self.engine);
             wasmtime_wasi::p2::add_to_linker_async(&mut linker)?;
             wasmtime_wasi_http::p2::add_only_http_to_linker_async(&mut linker)?;
@@ -136,6 +230,120 @@ mod tests {
     use crate::exec::{Command, Exec, Options};
     use std::{future::Future, sync::atomic::Ordering};
 
+    fn fixture(component: bool) -> Vec<u8> {
+        wat::parse_str(if component {
+            r#"(component
+                (core module $m (func (export "run") (result i32) i32.const 0))
+                (core instance $i (instantiate $m))
+                (func $run (result (result)) (canon lift (core func $i "run")))
+                (instance $cli (export "run" (func $run)))
+                (export "wasi:cli/run@0.2.6" (instance $cli)))"#
+        } else {
+            r#"(module (memory (export "memory") 1) (func (export "_start")))"#
+        })
+        .unwrap()
+    }
+
+    #[tokio::test]
+    async fn precompile_deduplicates_and_readonly_hits_compile_and_write_nothing()
+    -> anyhow::Result<()> {
+        use std::os::unix::fs::PermissionsExt;
+        let temp = tempfile::tempdir()?;
+        let bundle = temp.path().join("bundle");
+        std::fs::create_dir_all(bundle.join("bin"))?;
+        for (name, component) in [("one", false), ("alias", false), ("component", true)] {
+            std::fs::write(bundle.join(format!("bin/{name}.wasm")), fixture(component))?;
+        }
+        std::fs::write(bundle.join("bin/not-wasm"), b"not a program")?;
+        let readonly = temp.path().join("precompiled");
+        assert_eq!(precompile(&bundle, &readonly)?, 2);
+        assert_eq!(std::fs::read_dir(&readonly)?.count(), 2);
+        let compiler =
+            Compiler::with_precompiled(&temp.path().join("writable"), Some(readonly.clone()))?;
+        for component in [false, true] {
+            let bytes = fixture(component);
+            let path = readonly.join(compiler.key(&bytes));
+            let before = std::fs::read(&path)?;
+            let modified = std::fs::metadata(&path)?.modified()?;
+            std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o444))?;
+            std::fs::set_permissions(&readonly, std::fs::Permissions::from_mode(0o555))?;
+            let directory_modified = std::fs::metadata(&readonly)?.modified()?;
+            let result = compiler.get(bytes).await;
+            std::fs::set_permissions(&readonly, std::fs::Permissions::from_mode(0o755))?;
+            result?;
+            assert_eq!(std::fs::read(&path)?, before);
+            assert_eq!(std::fs::metadata(&path)?.modified()?, modified);
+            assert_eq!(
+                std::fs::metadata(&readonly)?.modified()?,
+                directory_modified
+            );
+        }
+        assert_eq!(compiler.compilations.load(Ordering::Relaxed), 0);
+        assert!(!temp.path().join("writable").exists());
+        assert_eq!(std::fs::read_dir(&readonly)?.count(), 2);
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn writable_hit_precedes_readonly_and_changed_content_compiles_lazily()
+    -> anyhow::Result<()> {
+        let temp = tempfile::tempdir()?;
+        let writable = temp.path().join("writable");
+        let readonly = temp.path().join("readonly");
+        let compiler = Compiler::with_precompiled(&writable, Some(readonly.clone()))?;
+        let bytes = fixture(false);
+        write_artifact(
+            &writable,
+            &compiler.key(&bytes),
+            &compiler.engine.precompile_module(&bytes)?,
+        )?;
+        write_artifact(&readonly, &compiler.key(&bytes), b"invalid artifact")?;
+        compiler.get(bytes).await?;
+        assert_eq!(compiler.compilations.load(Ordering::Relaxed), 0);
+        let changed =
+            wat::parse_str(r#"(module (memory (export "memory") 2) (func (export "_start")))"#)?;
+        compiler.get(changed.clone()).await?;
+        assert_eq!(compiler.compilations.load(Ordering::Relaxed), 1);
+        assert!(writable.join(compiler.key(&changed)).is_file());
+        let reopened = Compiler::with_precompiled(&writable, None)?;
+        reopened.get(changed).await?;
+        assert_eq!(reopened.compilations.load(Ordering::Relaxed), 0);
+        assert_eq!(std::fs::read_dir(&readonly)?.count(), 1);
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn corrupted_and_incompatible_readonly_entries_fall_back_to_lazy_compilation()
+    -> anyhow::Result<()> {
+        for component in [false, true] {
+            for incompatible in [false, true] {
+                let temp = tempfile::tempdir()?;
+                let writable = temp.path().join("writable");
+                let readonly = temp.path().join("readonly");
+                let compiler = Compiler::with_precompiled(&writable, Some(readonly.clone()))?;
+                let bytes = fixture(component);
+                let artifact = if incompatible {
+                    // The runtime enables epoch interruption. This code deliberately does not.
+                    let engine = Engine::new(&wasmtime::Config::new())?;
+                    if component {
+                        engine.precompile_component(&bytes)?
+                    } else {
+                        engine.precompile_module(&bytes)?
+                    }
+                } else {
+                    b"corrupted artifact".to_vec()
+                };
+                let key = compiler.key(&bytes);
+                write_artifact(&readonly, &key, &artifact)?;
+                compiler.get(bytes).await?;
+                assert_eq!(compiler.compilations.load(Ordering::Relaxed), 1);
+                assert_eq!(std::fs::read(readonly.join(&key))?, artifact);
+                assert!(writable.join(key).is_file());
+            }
+        }
+        Ok(())
+    }
+
     #[tokio::test]
     async fn startup_is_lazy_and_identical_programs_share_first_compilation() -> anyhow::Result<()>
     {

host/src/process/mod.rs

@@ -55,7 +55,7 @@ pub mod net;
 pub fn default_script() -> Arc<dyn crate::cred::script::Script> {
     Arc::new(crate::cred::quickjs::QuickJs::default())
 }
-pub use cache::configure as configure_cache;
+pub use cache::{configure as configure_cache, precompile};
 mod services;
 mod watcher;
 pub use watcher::Watcher;