surma/sausage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Load world extensions from commit snapshots with last-good recovery

Browse Source
This commit is contained in:
surma
2026-11-07 10:38:20 +00:00
parent 534fa50615
commit bcb3e4d8e5
17 changed files with 42251 additions and 399 deletions
Download Patch File Download Diff File Expand all files Collapse all files
host/src/harness/environment.rs
+58 -5
View File
@@ -40,9 +40,15 @@ pub async fn op_file(
Ok(Value::Null)
}
"mkdir" => { root.mkdir(&path).await?; Ok(Value::Null) }
"stat" => {
let s = root.stat(&path, true).await?;
Ok(json!({"path": path, "name": path.rsplit('/').next().unwrap_or(""), "kind": if s.directory {"directory"} else {"file"}, "size": s.size, "mtimeMs": 0}))
"list" => Ok(json!(root.list(&path).await?.into_iter().map(|e| json!({"name":e.name,"kind":if e.symlink {"symlink"} else if e.directory {"directory"} else {"file"}})).collect::<Vec<_>>())),
"remove" => {
if root.stat(&path, false).await?.directory { root.remove_dir(&path).await?; } else { root.remove_file(&path).await?; }
Ok(Value::Null)
}
"rename" => { root.rename(&path, &String::from_utf8(bytes).map_err(|_| std::io::ErrorKind::InvalidInput)?).await?; Ok(Value::Null) }
"stat" | "lstat" => {
let s = root.stat(&path, kind == "stat").await?;
Ok(json!({"path": path, "name": path.rsplit('/').next().unwrap_or(""), "kind": if s.symlink {"symlink"} else if s.directory {"directory"} else {"file"}, "size": s.size, "mtimeMs": 0}))
}
_ => Err(std::io::ErrorKind::Unsupported.into()),
}
@@ -50,7 +56,7 @@ pub async fn op_file(
Ok(match result {
Ok(value) => json!({"ok": true, "value": value}),
Err(e) => {
json!({"ok": false, "error": {"code": match e.kind() { std::io::ErrorKind::NotFound => "not_found", std::io::ErrorKind::PermissionDenied => "permission_denied", _ => "unknown" }, "message": e.to_string(), "path": path}})
json!({"ok": false, "error": {"code": match e.kind() { std::io::ErrorKind::NotFound => "not_found", std::io::ErrorKind::AlreadyExists => "already_exists", std::io::ErrorKind::PermissionDenied => "permission_denied", _ => "unknown" }, "message": e.to_string(), "path": path}})
}
})
}
@@ -77,6 +83,8 @@ pub fn op_exec(
#[string] cwd: String,
timeout: Option<f64>,
#[string] conversation: Option<String>,
#[serde] env: Option<Vec<(String, String)>>,
#[serde] conversations: Option<Vec<String>>,
) -> Result<u32, JsErrorBox> {
let exec = state.borrow::<Environment>().exec.clone();
let command = if let Some(s) = command.as_str() {
@@ -88,7 +96,7 @@ pub fn op_exec(
};
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel();
let task = tokio::spawn(async move {
let result = exec.exec(command, Options { cwd, conversations: conversation.into_iter().collect(), timeout: timeout.map(std::time::Duration::from_secs_f64), ..Default::default() }, &mut |stream, bytes| { let _ = sender.send(json!({"stream": if stream == Stream::Stdout {"stdout"} else {"stderr"}, "bytes": bytes})); }).await;
let result = exec.exec(command, Options { cwd, env: env.unwrap_or_default(), conversations: conversations.unwrap_or_else(|| conversation.into_iter().collect()), timeout: timeout.map(std::time::Duration::from_secs_f64) }, &mut |stream, bytes| { let _ = sender.send(json!({"stream": if stream == Stream::Stdout {"stdout"} else {"stderr"}, "bytes": bytes})); }).await;
let _ = sender.send(match result {
Ok(exit) => json!({"exitCode": exit}),
Err(e) => json!({"error": e.to_string()}),
@@ -129,6 +137,51 @@ pub async fn op_exec_next(
Ok(value)
}
struct Watch {
events: tokio::sync::Mutex<
tokio::sync::broadcast::Receiver<(String, sausage_vfs::observe::ChangeKind)>,
>,
closed: tokio::sync::Notify,
}
impl deno_core::Resource for Watch {
fn close(self: Rc<Self>) {
self.closed.notify_one();
}
}
#[op2]
pub fn op_watch(state: &mut OpState, #[serde] paths: Vec<String>) -> Result<u32, JsErrorBox> {
let observer = state
.borrow::<Environment>()
.root
.observer()
.ok_or_else(|| JsErrorBox::generic("filesystem has no observer"))?;
Ok(state.resource_table.add(Watch {
events: tokio::sync::Mutex::new(observer.subscribe(paths)),
closed: Default::default(),
}))
}
#[op2]
#[serde]
pub async fn op_watch_next(
state: Rc<RefCell<OpState>>,
rid: u32,
) -> Result<serde_json::Value, JsErrorBox> {
let watch = state
.borrow()
.resource_table
.get::<Watch>(rid)
.map_err(|e| JsErrorBox::generic(e.to_string()))?;
let mut events = watch.events.lock().await;
let result = tokio::select! { value = events.recv() => value, _ = watch.closed.notified() => return Ok(Value::Null) };
Ok(match result {
Ok((path, kind)) => json!({"path":path,"kind":format!("{kind:?}")}),
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
json!({"path":"/","kind":"Overflow"})
}
Err(_) => Value::Null,
})
}
#[op2(fast)]
pub fn op_output(#[string] text: &str) {
print!("{text}");
host/src/harness/loader.rs
+144
View File
@@ -0,0 +1,144 @@
//! Loads committed world modules and shares the bundle's package instances and status.
use deno_core::{
ModuleLoadOptions, ModuleLoadReferrer, ModuleLoadResponse, ModuleLoader, ModuleSource,
ModuleSourceCode, ModuleSpecifier, ModuleType, OpState, ResolutionKind, op2,
};
use deno_error::JsErrorBox;
use sausage_vfs::{Layer, Open};
use std::{
cell::RefCell,
collections::BTreeMap,
path::{Path, PathBuf},
rc::Rc,
sync::{Arc, Mutex, OnceLock},
};
pub(crate) type Status = Arc<Mutex<String>>;
pub(crate) fn status(path: &Path) -> Status {
static STATES: OnceLock<Mutex<BTreeMap<PathBuf, Status>>> = OnceLock::new();
STATES
.get_or_init(Default::default)
.lock()
.unwrap()
.entry(path.to_owned())
.or_insert_with(|| {
Arc::new(Mutex::new(
"{\"good\":null,\"current\":null,\"error\":null}".into(),
))
})
.clone()
}
pub(super) struct Loader {
pub root: Arc<dyn Layer>,
pub modules: RefCell<BTreeMap<String, String>>,
}
#[op2]
pub fn op_modules(state: &mut OpState, #[serde] modules: BTreeMap<String, Vec<String>>) {
let loader = state.borrow::<Rc<Loader>>();
for (name, exports) in modules {
let quoted = serde_json::to_string(&name).unwrap();
let code = exports.into_iter().enumerate().map(|(index,key)| format!("const e{index} = globalThis.__sausage.modules[{quoted}].{key}; export {{ e{index} as {key} }};\n")).collect();
loader
.modules
.borrow_mut()
.insert(format!("sausage:{name}"), code);
}
}
#[op2(fast)]
pub fn op_extension_status(state: &mut OpState, #[string] value: String) {
*state.borrow::<Status>().lock().unwrap() = value;
}
pub(super) fn transpile(name: &str, source: String) -> Result<String, JsErrorBox> {
if !name.ends_with(".ts") {
return Ok(source);
}
let parsed = deno_ast::parse_module(deno_ast::ParseParams {
specifier: ModuleSpecifier::parse(name).map_err(|e| JsErrorBox::generic(e.to_string()))?,
text: source.into(),
media_type: deno_ast::MediaType::TypeScript,
capture_tokens: false,
scope_analysis: false,
maybe_syntax: None,
})
.map_err(|e| JsErrorBox::generic(e.to_string()))?;
Ok(parsed
.transpile(
&Default::default(),
&Default::default(),
&deno_ast::EmitOptions {
source_map: deno_ast::SourceMapOption::None,
..Default::default()
},
)
.map_err(|e| JsErrorBox::generic(e.to_string()))?
.into_source()
.text)
}
impl ModuleLoader for Loader {
fn resolve(
&self,
specifier: &str,
referrer: &str,
_: ResolutionKind,
) -> Result<ModuleSpecifier, JsErrorBox> {
if specifier.starts_with("@earendil-works/") {
return ModuleSpecifier::parse(&format!("sausage:{specifier}"))
.map_err(|e| JsErrorBox::generic(e.to_string()));
}
deno_core::resolve_import(specifier, referrer)
.map_err(|e| JsErrorBox::generic(e.to_string()))
}
fn load(
&self,
specifier: &ModuleSpecifier,
_: Option<&ModuleLoadReferrer>,
_: ModuleLoadOptions,
) -> ModuleLoadResponse {
if let Some(source) = self.modules.borrow().get(specifier.as_str()) {
return ModuleLoadResponse::Sync(Ok(ModuleSource::new(
ModuleType::JavaScript,
ModuleSourceCode::String(source.clone().into()),
specifier,
None,
)));
}
let root = self.root.clone();
let specifier = specifier.clone();
ModuleLoadResponse::Async(Box::pin(async move {
let work = async {
let path = specifier
.to_file_path()
.map_err(|_| JsErrorBox::generic(format!("Unknown module {specifier}")))?;
let file = root
.open(
path.to_str().unwrap(),
Open {
follow: true,
..Default::default()
},
)
.await
.map_err(|e| JsErrorBox::generic(e.to_string()))?;
let size = file
.stat()
.await
.map_err(|e| JsErrorBox::generic(e.to_string()))?
.size;
let source = String::from_utf8(
file.read(0, size as usize)
.await
.map_err(|e| JsErrorBox::generic(e.to_string()))?,
)
.map_err(|e| JsErrorBox::generic(e.to_string()))?;
transpile(specifier.as_str(), source)
}
.await?;
Ok(ModuleSource::new(
ModuleType::JavaScript,
ModuleSourceCode::String(work.into()),
&specifier,
None,
))
}))
}
}
host/src/harness/mod.rs
+16 -24
View File
@@ -2,6 +2,7 @@
mod credentials;
mod environment;
pub mod input;
pub(crate) mod loader;
use crate::{exec::Exec, world::World};
use deno_core::{JsRuntime, OpState, RuntimeOptions, op2};
use deno_error::JsErrorBox;
@@ -87,6 +88,10 @@ deno_core::extension!(
environment::op_exec,
environment::op_exec_next,
input::op_input,
environment::op_watch,
environment::op_watch_next,
loader::op_modules,
loader::op_extension_status,
environment::op_output
]
);
@@ -150,7 +155,12 @@ pub async fn run_module_with_input(
source: &str,
) -> anyhow::Result<()> {
let proxy = credentials::proxy(&cred)?;
let loader = std::rc::Rc::new(loader::Loader {
root: world.root.clone(),
modules: Default::default(),
});
let mut runtime = JsRuntime::new(RuntimeOptions {
module_loader: Some(loader.clone()),
extensions: vec![
deno_telemetry::deno_telemetry::init(),
deno_webidl::deno_webidl::init(),
@@ -166,33 +176,15 @@ pub async fn run_module_with_input(
storage::init(),
],
extension_transpiler: Some(std::rc::Rc::new(|name, source| {
if !name.ends_with(".ts") {
return Ok((source, None));
}
let parsed = deno_ast::parse_module(deno_ast::ParseParams {
specifier: deno_core::url::Url::parse(&name).unwrap(),
text: source.into(),
media_type: deno_ast::MediaType::TypeScript,
capture_tokens: false,
scope_analysis: false,
maybe_syntax: None,
})
.map_err(|e| JsErrorBox::generic(e.to_string()))?;
let emitted = parsed
.transpile(
&Default::default(),
&Default::default(),
&deno_ast::EmitOptions {
source_map: deno_ast::SourceMapOption::None,
..Default::default()
},
)
.map_err(|e| JsErrorBox::generic(e.to_string()))?
.into_source();
Ok((emitted.text.into(), None))
Ok((loader::transpile(&name, source.to_string())?.into(), None))
})),
..Default::default()
});
runtime.op_state().borrow_mut().put(loader);
runtime
.op_state()
.borrow_mut()
.put(loader::status(&world.host_dir));
runtime
.op_state()
.borrow_mut()
host/src/world.rs
+4
View File
@@ -61,6 +61,10 @@ impl World {
Arc::new(std::sync::Mutex::new(Default::default()));
let snapshot = processes.clone();
let mut proc = Generated::default();
let extensions = crate::harness::loader::status(&host_dir);
proc.insert("/extensions", move || {
Ok(extensions.lock().unwrap().as_bytes().to_vec())
});
proc.insert("/processes", move || {
let mut text = String::from("PID\tPPID\tSTATE\tCWD\tCONVERSATION\tARGV\n");
for (pid, info) in snapshot.lock().unwrap().iter() {