Merge branch 'v3' into v3-t29
# Conflicts: # host/Cargo.toml # host/src/harness/mod.rs
This commit is contained in:
@@ -41,6 +41,7 @@ mod bindings {
|
||||
with: {
|
||||
"wasi:io": wasmtime_wasi::p2::bindings::io,
|
||||
"sausage:process/host.child": super::Child,
|
||||
"sausage:process/host.watcher": super::Watcher,
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -55,6 +56,8 @@ pub fn default_script() -> Arc<dyn crate::cred::script::Script> {
|
||||
}
|
||||
pub use cache::configure as configure_cache;
|
||||
mod services;
|
||||
mod watcher;
|
||||
pub use watcher::Watcher;
|
||||
mod stdio;
|
||||
pub use services::supervise;
|
||||
use stdio::Io;
|
||||
@@ -567,7 +570,23 @@ impl Host for State {
|
||||
Ok(self.runtime.kill_pid(pid).map_err(|e| e.to_string()))
|
||||
}
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl wasmtime_wasi::p2::Pollable for Child {
|
||||
async fn ready(&mut self) {
|
||||
while self.code.borrow_and_update().is_none() {
|
||||
if self.code.changed().await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
impl HostChild for State {
|
||||
async fn subscribe(
|
||||
&mut self,
|
||||
child: Resource<Child>,
|
||||
) -> wasmtime::Result<Resource<wasmtime_wasi::p2::DynPollable>> {
|
||||
wasmtime_wasi::p2::subscribe(&mut self.table, child)
|
||||
}
|
||||
async fn pid(&mut self, child: Resource<Child>) -> wasmtime::Result<u32> {
|
||||
Ok(self.table.get(&child)?.pid)
|
||||
}
|
||||
|
||||
+65
-146
@@ -1,49 +1,28 @@
|
||||
//! Supervises the world's services list, live edits, crash limits, and graceful stops.
|
||||
use super::{GroupGuard, Request, Runtime, Started, Stdio};
|
||||
use serde::Deserialize;
|
||||
use std::{
|
||||
collections::{BTreeMap, VecDeque},
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::time::Instant;
|
||||
|
||||
#[derive(Clone, Deserialize, PartialEq, Eq)]
|
||||
struct Spec {
|
||||
name: String,
|
||||
argv: Vec<String>,
|
||||
#[serde(default = "root")]
|
||||
cwd: String,
|
||||
#[serde(default)]
|
||||
env: BTreeMap<String, String>,
|
||||
//! Starts world init, reaps its command tree, and applies host restart and shutdown deadlines.
|
||||
use super::{GroupGuard, Request, Runtime, Stdio};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
pub struct Supervisor {
|
||||
shutdown: tokio::sync::oneshot::Sender<()>,
|
||||
task: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
fn root() -> String {
|
||||
"/".into()
|
||||
impl Supervisor {
|
||||
pub async fn shutdown(self) {
|
||||
let _ = self.shutdown.send(());
|
||||
let _ = self.task.await;
|
||||
}
|
||||
}
|
||||
struct Live {
|
||||
child: Started,
|
||||
_guard: GroupGuard,
|
||||
}
|
||||
struct Service {
|
||||
spec: Spec,
|
||||
live: Option<Live>,
|
||||
crashes: VecDeque<Instant>,
|
||||
stopped: bool,
|
||||
}
|
||||
impl Service {
|
||||
async fn start(&mut self, runtime: &Arc<Runtime>) {
|
||||
match runtime
|
||||
.start(
|
||||
/// The host knows only init. Unit policy belongs to the WASI program.
|
||||
pub fn supervise(runtime: Arc<Runtime>) -> Supervisor {
|
||||
let (shutdown, mut stop) = tokio::sync::oneshot::channel();
|
||||
let task = tokio::spawn(async move {
|
||||
let mut backoff = Duration::from_secs(1);
|
||||
loop {
|
||||
let start = runtime.start(
|
||||
Request {
|
||||
program: self.spec.argv.first().cloned().unwrap_or_default(),
|
||||
argv: self.spec.argv.clone(),
|
||||
cwd: self.spec.cwd.clone(),
|
||||
env: self
|
||||
.spec
|
||||
.env
|
||||
.iter()
|
||||
.map(|(k, v)| (k.clone(), v.clone()))
|
||||
.collect(),
|
||||
program: "/bin/init.wasm".into(),
|
||||
argv: vec!["/bin/init.wasm".into()],
|
||||
cwd: "/".into(),
|
||||
env: Vec::new(),
|
||||
group: None,
|
||||
stdin: Stdio::Pipe,
|
||||
stdout: Stdio::Null,
|
||||
@@ -54,116 +33,56 @@ impl Service {
|
||||
0,
|
||||
Vec::new(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(child) => {
|
||||
);
|
||||
let child = tokio::select! { _ = &mut stop => return, child = start => child };
|
||||
if let Ok(mut child) = child {
|
||||
let guard = GroupGuard {
|
||||
runtime: runtime.clone(),
|
||||
group: child.pid,
|
||||
};
|
||||
self.live = Some(Live {
|
||||
child,
|
||||
_guard: guard,
|
||||
});
|
||||
let exiting = async {
|
||||
while child.code.borrow_and_update().is_none() {
|
||||
if child.code.changed().await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
};
|
||||
let shutdown = tokio::select! { _ = &mut stop => true, _ = exiting => false };
|
||||
if shutdown {
|
||||
drop(child.input.take());
|
||||
let _ = tokio::time::timeout(Duration::from_secs(15), async {
|
||||
while child.code.borrow_and_update().is_none() {
|
||||
if child.code.changed().await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
drop(guard);
|
||||
reaped(&runtime, child.pid).await;
|
||||
return;
|
||||
}
|
||||
// This guard kills all descendants, including separate process groups.
|
||||
drop(guard);
|
||||
reaped(&runtime, child.pid).await;
|
||||
}
|
||||
Err(_) => self.crashed(),
|
||||
tokio::select! { _ = &mut stop => return, _ = tokio::time::sleep(backoff) => () }
|
||||
backoff = (backoff * 2).min(Duration::from_secs(30));
|
||||
}
|
||||
}
|
||||
fn crashed(&mut self) {
|
||||
let now = Instant::now();
|
||||
while self
|
||||
.crashes
|
||||
.front()
|
||||
.is_some_and(|t| now.duration_since(*t) >= Duration::from_secs(10))
|
||||
});
|
||||
Supervisor { shutdown, task }
|
||||
}
|
||||
async fn reaped(runtime: &Runtime, command: u32) {
|
||||
loop {
|
||||
if !runtime
|
||||
.entries
|
||||
.lock()
|
||||
.unwrap()
|
||||
.values()
|
||||
.any(|entry| entry.command == command)
|
||||
{
|
||||
self.crashes.pop_front();
|
||||
}
|
||||
self.crashes.push_back(now);
|
||||
self.stopped = self.crashes.len() > 5;
|
||||
}
|
||||
async fn stop(&mut self) {
|
||||
if let Some(mut live) = self.live.take() {
|
||||
drop(live.child.input.take());
|
||||
let _ = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if live.child.code.borrow().is_some() {
|
||||
break;
|
||||
}
|
||||
if live.child.code.changed().await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
// The guard stops remaining descendants, or the process after its grace period.
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
/// Start the supervisor. Aborting its task stops its service processes.
|
||||
/// The YAML file contains name, argv, optional cwd, and an optional map of environment variables.
|
||||
pub fn supervise(runtime: Arc<Runtime>) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
let mut services: BTreeMap<String, Service> = BTreeMap::new();
|
||||
let mut previous = Vec::new();
|
||||
let mut interval = tokio::time::interval(Duration::from_millis(100));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
let content = match runtime
|
||||
.world
|
||||
.root
|
||||
.open("/config/services.yaml", Default::default())
|
||||
.await
|
||||
{
|
||||
Ok(file) => match file.stat().await {
|
||||
Ok(stat) => file.read(0, stat.size as usize).await.ok(),
|
||||
Err(_) => None,
|
||||
},
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Some(b"[]".to_vec()),
|
||||
Err(_) => None,
|
||||
};
|
||||
if let Some(content) = content.filter(|c| *c != previous)
|
||||
&& let Ok(specs) = serde_yaml_ng::from_slice::<Vec<Spec>>(&content)
|
||||
{
|
||||
let specs: BTreeMap<_, _> =
|
||||
specs.into_iter().map(|s| (s.name.clone(), s)).collect();
|
||||
let removed: Vec<_> = services
|
||||
.keys()
|
||||
.filter(|name| !specs.contains_key(*name))
|
||||
.cloned()
|
||||
.collect();
|
||||
for name in removed {
|
||||
services.remove(&name).unwrap().stop().await;
|
||||
}
|
||||
for (name, spec) in specs {
|
||||
let service = services.entry(name).or_insert_with(|| Service {
|
||||
spec: spec.clone(),
|
||||
live: None,
|
||||
crashes: VecDeque::new(),
|
||||
stopped: false,
|
||||
});
|
||||
if service.spec != spec {
|
||||
service.stop().await;
|
||||
}
|
||||
service.spec = spec;
|
||||
service.crashes.clear();
|
||||
service.stopped = false;
|
||||
}
|
||||
previous = content;
|
||||
}
|
||||
for service in services.values_mut() {
|
||||
if service
|
||||
.live
|
||||
.as_ref()
|
||||
.is_some_and(|live| live.child.code.borrow().is_some())
|
||||
{
|
||||
service.live = None;
|
||||
service.crashed();
|
||||
}
|
||||
if service.live.is_none() && !service.stopped {
|
||||
service.start(&runtime).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
//! Adapts VFS observation to bounded, pollable world-path change queues.
|
||||
use super::{
|
||||
State,
|
||||
bindings::sausage::process::host::{Change, ChangeKind, HostWatcher},
|
||||
};
|
||||
use sausage_vfs::observe::{ChangeKind as Kind, under};
|
||||
use tokio::sync::broadcast::{Receiver, error::TryRecvError};
|
||||
use wasmtime::component::Resource;
|
||||
use wasmtime_wasi::p2::{DynPollable, Pollable, subscribe};
|
||||
|
||||
pub struct Watcher {
|
||||
paths: Vec<String>,
|
||||
events: Receiver<(String, Kind)>,
|
||||
pending: Vec<Change>,
|
||||
}
|
||||
impl Watcher {
|
||||
fn push(&mut self, path: String, kind: Kind) {
|
||||
if self.paths.iter().any(|base| under(&path, base)) {
|
||||
if self
|
||||
.pending
|
||||
.iter()
|
||||
.any(|c| matches!(c.kind, ChangeKind::Overflow))
|
||||
{
|
||||
return;
|
||||
}
|
||||
if self.pending.len() >= 256 {
|
||||
self.overflow();
|
||||
return;
|
||||
}
|
||||
self.pending.push(Change {
|
||||
path,
|
||||
kind: match kind {
|
||||
Kind::Created => ChangeKind::Created,
|
||||
Kind::Modified => ChangeKind::Modified,
|
||||
Kind::Removed => ChangeKind::Removed,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
fn overflow(&mut self) {
|
||||
self.pending.clear();
|
||||
self.pending.push(Change {
|
||||
path: "/".into(),
|
||||
kind: ChangeKind::Overflow,
|
||||
});
|
||||
}
|
||||
fn drain(&mut self) {
|
||||
loop {
|
||||
match self.events.try_recv() {
|
||||
Ok((path, kind)) => self.push(path, kind),
|
||||
Err(TryRecvError::Lagged(_)) => self.overflow(),
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl Pollable for Watcher {
|
||||
async fn ready(&mut self) {
|
||||
loop {
|
||||
self.drain();
|
||||
if !self.pending.is_empty() {
|
||||
return;
|
||||
}
|
||||
match self.events.recv().await {
|
||||
Ok((path, kind)) => self.push(path, kind),
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => self.overflow(),
|
||||
Err(_) => return,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
impl HostWatcher for State {
|
||||
async fn new(&mut self, paths: Vec<String>) -> wasmtime::Result<Resource<Watcher>> {
|
||||
let observer = self
|
||||
.runtime
|
||||
.world
|
||||
.root
|
||||
.observer()
|
||||
.ok_or_else(|| wasmtime::format_err!("filesystem has no observer"))?;
|
||||
let paths: Vec<String> = paths
|
||||
.iter()
|
||||
.map(|p| {
|
||||
sausage_vfs::resolve("/", p)
|
||||
.trim_end_matches('/')
|
||||
.to_owned()
|
||||
})
|
||||
.map(|p| if p.is_empty() { "/".into() } else { p })
|
||||
.collect();
|
||||
Ok(self.table.push(Watcher {
|
||||
events: observer.subscribe(paths.clone()),
|
||||
paths,
|
||||
pending: Vec::new(),
|
||||
})?)
|
||||
}
|
||||
async fn subscribe(
|
||||
&mut self,
|
||||
watcher: Resource<Watcher>,
|
||||
) -> wasmtime::Result<Resource<DynPollable>> {
|
||||
subscribe(&mut self.table, watcher)
|
||||
}
|
||||
async fn changes(&mut self, watcher: Resource<Watcher>) -> wasmtime::Result<Vec<Change>> {
|
||||
let watcher = self.table.get_mut(&watcher)?;
|
||||
watcher.drain();
|
||||
Ok(std::mem::take(&mut watcher.pending))
|
||||
}
|
||||
async fn drop(&mut self, watcher: Resource<Watcher>) -> wasmtime::Result<()> {
|
||||
self.table.delete(watcher)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user