Merge branch 'v3' into v3-t29
# Conflicts: # host/Cargo.toml # host/src/harness/mod.rs
This commit is contained in:
+65
-146
@@ -1,49 +1,28 @@
|
||||
//! Supervises the world's services list, live edits, crash limits, and graceful stops.
|
||||
use super::{GroupGuard, Request, Runtime, Started, Stdio};
|
||||
use serde::Deserialize;
|
||||
use std::{
|
||||
collections::{BTreeMap, VecDeque},
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::time::Instant;
|
||||
|
||||
#[derive(Clone, Deserialize, PartialEq, Eq)]
|
||||
struct Spec {
|
||||
name: String,
|
||||
argv: Vec<String>,
|
||||
#[serde(default = "root")]
|
||||
cwd: String,
|
||||
#[serde(default)]
|
||||
env: BTreeMap<String, String>,
|
||||
//! Starts world init, reaps its command tree, and applies host restart and shutdown deadlines.
|
||||
use super::{GroupGuard, Request, Runtime, Stdio};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
pub struct Supervisor {
|
||||
shutdown: tokio::sync::oneshot::Sender<()>,
|
||||
task: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
fn root() -> String {
|
||||
"/".into()
|
||||
impl Supervisor {
|
||||
pub async fn shutdown(self) {
|
||||
let _ = self.shutdown.send(());
|
||||
let _ = self.task.await;
|
||||
}
|
||||
}
|
||||
struct Live {
|
||||
child: Started,
|
||||
_guard: GroupGuard,
|
||||
}
|
||||
struct Service {
|
||||
spec: Spec,
|
||||
live: Option<Live>,
|
||||
crashes: VecDeque<Instant>,
|
||||
stopped: bool,
|
||||
}
|
||||
impl Service {
|
||||
async fn start(&mut self, runtime: &Arc<Runtime>) {
|
||||
match runtime
|
||||
.start(
|
||||
/// The host knows only init. Unit policy belongs to the WASI program.
|
||||
pub fn supervise(runtime: Arc<Runtime>) -> Supervisor {
|
||||
let (shutdown, mut stop) = tokio::sync::oneshot::channel();
|
||||
let task = tokio::spawn(async move {
|
||||
let mut backoff = Duration::from_secs(1);
|
||||
loop {
|
||||
let start = runtime.start(
|
||||
Request {
|
||||
program: self.spec.argv.first().cloned().unwrap_or_default(),
|
||||
argv: self.spec.argv.clone(),
|
||||
cwd: self.spec.cwd.clone(),
|
||||
env: self
|
||||
.spec
|
||||
.env
|
||||
.iter()
|
||||
.map(|(k, v)| (k.clone(), v.clone()))
|
||||
.collect(),
|
||||
program: "/bin/init.wasm".into(),
|
||||
argv: vec!["/bin/init.wasm".into()],
|
||||
cwd: "/".into(),
|
||||
env: Vec::new(),
|
||||
group: None,
|
||||
stdin: Stdio::Pipe,
|
||||
stdout: Stdio::Null,
|
||||
@@ -54,116 +33,56 @@ impl Service {
|
||||
0,
|
||||
Vec::new(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(child) => {
|
||||
);
|
||||
let child = tokio::select! { _ = &mut stop => return, child = start => child };
|
||||
if let Ok(mut child) = child {
|
||||
let guard = GroupGuard {
|
||||
runtime: runtime.clone(),
|
||||
group: child.pid,
|
||||
};
|
||||
self.live = Some(Live {
|
||||
child,
|
||||
_guard: guard,
|
||||
});
|
||||
let exiting = async {
|
||||
while child.code.borrow_and_update().is_none() {
|
||||
if child.code.changed().await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
};
|
||||
let shutdown = tokio::select! { _ = &mut stop => true, _ = exiting => false };
|
||||
if shutdown {
|
||||
drop(child.input.take());
|
||||
let _ = tokio::time::timeout(Duration::from_secs(15), async {
|
||||
while child.code.borrow_and_update().is_none() {
|
||||
if child.code.changed().await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
drop(guard);
|
||||
reaped(&runtime, child.pid).await;
|
||||
return;
|
||||
}
|
||||
// This guard kills all descendants, including separate process groups.
|
||||
drop(guard);
|
||||
reaped(&runtime, child.pid).await;
|
||||
}
|
||||
Err(_) => self.crashed(),
|
||||
tokio::select! { _ = &mut stop => return, _ = tokio::time::sleep(backoff) => () }
|
||||
backoff = (backoff * 2).min(Duration::from_secs(30));
|
||||
}
|
||||
}
|
||||
fn crashed(&mut self) {
|
||||
let now = Instant::now();
|
||||
while self
|
||||
.crashes
|
||||
.front()
|
||||
.is_some_and(|t| now.duration_since(*t) >= Duration::from_secs(10))
|
||||
});
|
||||
Supervisor { shutdown, task }
|
||||
}
|
||||
async fn reaped(runtime: &Runtime, command: u32) {
|
||||
loop {
|
||||
if !runtime
|
||||
.entries
|
||||
.lock()
|
||||
.unwrap()
|
||||
.values()
|
||||
.any(|entry| entry.command == command)
|
||||
{
|
||||
self.crashes.pop_front();
|
||||
}
|
||||
self.crashes.push_back(now);
|
||||
self.stopped = self.crashes.len() > 5;
|
||||
}
|
||||
async fn stop(&mut self) {
|
||||
if let Some(mut live) = self.live.take() {
|
||||
drop(live.child.input.take());
|
||||
let _ = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if live.child.code.borrow().is_some() {
|
||||
break;
|
||||
}
|
||||
if live.child.code.changed().await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
// The guard stops remaining descendants, or the process after its grace period.
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
/// Start the supervisor. Aborting its task stops its service processes.
|
||||
/// The YAML file contains name, argv, optional cwd, and an optional map of environment variables.
|
||||
pub fn supervise(runtime: Arc<Runtime>) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
let mut services: BTreeMap<String, Service> = BTreeMap::new();
|
||||
let mut previous = Vec::new();
|
||||
let mut interval = tokio::time::interval(Duration::from_millis(100));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
let content = match runtime
|
||||
.world
|
||||
.root
|
||||
.open("/config/services.yaml", Default::default())
|
||||
.await
|
||||
{
|
||||
Ok(file) => match file.stat().await {
|
||||
Ok(stat) => file.read(0, stat.size as usize).await.ok(),
|
||||
Err(_) => None,
|
||||
},
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Some(b"[]".to_vec()),
|
||||
Err(_) => None,
|
||||
};
|
||||
if let Some(content) = content.filter(|c| *c != previous)
|
||||
&& let Ok(specs) = serde_yaml_ng::from_slice::<Vec<Spec>>(&content)
|
||||
{
|
||||
let specs: BTreeMap<_, _> =
|
||||
specs.into_iter().map(|s| (s.name.clone(), s)).collect();
|
||||
let removed: Vec<_> = services
|
||||
.keys()
|
||||
.filter(|name| !specs.contains_key(*name))
|
||||
.cloned()
|
||||
.collect();
|
||||
for name in removed {
|
||||
services.remove(&name).unwrap().stop().await;
|
||||
}
|
||||
for (name, spec) in specs {
|
||||
let service = services.entry(name).or_insert_with(|| Service {
|
||||
spec: spec.clone(),
|
||||
live: None,
|
||||
crashes: VecDeque::new(),
|
||||
stopped: false,
|
||||
});
|
||||
if service.spec != spec {
|
||||
service.stop().await;
|
||||
}
|
||||
service.spec = spec;
|
||||
service.crashes.clear();
|
||||
service.stopped = false;
|
||||
}
|
||||
previous = content;
|
||||
}
|
||||
for service in services.values_mut() {
|
||||
if service
|
||||
.live
|
||||
.as_ref()
|
||||
.is_some_and(|live| live.child.code.borrow().is_some())
|
||||
{
|
||||
service.live = None;
|
||||
service.crashed();
|
||||
}
|
||||
if service.live.is_none() && !service.stopped {
|
||||
service.start(&runtime).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user