Supervise crashed boot service lifetimes with a latched crash limit
Keep one registration and restart owner across process cleanup and upgrades. Restart on failed process exit before or after readiness; give up after more than five crashes in a sliding ten-second window. Respect explicit stop and shutdown, and suppress boot-service fallbacks from retired hooks. Cover deterministic lifetime, boundary, give-up and admission races, plus real WASI failures on both sides of readiness. Document the host contract. Card: SAU-358
This commit is contained in:
SAU-358 implementer
@@ -0,0 +1,362 @@
|
||||
//! Deterministic service supervision tests: native tasks stand in for guest lifetimes.
|
||||
use super::*;
|
||||
use crate::{NativePlugin, runtime::Runtime};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::{mpsc, oneshot, watch};
|
||||
|
||||
struct Rig {
|
||||
host: Host,
|
||||
starts: mpsc::UnboundedReceiver<(String, oneshot::Sender<()>)>,
|
||||
_world: tempfile::TempDir,
|
||||
}
|
||||
impl Rig {
|
||||
async fn new() -> Self {
|
||||
let world = tempfile::tempdir().unwrap();
|
||||
let runtime = Runtime::new(world.path()).unwrap();
|
||||
let host = Host::test_world(world.path(), runtime).await.unwrap();
|
||||
let (tx, starts) = mpsc::unbounded_channel();
|
||||
let plugin: NativePlugin = Arc::new(move |ctx, method, _| {
|
||||
let tx = tx.clone();
|
||||
Box::pin(async move {
|
||||
if method != "serve" {
|
||||
return Ok(Value::Null);
|
||||
}
|
||||
let (end, ended) = oneshot::channel();
|
||||
tx.send((ctx.pid, end)).unwrap();
|
||||
let _ = ended.await;
|
||||
Ok(Value::Null)
|
||||
})
|
||||
});
|
||||
host.register_native("demo", plugin.clone());
|
||||
host.register_native("other", plugin);
|
||||
Self {
|
||||
host,
|
||||
starts,
|
||||
_world: world,
|
||||
}
|
||||
}
|
||||
async fn start(&mut self, method: &str) -> (String, oneshot::Sender<()>) {
|
||||
let host = self.host.clone();
|
||||
let method = method.to_string();
|
||||
tokio::spawn(async move {
|
||||
host.start_service(method, json!({"kept":true})).await;
|
||||
});
|
||||
self.next().await
|
||||
}
|
||||
async fn next(&mut self) -> (String, oneshot::Sender<()>) {
|
||||
// Reuse the host integration call bound (activation.rs::call).
|
||||
tokio::time::timeout(Duration::from_secs(60), self.starts.recv())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
}
|
||||
fn ended(&self, pid: &str) -> watch::Receiver<bool> {
|
||||
self.host.0.state.lock().unwrap().processes[pid]
|
||||
.ended
|
||||
.subscribe()
|
||||
}
|
||||
async fn crash(&self, pid: &str) {
|
||||
let mut ended = self.ended(pid);
|
||||
// Abort execution without asking the supervisor for an intentional stop.
|
||||
self.host.0.state.lock().unwrap().processes[pid]
|
||||
.abort
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.abort();
|
||||
ended.wait_for(|done| *done).await.unwrap();
|
||||
}
|
||||
async fn clean(&self, pid: &str, end: oneshot::Sender<()>) {
|
||||
let mut ended = self.ended(pid);
|
||||
end.send(()).unwrap();
|
||||
ended.wait_for(|done| *done).await.unwrap();
|
||||
}
|
||||
fn assert_single(&self, pid: &str, registrations: usize) {
|
||||
let state = self.host.0.state.lock().unwrap();
|
||||
assert_eq!(state.services.len(), registrations);
|
||||
let p = &state.processes[pid];
|
||||
assert_eq!(p.info.params, json!({"kept":true}));
|
||||
assert!(p.info.detached);
|
||||
assert_eq!(p.caller, "core");
|
||||
assert_eq!(
|
||||
state
|
||||
.services
|
||||
.iter()
|
||||
.filter(|s| s.owner.as_deref() == Some(pid))
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
}
|
||||
fn no_start(&mut self) {
|
||||
assert!(self.starts.try_recv().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn crash_before_and_after_readiness_observes_process_not_response() {
|
||||
let mut rig = Rig::new().await;
|
||||
let (pid, _end) = rig.start("demo.serve").await;
|
||||
rig.assert_single(&pid, 1);
|
||||
rig.crash(&pid).await; // No ready RPC response was sent.
|
||||
let (pid, end) = rig.next().await;
|
||||
rig.assert_single(&pid, 1);
|
||||
rig.host.answer(&pid, Ok(json!({"ready":true})));
|
||||
tokio::task::yield_now().await;
|
||||
rig.no_start(); // Ready does not end the service lifetime.
|
||||
rig.crash(&pid).await;
|
||||
drop(end);
|
||||
let (pid, end) = rig.next().await;
|
||||
rig.assert_single(&pid, 1);
|
||||
rig.clean(&pid, end).await;
|
||||
rig.no_start(); // A clean, intentional return is not a crash.
|
||||
rig.host.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn fifth_restarts_sixth_gives_up_and_journals_once() {
|
||||
let mut rig = Rig::new().await;
|
||||
let (mut pid, mut end) = rig.start("demo.serve").await;
|
||||
for crash in 1..=CRASH_LIMIT {
|
||||
rig.crash(&pid).await;
|
||||
drop(end);
|
||||
(pid, end) = rig.next().await;
|
||||
rig.assert_single(&pid, 1);
|
||||
assert_eq!(
|
||||
rig.host.0.state.lock().unwrap().services[0].crashes.len(),
|
||||
crash
|
||||
);
|
||||
}
|
||||
rig.crash(&pid).await;
|
||||
drop(end);
|
||||
rig.no_start();
|
||||
let logs = rig
|
||||
.host
|
||||
.call("core.log", json!({"grep":"service gave up"}))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(logs.as_array().unwrap().len(), 1);
|
||||
assert!(
|
||||
logs[0]["line"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.contains("more than 5 crashes within 10 seconds")
|
||||
);
|
||||
assert!(!rig.host.0.state.lock().unwrap().services[0].enabled);
|
||||
// A retired hook cannot bypass give-up while its supervisor has no owner.
|
||||
rig.host.register_native(
|
||||
"fallback",
|
||||
Arc::new(move |ctx, _, _| {
|
||||
Box::pin(async move {
|
||||
ctx.call(
|
||||
"core.call",
|
||||
json!({"method":"demo.serve","params":{"kept":true},"detach":true}),
|
||||
)
|
||||
.await
|
||||
})
|
||||
}),
|
||||
);
|
||||
assert_eq!(
|
||||
rig.host
|
||||
.call("fallback.rpc.retired", json!({}))
|
||||
.await
|
||||
.unwrap(),
|
||||
Value::Null
|
||||
);
|
||||
rig.no_start();
|
||||
// Neither time passing nor an upgrade re-arms a given-up service.
|
||||
tokio::time::advance(CRASH_WINDOW).await;
|
||||
let upgrade = rig.host.snapshot_upgrade("demo", "old".into(), "new", 0.0);
|
||||
rig.host.begin_upgrade(upgrade);
|
||||
tokio::task::yield_now().await;
|
||||
rig.no_start();
|
||||
rig.host.stop().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sliding_window_boundary_and_progress() {
|
||||
let now = Instant::now();
|
||||
let service = || Service {
|
||||
method: "demo.serve".into(),
|
||||
params: json!({}),
|
||||
request: json!("request"),
|
||||
owner: None,
|
||||
enabled: true,
|
||||
pending: false,
|
||||
version: None,
|
||||
crashes: VecDeque::new(),
|
||||
};
|
||||
let mut before = service();
|
||||
let mut exact = service();
|
||||
let mut after = service();
|
||||
for _ in 0..CRASH_LIMIT {
|
||||
assert!(before.crashed(now));
|
||||
assert!(exact.crashed(now));
|
||||
assert!(after.crashed(now));
|
||||
}
|
||||
assert!(!before.crashed(now + CRASH_WINDOW - Duration::from_nanos(1)));
|
||||
assert!(exact.crashed(now + CRASH_WINDOW));
|
||||
assert_eq!(exact.crashes.len(), 1);
|
||||
assert!(after.crashed(now + CRASH_WINDOW + Duration::from_nanos(1)));
|
||||
// Repeated progress beyond the window never accumulates old failures.
|
||||
for step in 2..=CRASH_LIMIT as u32 + 1 {
|
||||
assert!(exact.crashed(now + CRASH_WINDOW * step));
|
||||
assert_eq!(exact.crashes.len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn services_have_independent_crash_budgets() {
|
||||
let mut rig = Rig::new().await;
|
||||
let (mut a, mut a_end) = rig.start("demo.serve").await;
|
||||
let (b, b_end) = rig.start("other.serve").await;
|
||||
for _ in 0..CRASH_LIMIT {
|
||||
rig.crash(&a).await;
|
||||
drop(a_end);
|
||||
(a, a_end) = rig.next().await;
|
||||
}
|
||||
rig.crash(&a).await;
|
||||
drop(a_end);
|
||||
rig.crash(&b).await;
|
||||
drop(b_end);
|
||||
let (b, b_end) = rig.next().await;
|
||||
rig.assert_single(&b, 2);
|
||||
rig.clean(&b, b_end).await;
|
||||
rig.no_start();
|
||||
rig.host.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn upgrade_crash_race_has_one_restart_and_retired_fallback_cannot_duplicate_it() {
|
||||
let mut rig = Rig::new().await;
|
||||
let (pid, end) = rig.start("demo.serve").await;
|
||||
let upgrade = rig.host.snapshot_upgrade("demo", "old".into(), "new", 0.0);
|
||||
rig.crash(&pid).await; // Exits between snapshot and begin_upgrade.
|
||||
drop(end);
|
||||
rig.host.begin_upgrade(upgrade);
|
||||
let (pid, end) = rig.next().await;
|
||||
rig.assert_single(&pid, 1);
|
||||
assert!(
|
||||
rig.host.0.state.lock().unwrap().services[0]
|
||||
.crashes
|
||||
.is_empty()
|
||||
);
|
||||
// A native retired hook exercises the same routing path as a guest fallback.
|
||||
rig.host.register_native(
|
||||
"fallback",
|
||||
Arc::new(move |ctx, _, _| {
|
||||
Box::pin(async move {
|
||||
ctx.call(
|
||||
"core.call",
|
||||
json!({"method":"demo.serve","params":{"kept":true},"detach":true}),
|
||||
)
|
||||
.await
|
||||
})
|
||||
}),
|
||||
);
|
||||
assert_eq!(
|
||||
rig.host
|
||||
.call("fallback.rpc.retired", json!({}))
|
||||
.await
|
||||
.unwrap(),
|
||||
Value::Null
|
||||
);
|
||||
rig.no_start();
|
||||
rig.clean(&pid, end).await;
|
||||
rig.host.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn crash_before_upgrade_snapshot_also_has_one_restart() {
|
||||
let mut rig = Rig::new().await;
|
||||
let (pid, end) = rig.start("demo.serve").await;
|
||||
rig.crash(&pid).await;
|
||||
drop(end);
|
||||
// Restart may be pending admission or already admitted: snapshot handles both.
|
||||
let upgrade = rig
|
||||
.host
|
||||
.snapshot_upgrade("demo", "old".into(), "native", 0.0);
|
||||
rig.host.begin_upgrade(upgrade);
|
||||
let (pid, end) = rig.next().await;
|
||||
rig.assert_single(&pid, 1);
|
||||
rig.no_start();
|
||||
rig.clean(&pid, end).await;
|
||||
rig.host.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn pending_admission_is_single_owned_across_upgrade() {
|
||||
let mut rig = Rig::new().await;
|
||||
let (pid, end) = rig.start("demo.serve").await;
|
||||
let host = rig.host.clone();
|
||||
let admission = host.0.admission.write().await;
|
||||
rig.crash(&pid).await;
|
||||
drop(end);
|
||||
let upgrade = rig
|
||||
.host
|
||||
.snapshot_upgrade("demo", "old".into(), "native", 0.0);
|
||||
rig.host.begin_upgrade(upgrade);
|
||||
assert!(rig.host.0.state.lock().unwrap().services[0].pending);
|
||||
drop(admission);
|
||||
let (pid, end) = rig.next().await;
|
||||
rig.assert_single(&pid, 1);
|
||||
rig.no_start();
|
||||
rig.clean(&pid, end).await;
|
||||
rig.host.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn shutdown_rejects_restart_already_waiting_for_admission() {
|
||||
let mut rig = Rig::new().await;
|
||||
let (pid, end) = rig.start("demo.serve").await;
|
||||
let host = rig.host.clone();
|
||||
let admission = host.0.admission.write().await;
|
||||
rig.crash(&pid).await;
|
||||
drop(end);
|
||||
rig.host.stop().await;
|
||||
drop(admission);
|
||||
tokio::task::yield_now().await;
|
||||
rig.no_start();
|
||||
assert!(rig.host.processes().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn clean_service_stays_dormant_until_upgrade_and_explicit_stop_stays_disabled() {
|
||||
let mut rig = Rig::new().await;
|
||||
let (pid, end) = rig.start("demo.serve").await;
|
||||
rig.clean(&pid, end).await;
|
||||
rig.no_start();
|
||||
let upgrade = rig.host.snapshot_upgrade("demo", "old".into(), "new", 0.0);
|
||||
rig.host.begin_upgrade(upgrade);
|
||||
let (pid, end) = rig.next().await;
|
||||
rig.assert_single(&pid, 1);
|
||||
// An explicit stop stays disabled even if the task subsequently exits cleanly.
|
||||
let mut ended = rig.host.begin_stop(&pid).unwrap();
|
||||
end.send(()).unwrap();
|
||||
ended.wait_for(|done| *done).await.unwrap();
|
||||
let upgrade = rig
|
||||
.host
|
||||
.snapshot_upgrade("demo", "old".into(), "newer", 0.0);
|
||||
rig.host.begin_upgrade(upgrade);
|
||||
tokio::task::yield_now().await;
|
||||
rig.no_start();
|
||||
rig.host.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn intentional_stop_and_shutdown_never_restart_even_during_upgrade() {
|
||||
let mut rig = Rig::new().await;
|
||||
let (pid, end) = rig.start("demo.serve").await;
|
||||
let upgrade = rig.host.snapshot_upgrade("demo", "old".into(), "new", 0.0);
|
||||
// An explicit kill after supersession overrides the automatic upgrade restart.
|
||||
rig.host
|
||||
.call("core.kill", json!({"pid":pid}))
|
||||
.await
|
||||
.unwrap();
|
||||
drop(end);
|
||||
rig.host.begin_upgrade(upgrade);
|
||||
tokio::task::yield_now().await;
|
||||
rig.no_start();
|
||||
let (_pid, end) = rig.start("other.serve").await;
|
||||
rig.host.stop().await;
|
||||
drop(end);
|
||||
rig.no_start();
|
||||
}
|
||||
Reference in New Issue
Block a user