surma/sausage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Supervise crashed boot service lifetimes with a latched crash limit

Browse Source 
Keep one registration and restart owner across process cleanup and upgrades.
Restart on failed process exit before or after readiness; give up after more
than five crashes in a sliding ten-second window. Respect explicit stop and
shutdown, and suppress boot-service fallbacks from retired hooks.

Cover deterministic lifetime, boundary, give-up and admission races, plus
real WASI failures on both sides of readiness. Document the host contract.

Card: SAU-358
This commit is contained in:
SAU-358 implementer
2026-10-30 19:49:06 +00:00
parent c04baee67c
commit a953d80ce6
9 changed files with 646 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/src/upgrades.rs
+25 -41
View File
@@ -1,5 +1,5 @@
//! Grace periods, service restarts, and retirement of replaced versions (PROTOCOL.md section 3.7).
use crate::{Host, services::Service};
use crate::Host;
use serde_json::json;
use std::{collections::BTreeMap, sync::atomic::Ordering, time::Duration};
use tokio::sync::watch;
@@ -15,7 +15,6 @@ pub(crate) struct Upgrade {
current: String,
grace: Duration,
processes: Vec<OldProcess>,
services: Vec<(Service, Option<watch::Receiver<bool>>)>,
}
impl Host {
/// Snapshots old admissions while activation holds the admission write guard.
@@ -29,49 +28,45 @@ impl Host {
let mut state = self.0.state.lock().unwrap();
let processes = state
.processes
.iter()
.iter_mut()
.filter(|(_, process)| {
process.info.plugin == plugin
&& process.info.version != current
&& process.info.method != "rpc.discover"
})
.map(|(pid, process)| OldProcess {
pid: pid.clone(),
version: process.info.version.clone(),
ended: process.ended.subscribe(),
.map(|(pid, process)| {
// The service supervisor owns the restart, including an exit on
// the superseded notice before the grace timer is installed.
process.superseded = !process.stopped;
OldProcess {
pid: pid.clone(),
version: process.info.version.clone(),
ended: process.ended.subscribe(),
}
})
.collect();
let mut services = Vec::new();
let mut retained = Vec::new();
for service in std::mem::take(&mut state.services) {
if service
.method
.split_once('.')
.is_some_and(|(name, _)| name == plugin)
// A cleanly returned service is dormant until an upgrade. A pending
// admission will choose the current version itself. Disabled services
// (explicit stop or crash-limit give-up) must never be re-armed here.
for service in &mut state.services {
if service.enabled
&& !service.pending
&& service.owner.is_none()
&& service.version.as_deref() != Some(current)
&& service
.method
.split_once('.')
.is_some_and(|(name, _)| name == plugin)
{
let process = state
.processes
.values()
.find(|process| process.id.as_ref() == Some(&service.request));
if process.is_some_and(|process| process.info.version == current)
|| (process.is_none() && service.starting.load(Ordering::SeqCst))
{
retained.push(service);
} else {
services.push((service, process.map(|process| process.ended.subscribe())));
}
} else {
retained.push(service);
self.queue_service(service);
}
}
state.services = retained;
Upgrade {
plugin: plugin.into(),
previous,
current: current.into(),
grace: Duration::from_secs_f64(grace),
processes,
services,
}
}
/// Lets old calls finish, stops them after grace, and starts each lifecycle hook once.
@@ -107,18 +102,7 @@ impl Host {
let mut ended = process.ended;
tokio::select! {
_ = ended.wait_for(|ended|*ended) => (),
_ = tokio::time::sleep(grace) => { host.begin_stop(&process.pid); }
}
});
}
for (service, ended) in upgrade.services {
let host = host.clone();
tokio::spawn(async move {
if let Some(mut ended) = ended {
let _ = ended.wait_for(|ended| *ended).await;
}
if !host.0.shutdown.load(Ordering::SeqCst) {
host.start_service(service.method, service.params).await;
_ = tokio::time::sleep(grace) => { host.begin_retire(&process.pid); }
}
});
}