surma/sausage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge '6649f4ca51a535632b3986c48e6f42ee0224d429'

Browse Source
This commit is contained in:
surma
2026-10-31 01:17:15 +00:00
parent 95b27f0948 6649f4ca51
commit b696e57dc5
15 changed files with 943 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/src/activation.rs
-5
View File
@@ -279,15 +279,10 @@ impl Host {
.get(&candidate.plugin)
.filter(|old| !old.hash.is_empty() && old.hash != candidate.interface.hash)
{
let grace = old.document["x-sausage-grace"]
.as_f64()
.filter(|value| value.is_finite() && *value >= 0.0)
.unwrap_or(60.0);
upgrades.push(self.snapshot_upgrade(
&candidate.plugin,
old.hash.clone(),
&candidate.interface.hash,
grace,
));
}
self.0.journal.write(
cli/src/graph.rs
+1 -10
View File
@@ -223,16 +223,7 @@ impl Host {
if let Some(old) = old_plugins.get(plugin).filter(|old| {
!old.hash.is_empty() && old.hash != current.hash && current.error.is_none()
}) {
let grace = old.document["x-sausage-grace"]
.as_f64()
.filter(|value| value.is_finite() && *value >= 0.0)
.unwrap_or(60.0);
upgrades.push(self.snapshot_upgrade(
plugin,
old.hash.clone(),
&current.hash,
grace,
));
upgrades.push(self.snapshot_upgrade(plugin, old.hash.clone(), &current.hash));
}
}
self.publish_graph(plugins);
cli/src/process.rs
+22 -1
View File
@@ -33,6 +33,8 @@ pub(crate) struct Process {
pub(crate) id: Option<Value>,
pub(crate) answer: Option<oneshot::Sender<RpcResult>>,
pub(crate) ended: watch::Sender<bool>,
pub(crate) stopped: bool,
pub(crate) superseded: bool,
}
impl Host {
/// Called only after the execution task ends. Wait for the store and file operations before releasing locks.
@@ -57,6 +59,7 @@ impl Host {
.map(|_| self.0.journal.query_later(json!({"pid":pid,"limit":20})));
self.0.journal.forget(pid);
Self::release_locks(&mut state, pid);
self.service_ended(&mut state, pid, status, p.stopped, p.superseded);
let status = status.to_string();
tokio::spawn(async move {
if let Some(answer) = p.answer {
@@ -84,13 +87,24 @@ impl Host {
}
/// Close stdin and start the same five-second deadline for each attached child.
pub(crate) fn begin_stop(&self, pid: &str) -> Option<watch::Receiver<bool>> {
self.stop_process(pid, false)
}
/// Grace expiry is retirement, not an operator stopping the service.
pub(crate) fn begin_retire(&self, pid: &str) -> Option<watch::Receiver<bool>> {
self.stop_process(pid, true)
}
fn stop_process(&self, pid: &str, retirement: bool) -> Option<watch::Receiver<bool>> {
let mut state = self.0.state.lock().unwrap();
let p = state.processes.get_mut(pid)?;
if !retirement {
p.superseded = false;
}
let ended = p.ended.subscribe();
if p.info.state == "stopping" {
return Some(ended);
}
p.info.state = "stopping".into();
p.stopped = true;
p.input.take();
self.0.journal.write(pid, &p.info.plugin, "core", "stop");
for lock in state.locks.values_mut() {
@@ -110,17 +124,24 @@ impl Host {
let pid = pid.to_string();
tokio::spawn(async move {
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
h.kill(&pid);
h.kill_process(&pid, retirement);
});
Some(ended)
}
/// Abort a process task and perform its ended-state cleanup.
pub(crate) fn kill(&self, pid: &str) {
self.kill_process(pid, false);
}
fn kill_process(&self, pid: &str, retirement: bool) {
let mut state = self.0.state.lock().unwrap();
for lock in state.locks.values_mut() {
lock.queue.retain(|(queued, _)| queued != pid);
}
if let Some(p) = state.processes.get_mut(pid) {
if !retirement {
p.superseded = false;
}
p.stopped = true;
p.info.state = "stopping".into();
p.input.take();
self.0.journal.write(pid, &p.info.plugin, "core", "kill");
cli/src/routing.rs
+29
View File
@@ -171,6 +171,33 @@ impl Host {
{
return Err(error(-32600, "caller stopped or ended"));
}
// Older plugins may still try to start their boot service from
// rpc.retired. The supervisor owns it even while a restart is
// pending, or after it has given up. Do not create a second owner.
if state
.processes
.get(&caller)
.is_some_and(|p| p.info.method == "rpc.retired")
&& state
.services
.iter()
.any(|s| s.method == format!("{plugin}.{method}") && s.params == params)
{
return Ok(Value::Null);
}
if caller == "core"
&& let Some(service) = state
.services
.iter_mut()
.find(|s| Some(&s.request) == id.as_ref())
{
if !service.enabled || service.owner.is_some() {
return Err(error(-32600, "service attempt is no longer current"));
}
service.owner = Some(pid.clone());
service.pending = false;
service.version = Some(info.version.clone());
}
host.0
.journal
.write(&pid, plugin, "core", &format!("start {method} {params}"));
@@ -187,6 +214,8 @@ impl Host {
id: id.clone(),
answer: if id.is_some() { Some(tx) } else { None },
ended: watch::channel(false).0,
stopped: false,
superseded: false,
},
);
}
cli/src/services.rs
+132 -19
View File
@@ -1,12 +1,36 @@
//! Ordered boot services and their identities for upgrade restarts (PROTOCOL.md section 8).
use crate::Host;
//! Ordered boot calls and the single restart owner for each service (PROTOCOL.md section 8).
use crate::{Host, host::State};
use serde_json::{Value, json};
#[derive(Clone)]
use std::{collections::VecDeque, sync::atomic::Ordering, time::Duration};
use tokio::time::Instant;
// Agreed systemd-derived values: /docs/design/plugin-handover/design.md,
// https://www.man7.org/linux/man-pages/man5/systemd-system.conf.5.html.
const CRASH_LIMIT: usize = 5;
const CRASH_WINDOW: Duration = Duration::from_secs(10);
pub(crate) struct Service {
pub(crate) method: String,
pub(crate) params: Value,
pub(crate) request: Value,
pub(crate) starting: std::sync::Arc<std::sync::atomic::AtomicBool>,
pub(crate) owner: Option<String>,
pub(crate) pending: bool,
pub(crate) version: Option<String>,
pub(crate) enabled: bool,
crashes: VecDeque<Instant>,
}
impl Service {
fn crashed(&mut self, now: Instant) -> bool {
while self
.crashes
.front()
.is_some_and(|t| now.duration_since(*t) >= CRASH_WINDOW)
{
self.crashes.pop_front();
}
self.crashes.push_back(now);
self.crashes.len() <= CRASH_LIMIT
}
}
impl Host {
/// Starts discovery in the background, then waits for each boot service in order.
@@ -24,30 +48,119 @@ impl Host {
}
Ok(())
}
/// Records a unique request identity before the service can answer or exit.
/// Register once, before admission or readiness. Restarts reuse this registration.
pub(crate) async fn start_service(&self, method: String, params: Value) {
if self.0.shutdown.load(std::sync::atomic::Ordering::SeqCst) {
return;
}
let request = json!(uuid::Uuid::now_v7().to_string());
let starting = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true));
self.0.state.lock().unwrap().services.push(Service {
method: method.clone(),
params: params.clone(),
request: request.clone(),
starting: starting.clone(),
});
{
let mut state = self.0.state.lock().unwrap();
if self.0.shutdown.load(Ordering::SeqCst) {
return;
}
state.services.push(Service {
method,
params,
request: request.clone(),
owner: None,
enabled: true,
pending: true,
version: None,
crashes: VecDeque::new(),
});
}
self.run_service(request).await;
}
/// Await readiness only; process cleanup, not this response, decides whether to restart.
async fn run_service(&self, request: Value) {
let (method, params) = {
let state = self.0.state.lock().unwrap();
if self.0.shutdown.load(Ordering::SeqCst) {
return;
}
let Some(service) = state
.services
.iter()
.find(|s| s.request == request && s.enabled)
else {
return;
};
(service.method.clone(), service.params.clone())
};
self.0
.journal
.write("core", "core", "core", &format!("service start {method}"));
if let Err(error) = self
.route("core".into(), Some(request), method, params, true)
.await
let result = self
.route("core".into(), Some(request.clone()), method, params, true)
.await;
{
let mut state = self.0.state.lock().unwrap();
if let Some(service) = state.services.iter_mut().find(|s| s.request == request) {
service.pending = false;
}
}
if let Err(error) = result {
// Admission failures have no process lifetime to supervise. A process crash
// before readiness is handled by finish just like one after readiness.
self.0
.journal
.write("core", "core", "core", &format!("service failed {error}"));
}
starting.store(false, std::sync::atomic::Ordering::SeqCst);
}
/// Called under the state lock after the old owner and its files/locks are gone.
pub(crate) fn service_ended(
&self,
state: &mut State,
pid: &str,
status: &str,
stopped: bool,
superseded: bool,
) {
let Some(service) = state
.services
.iter_mut()
.find(|s| s.owner.as_deref() == Some(pid))
else {
return;
};
service.owner = None;
if self.0.shutdown.load(Ordering::SeqCst) || !service.enabled {
return;
}
if !superseded {
if stopped {
service.enabled = false;
return;
}
if status == "exit 0" {
return;
}
if !service.crashed(Instant::now()) {
service.enabled = false;
self.0.journal.write(
"core",
"core",
"core",
&format!(
"service gave up {}: more than {CRASH_LIMIT} crashes within {} seconds",
service.method,
CRASH_WINDOW.as_secs(),
),
);
return;
}
}
self.queue_service(service);
}
/// Reserve the next attempt before spawning it; upgrades and crashes share this owner.
pub(crate) fn queue_service(&self, service: &mut Service) {
service.request = json!(uuid::Uuid::now_v7().to_string());
service.pending = true;
let request = service.request.clone();
let host = self.clone();
tokio::spawn(async move {
host.run_service(request).await;
});
}
}
#[cfg(test)]
mod tests;
cli/src/services/tests.rs
+356
View File
@@ -0,0 +1,356 @@
//! Deterministic service supervision tests: native tasks stand in for guest lifetimes.
use super::*;
use crate::{NativePlugin, runtime::Runtime};
use std::sync::Arc;
use tokio::sync::{mpsc, oneshot, watch};
struct Rig {
host: Host,
starts: mpsc::UnboundedReceiver<(String, oneshot::Sender<()>)>,
_world: tempfile::TempDir,
}
impl Rig {
async fn new() -> Self {
let world = tempfile::tempdir().unwrap();
let runtime = Runtime::new(world.path()).unwrap();
let host = Host::test_world(world.path(), runtime).await.unwrap();
let (tx, starts) = mpsc::unbounded_channel();
let plugin: NativePlugin = Arc::new(move |ctx, method, _| {
let tx = tx.clone();
Box::pin(async move {
if method != "serve" {
return Ok(Value::Null);
}
let (end, ended) = oneshot::channel();
tx.send((ctx.pid, end)).unwrap();
let _ = ended.await;
Ok(Value::Null)
})
});
host.register_native("demo", plugin.clone());
host.register_native("other", plugin);
Self {
host,
starts,
_world: world,
}
}
async fn start(&mut self, method: &str) -> (String, oneshot::Sender<()>) {
let host = self.host.clone();
let method = method.to_string();
tokio::spawn(async move {
host.start_service(method, json!({"kept":true})).await;
});
self.next().await
}
async fn next(&mut self) -> (String, oneshot::Sender<()>) {
// Reuse the host integration call bound (activation.rs::call).
tokio::time::timeout(Duration::from_secs(60), self.starts.recv())
.await
.unwrap()
.unwrap()
}
fn ended(&self, pid: &str) -> watch::Receiver<bool> {
self.host.0.state.lock().unwrap().processes[pid]
.ended
.subscribe()
}
async fn crash(&self, pid: &str) {
let mut ended = self.ended(pid);
// Abort execution without asking the supervisor for an intentional stop.
self.host.0.state.lock().unwrap().processes[pid]
.abort
.as_ref()
.unwrap()
.abort();
ended.wait_for(|done| *done).await.unwrap();
}
async fn clean(&self, pid: &str, end: oneshot::Sender<()>) {
let mut ended = self.ended(pid);
end.send(()).unwrap();
ended.wait_for(|done| *done).await.unwrap();
}
fn assert_single(&self, pid: &str, registrations: usize) {
let state = self.host.0.state.lock().unwrap();
assert_eq!(state.services.len(), registrations);
let p = &state.processes[pid];
assert_eq!(p.info.params, json!({"kept":true}));
assert!(p.info.detached);
assert_eq!(p.caller, "core");
assert_eq!(
state
.services
.iter()
.filter(|s| s.owner.as_deref() == Some(pid))
.count(),
1
);
}
fn no_start(&mut self) {
assert!(self.starts.try_recv().is_err());
}
}
#[tokio::test(start_paused = true)]
async fn crash_before_and_after_readiness_observes_process_not_response() {
let mut rig = Rig::new().await;
let (pid, _end) = rig.start("demo.serve").await;
rig.assert_single(&pid, 1);
rig.crash(&pid).await; // No ready RPC response was sent.
let (pid, end) = rig.next().await;
rig.assert_single(&pid, 1);
rig.host.answer(&pid, Ok(json!({"ready":true})));
tokio::task::yield_now().await;
rig.no_start(); // Ready does not end the service lifetime.
rig.crash(&pid).await;
drop(end);
let (pid, end) = rig.next().await;
rig.assert_single(&pid, 1);
rig.clean(&pid, end).await;
rig.no_start(); // A clean, intentional return is not a crash.
rig.host.stop().await;
}
#[tokio::test(start_paused = true)]
async fn fifth_restarts_sixth_gives_up_and_journals_once() {
let mut rig = Rig::new().await;
let (mut pid, mut end) = rig.start("demo.serve").await;
for crash in 1..=CRASH_LIMIT {
rig.crash(&pid).await;
drop(end);
(pid, end) = rig.next().await;
rig.assert_single(&pid, 1);
assert_eq!(
rig.host.0.state.lock().unwrap().services[0].crashes.len(),
crash
);
}
rig.crash(&pid).await;
drop(end);
rig.no_start();
let logs = rig
.host
.call("core.log", json!({"grep":"service gave up"}))
.await
.unwrap();
assert_eq!(logs.as_array().unwrap().len(), 1);
assert!(
logs[0]["line"]
.as_str()
.unwrap()
.contains("more than 5 crashes within 10 seconds")
);
assert!(!rig.host.0.state.lock().unwrap().services[0].enabled);
// A retired hook cannot bypass give-up while its supervisor has no owner.
rig.host.register_native(
"fallback",
Arc::new(move |ctx, _, _| {
Box::pin(async move {
ctx.call(
"core.call",
json!({"method":"demo.serve","params":{"kept":true},"detach":true}),
)
.await
})
}),
);
assert_eq!(
rig.host
.call("fallback.rpc.retired", json!({}))
.await
.unwrap(),
Value::Null
);
rig.no_start();
// Neither time passing nor an upgrade re-arms a given-up service.
tokio::time::advance(CRASH_WINDOW).await;
let upgrade = rig.host.snapshot_upgrade("demo", "old".into(), "new");
rig.host.begin_upgrade(upgrade);
tokio::task::yield_now().await;
rig.no_start();
rig.host.stop().await;
}
#[test]
fn sliding_window_boundary_and_progress() {
let now = Instant::now();
let service = || Service {
method: "demo.serve".into(),
params: json!({}),
request: json!("request"),
owner: None,
enabled: true,
pending: false,
version: None,
crashes: VecDeque::new(),
};
let mut before = service();
let mut exact = service();
let mut after = service();
for _ in 0..CRASH_LIMIT {
assert!(before.crashed(now));
assert!(exact.crashed(now));
assert!(after.crashed(now));
}
assert!(!before.crashed(now + CRASH_WINDOW - Duration::from_nanos(1)));
assert!(exact.crashed(now + CRASH_WINDOW));
assert_eq!(exact.crashes.len(), 1);
assert!(after.crashed(now + CRASH_WINDOW + Duration::from_nanos(1)));
// Repeated progress beyond the window never accumulates old failures.
for step in 2..=CRASH_LIMIT as u32 + 1 {
assert!(exact.crashed(now + CRASH_WINDOW * step));
assert_eq!(exact.crashes.len(), 1);
}
}
#[tokio::test(start_paused = true)]
async fn services_have_independent_crash_budgets() {
let mut rig = Rig::new().await;
let (mut a, mut a_end) = rig.start("demo.serve").await;
let (b, b_end) = rig.start("other.serve").await;
for _ in 0..CRASH_LIMIT {
rig.crash(&a).await;
drop(a_end);
(a, a_end) = rig.next().await;
}
rig.crash(&a).await;
drop(a_end);
rig.crash(&b).await;
drop(b_end);
let (b, b_end) = rig.next().await;
rig.assert_single(&b, 2);
rig.clean(&b, b_end).await;
rig.no_start();
rig.host.stop().await;
}
#[tokio::test(start_paused = true)]
async fn upgrade_crash_race_has_one_restart_and_retired_fallback_cannot_duplicate_it() {
let mut rig = Rig::new().await;
let (pid, end) = rig.start("demo.serve").await;
let upgrade = rig.host.snapshot_upgrade("demo", "old".into(), "new");
rig.crash(&pid).await; // Exits between snapshot and begin_upgrade.
drop(end);
rig.host.begin_upgrade(upgrade);
let (pid, end) = rig.next().await;
rig.assert_single(&pid, 1);
assert!(
rig.host.0.state.lock().unwrap().services[0]
.crashes
.is_empty()
);
// A native retired hook exercises the same routing path as a guest fallback.
rig.host.register_native(
"fallback",
Arc::new(move |ctx, _, _| {
Box::pin(async move {
ctx.call(
"core.call",
json!({"method":"demo.serve","params":{"kept":true},"detach":true}),
)
.await
})
}),
);
assert_eq!(
rig.host
.call("fallback.rpc.retired", json!({}))
.await
.unwrap(),
Value::Null
);
rig.no_start();
rig.clean(&pid, end).await;
rig.host.stop().await;
}
#[tokio::test(start_paused = true)]
async fn crash_before_upgrade_snapshot_also_has_one_restart() {
let mut rig = Rig::new().await;
let (pid, end) = rig.start("demo.serve").await;
rig.crash(&pid).await;
drop(end);
// Restart may be pending admission or already admitted: snapshot handles both.
let upgrade = rig.host.snapshot_upgrade("demo", "old".into(), "native");
rig.host.begin_upgrade(upgrade);
let (pid, end) = rig.next().await;
rig.assert_single(&pid, 1);
rig.no_start();
rig.clean(&pid, end).await;
rig.host.stop().await;
}
#[tokio::test(start_paused = true)]
async fn pending_admission_is_single_owned_across_upgrade() {
let mut rig = Rig::new().await;
let (pid, end) = rig.start("demo.serve").await;
let host = rig.host.clone();
let admission = host.0.admission.write().await;
rig.crash(&pid).await;
drop(end);
let upgrade = rig.host.snapshot_upgrade("demo", "old".into(), "native");
rig.host.begin_upgrade(upgrade);
assert!(rig.host.0.state.lock().unwrap().services[0].pending);
drop(admission);
let (pid, end) = rig.next().await;
rig.assert_single(&pid, 1);
rig.no_start();
rig.clean(&pid, end).await;
rig.host.stop().await;
}
#[tokio::test(start_paused = true)]
async fn shutdown_rejects_restart_already_waiting_for_admission() {
let mut rig = Rig::new().await;
let (pid, end) = rig.start("demo.serve").await;
let host = rig.host.clone();
let admission = host.0.admission.write().await;
rig.crash(&pid).await;
drop(end);
rig.host.stop().await;
drop(admission);
tokio::task::yield_now().await;
rig.no_start();
assert!(rig.host.processes().is_empty());
}
#[tokio::test(start_paused = true)]
async fn clean_service_stays_dormant_until_upgrade_and_explicit_stop_stays_disabled() {
let mut rig = Rig::new().await;
let (pid, end) = rig.start("demo.serve").await;
rig.clean(&pid, end).await;
rig.no_start();
let upgrade = rig.host.snapshot_upgrade("demo", "old".into(), "new");
rig.host.begin_upgrade(upgrade);
let (pid, end) = rig.next().await;
rig.assert_single(&pid, 1);
// An explicit stop stays disabled even if the task subsequently exits cleanly.
let mut ended = rig.host.begin_stop(&pid).unwrap();
end.send(()).unwrap();
ended.wait_for(|done| *done).await.unwrap();
let upgrade = rig.host.snapshot_upgrade("demo", "old".into(), "newer");
rig.host.begin_upgrade(upgrade);
tokio::task::yield_now().await;
rig.no_start();
rig.host.stop().await;
}
#[tokio::test(start_paused = true)]
async fn intentional_stop_and_shutdown_never_restart_even_during_upgrade() {
let mut rig = Rig::new().await;
let (pid, end) = rig.start("demo.serve").await;
let upgrade = rig.host.snapshot_upgrade("demo", "old".into(), "new");
// An explicit kill after supersession overrides the automatic upgrade restart.
rig.host
.call("core.kill", json!({"pid":pid}))
.await
.unwrap();
drop(end);
rig.host.begin_upgrade(upgrade);
tokio::task::yield_now().await;
rig.no_start();
let (_pid, end) = rig.start("other.serve").await;
rig.host.stop().await;
drop(end);
rig.no_start();
}
cli/src/upgrades.rs
+32 -46
View File
@@ -1,9 +1,15 @@
//! Grace periods, service restarts, and retirement of replaced versions (PROTOCOL.md section 3.7).
use crate::{Host, services::Service};
use crate::Host;
use serde_json::json;
use std::{collections::BTreeMap, sync::atomic::Ordering, time::Duration};
use tokio::sync::watch;
/// Uniform host policy (PROTOCOL 3.7), independent of discovery metadata.
const UPGRADE_GRACE: Duration = Duration::from_secs(5);
#[cfg(test)]
mod tests;
pub(crate) struct OldProcess {
pid: String,
version: String,
@@ -13,9 +19,7 @@ pub(crate) struct Upgrade {
plugin: String,
previous: String,
current: String,
grace: Duration,
processes: Vec<OldProcess>,
services: Vec<(Service, Option<watch::Receiver<bool>>)>,
}
impl Host {
/// Snapshots old admissions while activation holds the admission write guard.
@@ -24,54 +28,48 @@ impl Host {
plugin: &str,
previous: String,
current: &str,
grace: f64,
) -> Upgrade {
let mut state = self.0.state.lock().unwrap();
let processes = state
.processes
.iter()
.iter_mut()
.filter(|(_, process)| {
process.info.plugin == plugin
&& process.info.version != current
&& process.info.method != "rpc.discover"
})
.map(|(pid, process)| OldProcess {
pid: pid.clone(),
version: process.info.version.clone(),
ended: process.ended.subscribe(),
.map(|(pid, process)| {
// The service supervisor owns the restart, including an exit on
// the superseded notice before the grace timer is installed.
process.superseded = !process.stopped;
OldProcess {
pid: pid.clone(),
version: process.info.version.clone(),
ended: process.ended.subscribe(),
}
})
.collect();
let mut services = Vec::new();
let mut retained = Vec::new();
for service in std::mem::take(&mut state.services) {
if service
.method
.split_once('.')
.is_some_and(|(name, _)| name == plugin)
// A cleanly returned service is dormant until an upgrade. A pending
// admission will choose the current version itself. Disabled services
// (explicit stop or crash-limit give-up) must never be re-armed here.
for service in &mut state.services {
if service.enabled
&& !service.pending
&& service.owner.is_none()
&& service.version.as_deref() != Some(current)
&& service
.method
.split_once('.')
.is_some_and(|(name, _)| name == plugin)
{
let process = state
.processes
.values()
.find(|process| process.id.as_ref() == Some(&service.request));
if process.is_some_and(|process| process.info.version == current)
|| (process.is_none() && service.starting.load(Ordering::SeqCst))
{
retained.push(service);
} else {
services.push((service, process.map(|process| process.ended.subscribe())));
}
} else {
retained.push(service);
self.queue_service(service);
}
}
state.services = retained;
Upgrade {
plugin: plugin.into(),
previous,
current: current.into(),
grace: Duration::from_secs_f64(grace),
processes,
services,
}
}
/// Lets old calls finish, stops them after grace, and starts each lifecycle hook once.
@@ -92,7 +90,7 @@ impl Host {
"core",
&upgrade.plugin,
"core",
&format!("upgrade grace {} seconds", upgrade.grace.as_secs_f64()),
&format!("upgrade grace {} seconds", UPGRADE_GRACE.as_secs()),
);
let mut versions: BTreeMap<String, Vec<watch::Receiver<bool>>> = BTreeMap::new();
versions.entry(upgrade.previous).or_default();
@@ -102,23 +100,11 @@ impl Host {
.or_default()
.push(process.ended.clone());
let host = host.clone();
let grace = upgrade.grace;
tokio::spawn(async move {
let mut ended = process.ended;
tokio::select! {
_ = ended.wait_for(|ended|*ended) => (),
_ = tokio::time::sleep(grace) => { host.begin_stop(&process.pid); }
}
});
}
for (service, ended) in upgrade.services {
let host = host.clone();
tokio::spawn(async move {
if let Some(mut ended) = ended {
let _ = ended.wait_for(|ended| *ended).await;
}
if !host.0.shutdown.load(Ordering::SeqCst) {
host.start_service(service.method, service.params).await;
_ = tokio::time::sleep(UPGRADE_GRACE) => { host.begin_retire(&process.pid); }
}
});
}
cli/src/upgrades/tests.rs
+115
View File
@@ -0,0 +1,115 @@
//! Controlled-clock coverage of both upgrade windows and discovery exclusion.
use super::*;
use crate::{
process::{Process, ProcessInfo},
runtime::Runtime,
world::World,
};
use tokio::{
sync::mpsc,
task::JoinHandle,
time::{Instant, advance},
};
/// A process with active work that ignores both superseded and EOF.
fn process(
host: &Host,
pid: &str,
method: &str,
) -> (mpsc::UnboundedReceiver<serde_json::Value>, JoinHandle<()>) {
let (input, incoming) = mpsc::unbounded_channel();
let task = tokio::spawn(std::future::pending());
host.0.state.lock().unwrap().processes.insert(
pid.into(),
Process {
info: ProcessInfo {
plugin: "notes".into(),
version: "old".into(),
method: method.into(),
params: json!({}),
parent: None,
detached: true,
start_time: 0,
state: "running".into(),
},
memory: sausage_vfs::Memory::default(),
files: sausage_vfs::Process::default(),
store_ended: watch::channel(true).0,
input: Some(input),
abort: Some(task.abort_handle()),
caller: "console".into(),
id: None,
answer: None,
ended: watch::channel(false).0,
stopped: false,
superseded: false,
},
);
(incoming, task)
}
/// PROTOCOL 3.7/3.6: five seconds running, then five seconds stopping; discovery is untouched.
#[tokio::test]
async fn two_five_second_windows_and_discovery_exclusion() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().join("world");
World::create(&root).unwrap();
let host = Host::test_world(&root, Runtime::new(&dir.path().join("cache")).unwrap())
.await
.unwrap();
let (mut input, work) = process(&host, "old-work", "work");
let (mut discovery_input, discovery) = process(&host, "old-discovery", "rpc.discover");
let upgrade = host.snapshot_upgrade("notes", "old".into(), "new");
assert_eq!(upgrade.processes.len(), 1);
tokio::time::pause();
let start = Instant::now();
host.begin_upgrade(upgrade);
assert_eq!(
input.recv().await.unwrap(),
json!({"jsonrpc":"2.0","method":"rpc.superseded","params":{"current":"new"}})
);
// Run the upgrade dispatcher, then its per-process task, so the deadline is armed.
tokio::task::yield_now().await;
tokio::task::yield_now().await;
let window = Duration::from_secs(5);
assert_eq!(UPGRADE_GRACE, window);
// tokio::time::sleep documents millisecond granularity; the paused driver
// may round a deadline up by one tick. Check each window independently.
let tick = Duration::from_millis(1);
let before_deadline = window - tick;
advance(before_deadline).await;
tokio::task::yield_now().await;
assert_eq!(host.processes()["old-work"].state, "running");
assert!(!work.is_finished());
advance(tick).await;
assert!(
input.recv().await.is_none(),
"EOF at grace, even with active work"
);
let stopped = Instant::now();
assert!((window..=window + tick).contains(&(stopped - start)));
assert_eq!(host.processes()["old-work"].state, "stopping");
// begin_retire schedules the independent PROTOCOL 3.6 deadline.
tokio::task::yield_now().await;
advance(before_deadline).await;
tokio::task::yield_now().await;
assert!(
!work.is_finished(),
"on_stop still has its full five seconds"
);
advance(tick).await;
assert!(work.await.unwrap_err().is_cancelled());
assert!((window..=window + tick).contains(&(Instant::now() - stopped)));
assert_eq!(host.processes()["old-discovery"].state, "running");
assert!(matches!(
discovery_input.try_recv(),
Err(mpsc::error::TryRecvError::Empty)
));
assert!(!discovery.is_finished());
discovery.abort();
let _ = discovery.await;
// Synthetic processes have no execution cleanup task; discard only this test table.
host.0.shutdown.store(true, Ordering::SeqCst);
host.0.state.lock().unwrap().processes.clear();
host.stop().await;
}