surma/sausage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Supervise boot services with uniform five-second upgrade grace

Browse Source
This commit is contained in:
Sausage
2026-11-01 20:51:08 +00:00
parent 4da8a28392
commit 959ddf46ff
15 changed files with 943 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/tests/host/activation.rs
+136 -8
View File
@@ -160,9 +160,9 @@ impl Drop for TestWorld {
let _ = std::fs::remove_dir_all(&self.0);
}
}
/// Builds a method-only document without an upgrade delay.
/// Builds a method-only document; all plugins use the host's five-second grace.
fn document(name: &str) -> Value {
json!({"methods":[{"name":name,"params":[],"result":{"schema":{"type":"string"}}}],"x-sausage-grace":0})
json!({"methods":[{"name":name,"params":[],"result":{"schema":{"type":"string"}}}]})
}
/// Adds an import of a provider's method list.
fn consumer(provider: &str, expected: Value) -> Value {
@@ -484,7 +484,7 @@ async fn review_dropped_activation_completes() {
host.stop().await;
}
/// PROTOCOL 10.1: discovery fails after its fixed ten-second deadline and receives EOF.
/// PROTOCOL 3.7: old processes retain their declared grace while new calls use new code.
/// PROTOCOL 3.7: old processes get five seconds regardless of declarations; new calls use new code.
/// PROTOCOL 3.7: retirement waits for all live admissions, including reverted versions.
/// PROTOCOL 3.7: rejected discovery candidates never receive retirement hooks.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -604,7 +604,7 @@ async fn source_changes_do_not_change_candidate() {
);
host.stop().await;
}
/// PROTOCOL 3.7: plugins without a declared grace period retain old calls before the default deadline.
/// PROTOCOL 3.7: missing metadata also retains old calls until the uniform deadline.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn default_grace_retains_old_process() {
let world = TestWorld::new();
@@ -623,6 +623,104 @@ async fn default_grace_retains_old_process() {
assert_eq!(host.processes()[&pid].state, "running");
host.stop().await;
}
/// PROTOCOL 3.7: metadata cannot shorten or extend the actual grace for active requests.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn uniform_grace_ignores_discovery_declarations() {
let world = TestWorld::new();
let declarations = [
("missing", None),
("default", Some(json!(60))),
("long", Some(json!(3600))),
("zero", Some(json!(0))),
("negative", Some(json!(-1))),
("invalid", Some(json!("long"))),
("huge", Some(json!(1e300))),
];
for (name, grace) in &declarations {
let mut old = document("work");
if let Some(grace) = grace {
old["x-sausage-grace"] = grace.clone();
}
world.plugin(name, old, document("work"));
}
let host = world.start().await;
let mut ready = host.subscribe();
let mut calls = Vec::new();
for (name, _) in &declarations {
let host = host.clone();
let method = format!("{name}.work");
calls.push(tokio::spawn(async move {
let result = host.call(&method, json!({})).await;
(result, std::time::Instant::now())
}));
}
// Guest notifications are events, not process/file polling.
for _ in &declarations {
let event = tokio::time::timeout(Duration::from_secs(30), ready.recv())
.await
.unwrap()
.unwrap();
assert_eq!(event["params"]["name"], "ready");
}
let before = std::time::Instant::now();
let plugins: Vec<_> = declarations
.iter()
.map(|(name, _)| json!({"plugin":name,"wasm":format!("/state/{name}/next.wasm")}))
.collect();
call(&host, "core.activate", json!({"plugins":plugins})).await;
// Use the existing discovery+stop test budget, not a new execution limit.
tokio::time::timeout(Duration::from_secs(15), async {
for task in calls {
let (result, ended) = task.await.unwrap();
assert_eq!(result.unwrap_err()["code"], -32002);
assert!(
ended.duration_since(before) >= Duration::from_secs(5),
"every declaration gets the full grace"
);
}
})
.await
.unwrap();
host.stop().await;
}
/// PROTOCOL 10.4/3.7: a watcher refresh uses the same policy as explicit activation.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn manual_copy_uses_uniform_grace() {
let world = TestWorld::new();
let mut old = document("work");
old["x-sausage-grace"] = json!(3600);
world.plugin("notes", old, document("work"));
let host = world.start().await;
// Wait for initial discovery without changing the installed version.
call(
&host,
"core.activate",
json!({"plugin":"notes","wasm":"/plugins/notes/plugin.wasm"}),
)
.await;
let mut ready = host.subscribe();
let work_host = host.clone();
let work = tokio::spawn(async move { work_host.call("notes.work", json!({})).await });
let event = tokio::time::timeout(Duration::from_secs(30), ready.recv())
.await
.unwrap()
.unwrap();
assert_eq!(event["params"]["name"], "ready");
let before = std::time::Instant::now();
std::fs::copy(fixture(2), world.0.join("plugins/notes/plugin.wasm")).unwrap();
assert_eq!(
tokio::time::timeout(Duration::from_secs(15), work)
.await
.unwrap()
.unwrap()
.unwrap_err()["code"],
-32002
);
assert!(before.elapsed() >= Duration::from_secs(5));
host.stop().await;
}
/// PROTOCOL 10.3: candidate imports must fit installed providers.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn checks_candidate_imports() {
@@ -818,11 +916,11 @@ async fn graph_missing_import() {
assert!(dot.as_str().unwrap().contains("absent"));
host.stop().await;
}
/// Checks the declared upgrade grace and dispatch to the new version.
/// Checks that an explicit long declaration cannot extend grace; new calls use the new version.
async fn old_process_grace_and_new_calls() {
let world = TestWorld::new();
let mut old = document("add");
old["x-sausage-grace"] = json!(0.5);
old["x-sausage-grace"] = json!(3600);
world.plugin("notes", old, document("add"));
let host = world.start().await;
call(&host, "notes.hold", json!({})).await;
@@ -868,7 +966,7 @@ async fn superseded_notifies_each_old_process() {
json!({"jsonrpc":"2.0","method":"rpc.superseded","params":{"current":fixture_hash(2)}})
.to_string();
for pid in pids {
// marker's existing 30-second bound is shorter than the old 60-second grace.
// The notice arrives while the old process is still in its five-second window.
marker(
world.0.join(format!("state/notes/superseded-{pid}")),
&expected,
@@ -993,7 +1091,7 @@ async fn upgrades_enter_the_journal() {
.as_array()
.unwrap()
.iter()
.any(|e| e["line"] == "upgrade grace 60 seconds")
.any(|e| e["line"] == "upgrade grace 5 seconds")
{
break entries;
}
@@ -1117,6 +1215,36 @@ async fn old_consumer_can_call_removed_provider_method() {
host.stop().await;
}
/// PROTOCOL 8: supervise actual WASI exits on either side of the readiness response.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn crashed_boot_service_restarts_before_and_after_ready() {
for when in ["before", "after"] {
let world = TestWorld::new();
world.plugin("notes", document("serve"), document("serve"));
std::fs::write(
world.0.join("config/boot.yaml"),
json!([{"method":"notes.serve","params":{"crash_once":when}}]).to_string(),
)
.unwrap();
let host = world.start().await;
marker(world.0.join("state/notes/service"), "1").await;
let entries = call(
&host,
"core.log",
json!({"grep":"service start notes.serve"}),
)
.await;
assert_eq!(entries.as_array().unwrap().len(), 2);
let processes = host.processes();
let services: Vec<_> = processes.values().filter(|p| p.method == "serve").collect();
assert_eq!(services.len(), 1);
assert_eq!(services[0].params, json!({"crash_once":when}));
let exits = call(&host, "core.log", json!({"plugin":"notes","grep":"exit 1"})).await;
assert_eq!(exits.as_array().unwrap().len(), 1);
host.stop().await;
}
}
/// PROTOCOL 10.2: activation and revert check interfaces but leave persistent state unchanged.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn state_is_outside_interface_checks() {
cli/tests/host/sdk.rs
+50
View File
@@ -178,6 +178,56 @@ async fn stop_runs_hook() {
assert_eq!(std::fs::read(world.0.join("stopped")).unwrap(), b"yes");
host.stop().await;
}
/// PROTOCOL 3.7: an idle SDK listener opts into drain and exits before EOF/grace.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn superseded_idle_listener_drains_before_grace() {
let world = TestWorld::new();
let host = world.start().await;
let (notice, mut notices) = tokio::sync::mpsc::unbounded_channel();
host.register_native(
"other",
std::sync::Arc::new(move |_, method, _| {
let notice = notice.clone();
Box::pin(async move {
assert_eq!(method, "superseded");
notice.send(std::time::Instant::now()).unwrap();
Ok(json!({}))
})
}),
);
let pid = call(&host, "probe.drain_service", json!({})).await;
// Replace with a different real component; force only skips unrelated test consumers.
std::fs::copy(
fixtures().join("notes.wasm"),
world.0.join("state/next.wasm"),
)
.unwrap();
call(
&host,
"core.activate",
json!({"plugin":"probe","wasm":"/state/next.wasm","force":true}),
)
.await;
// Activation preparation/discovery is outside upgrade grace. Time from the
// hook's outgoing call, not from the start of candidate compilation.
let before = tokio::time::timeout(Duration::from_secs(60), notices.recv())
.await
.unwrap()
.unwrap();
// Waiting on its lifetime lock is an exit event, not process-liveness polling.
call(&host, "core.lock", json!({"path":"/drain-owner"})).await;
assert!(before.elapsed() < Duration::from_secs(5));
assert!(!host.processes().contains_key(pid.as_str().unwrap()));
assert!(
!world
.0
.join(format!("stopped-{}", pid.as_str().unwrap()))
.exists(),
"drain must not run on_stop (discovery processes may run their own hooks)"
);
host.stop().await;
}
/// SDK framing carries payloads above 64 KiB in both directions.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn large_messages() {