Trim the harness to what it needs to keep going
Host: - test first: a new version is tested where it lies; only an installed version goes to versions/, and instances run from there - data/ and public/ are shared by all versions; no snapshots; rollback restores code only - no re-delivery, status.json, list, remove/archive, smoke test, data/ hiding, or stdin timeout; rollback needs a version - supervisor.rs split out; one place for installing a version - init only writes the agent folder; run installs it like any actor - console: @actor, text to the agent, /help, /quit Agent: - every text goes out as a broadcast; 5-line prompt; 5 tools that describe themselves; turn.json only marks an active turn Docs: PROTOCOL.md, README.md, DECISIONS.md, AGENTS.md rewritten. Examples: only notes and notes-v2.
This commit is contained in:
+67
-586
@@ -1,5 +1,6 @@
|
||||
//! The broker: message routing, mailboxes, subscriptions, and the lifecycle of
|
||||
//! WASM actors (spawn, upgrade, rollback, stop, start, remove, scan).
|
||||
//! The broker: message routing, mailboxes, subscriptions, the version store,
|
||||
//! and the requests to `broker` (spawn, upgrade, rollback, stop, start,
|
||||
//! subscribe, unsubscribe, scan). `supervisor.rs` runs each WASM actor.
|
||||
//!
|
||||
//! The broker accepts only `.wasm` files. It does not know any source language.
|
||||
//! The world folder is the source of truth: `scan` boots every
|
||||
@@ -18,16 +19,12 @@ use wasmtime::component::Component;
|
||||
|
||||
use crate::candidate;
|
||||
use crate::fs::Mount;
|
||||
use crate::runtime::{Exit, Instance, InstanceConfig, Runtime};
|
||||
use crate::runtime::Runtime;
|
||||
use crate::supervisor::{Control, Reply, Supervisor};
|
||||
use crate::world::{self, ActorMeta, NATIVE, VersionMeta, World};
|
||||
|
||||
pub const MAILBOX_SIZE: usize = 1000;
|
||||
pub const SHUTDOWN_DEADLINE: Duration = Duration::from_secs(5);
|
||||
/// A new version that crashes this often within `CRASH_WINDOW` after a swap
|
||||
/// is rolled back automatically.
|
||||
const CRASH_LIMIT: usize = 3;
|
||||
const CRASH_WINDOW: Duration = Duration::from_secs(30);
|
||||
const STDIN_WRITE_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Envelope {
|
||||
@@ -53,17 +50,6 @@ pub fn shutdown_line() -> String {
|
||||
json!({"id": "broker/shutdown", "from": "broker", "body": {"type": "shutdown"}}).to_string()
|
||||
}
|
||||
|
||||
type Reply = oneshot::Sender<Result<Value>>;
|
||||
|
||||
enum Control {
|
||||
Upgrade { version: u64, requester: String, reply: Reply },
|
||||
Rollback { target: Option<u64>, reply: Reply },
|
||||
Stop { reply: Reply },
|
||||
Start { reply: Reply },
|
||||
Remove { reply: Reply },
|
||||
Shutdown { reply: Reply },
|
||||
}
|
||||
|
||||
pub struct Broker {
|
||||
pub world: World,
|
||||
pub runtime: Arc<Runtime>,
|
||||
@@ -235,58 +221,51 @@ impl Broker {
|
||||
tasks.join_all().await;
|
||||
}
|
||||
|
||||
/// Copy a `.wasm` file and its source folder into `versions/<n>/`.
|
||||
/// Every version that the broker sees stays there, also rejected ones.
|
||||
fn store_version(&self, id: &str, meta: &VersionMeta, wasm: &Path, source: Option<&Path>) -> Result<Component> {
|
||||
/// Compile `versions/<version>/actor.wasm`.
|
||||
pub fn load_version(&self, id: &str, version: u64) -> Result<Component> {
|
||||
self.load(id, &self.world.version_dir(id, version).join("actor.wasm"))
|
||||
}
|
||||
|
||||
fn load(&self, id: &str, wasm: &Path) -> Result<Component> {
|
||||
let started = Instant::now();
|
||||
let component = self.runtime.load(wasm)?;
|
||||
self.log(id, &format!("compiled {} in {} ms", wasm.display(), started.elapsed().as_millis()));
|
||||
Ok(component)
|
||||
}
|
||||
|
||||
/// Test a new version where it lies. Only when the tests pass, copy it
|
||||
/// to `versions/<n>/`: that folder holds only versions that run(ran).
|
||||
/// Returns the component and the test report.
|
||||
async fn try_version(&self, id: &str, meta: &VersionMeta, wasm: &Path, source: Option<&Path>) -> Result<(Component, Value)> {
|
||||
let component = self.load(id, wasm)?;
|
||||
let tests = candidate::test(self, id, meta.version, &component, source).await;
|
||||
if let Err(e) = &tests {
|
||||
self.log(id, &format!("version {} from {} failed its tests: {e:#}", meta.version, meta.wasm));
|
||||
}
|
||||
let tests = tests?;
|
||||
let dir = self.world.version_dir(id, meta.version);
|
||||
std::fs::create_dir_all(&dir)?;
|
||||
std::fs::copy(wasm, dir.join("actor.wasm")).with_context(|| format!("copy {}", wasm.display()))?;
|
||||
world::replace_source(source.unwrap_or(&dir.join("no-source")), &dir.join("source"))?;
|
||||
world::write_atomic(&dir.join("meta.json"), serde_json::to_string_pretty(meta)?.as_bytes())?;
|
||||
self.load_version(id, meta.version)
|
||||
Ok((component, tests))
|
||||
}
|
||||
|
||||
/// Compile `versions/<version>/actor.wasm`.
|
||||
pub fn load_version(&self, id: &str, version: u64) -> Result<Component> {
|
||||
let started = Instant::now();
|
||||
let component = self.runtime.load(&self.world.version_dir(id, version).join("actor.wasm"))?;
|
||||
self.log(id, &format!("compiled version {version} in {} ms", started.elapsed().as_millis()));
|
||||
Ok(component)
|
||||
}
|
||||
|
||||
/// Record in `versions/<n>/meta.json` why the broker refused a version.
|
||||
fn reject(&self, id: &str, version: u64, reason: &anyhow::Error) {
|
||||
let path = self.world.version_dir(id, version).join("meta.json");
|
||||
let mut meta: VersionMeta = std::fs::read_to_string(&path)
|
||||
.ok()
|
||||
.and_then(|s| serde_json::from_str(&s).ok())
|
||||
.unwrap_or_default();
|
||||
meta.rejected = Some(format!("{reason:#}"));
|
||||
let _ = world::write_atomic(&path, serde_json::to_string_pretty(&meta).unwrap_or_default().as_bytes());
|
||||
}
|
||||
|
||||
fn version_rejected(&self, id: &str, version: u64) -> bool {
|
||||
std::fs::read_to_string(self.world.version_dir(id, version).join("meta.json"))
|
||||
.ok()
|
||||
.and_then(|s| serde_json::from_str::<VersionMeta>(&s).ok())
|
||||
.is_some_and(|m| m.rejected.is_some())
|
||||
}
|
||||
|
||||
/// Install `version` as the current files of a new actor.
|
||||
fn install_files(&self, id: &str, version: u64) -> Result<()> {
|
||||
let dir = self.world.version_dir(id, version);
|
||||
self.world.ensure_actor_dirs(id)?;
|
||||
world::write_atomic(&self.world.wasm_path(id), &std::fs::read(dir.join("actor.wasm"))?)?;
|
||||
world::replace_tree(&dir.join("source"), &self.world.source_dir(id))?;
|
||||
/// Make `version` the current `actor.wasm`, `source/`, and `actor.json`.
|
||||
pub fn install(&self, id: &str, version: u64) -> Result<()> {
|
||||
self.world.install_files(id, version)?;
|
||||
self.update_meta(id, |m| {
|
||||
m.version = Some(version);
|
||||
m.running = true;
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a new actor from a `.wasm` file. With `start`, also start its
|
||||
/// supervisor (in a running host). Without `start` (`init`), only install it.
|
||||
pub async fn create_actor(self: &Arc<Self>, id: &str, wasm: &Path, source: Option<&Path>, meta: VersionMeta, start: bool) -> Result<Value> {
|
||||
pub fn set_running(&self, id: &str, running: bool) -> Result<()> {
|
||||
self.update_meta(id, |m| m.running = running)
|
||||
}
|
||||
|
||||
/// Create a new actor from a `.wasm` file: test it, install it, start it.
|
||||
pub async fn create_actor(self: &Arc<Self>, id: &str, wasm: &Path, source: Option<&Path>, meta: VersionMeta) -> Result<Value> {
|
||||
if !world::valid_id(id) {
|
||||
bail!("invalid actor ID {id:?}: use a-z, 0-9, '-' and '_'");
|
||||
}
|
||||
@@ -295,31 +274,15 @@ impl Broker {
|
||||
if NATIVE.contains(&id) || self.is_supervised(id) || self.world.wasm_path(id).exists() {
|
||||
bail!("an actor with ID {id} already exists");
|
||||
}
|
||||
let version = self.world.next_version(id);
|
||||
let meta = VersionMeta { version, ..meta };
|
||||
self.create_version(id, version, &meta, wasm, source, start).await
|
||||
let meta = VersionMeta { version: self.world.next_version(id), ..meta };
|
||||
self.create_version(id, &meta, wasm, source).await
|
||||
}
|
||||
|
||||
async fn create_version(self: &Arc<Self>, id: &str, version: u64, meta: &VersionMeta, wasm: &Path, source: Option<&Path>, start: bool) -> Result<Value> {
|
||||
let tested = async {
|
||||
let component = self.store_version(id, meta, wasm, source)?;
|
||||
let source = self.world.version_dir(id, version).join("source");
|
||||
candidate::test(self, id, version, &component, &source, &self.world.data_dir(id), &self.world.public_dir(id)).await
|
||||
}
|
||||
.await;
|
||||
let report = match tested {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
self.reject(id, version, &e);
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
self.install_files(id, version)?;
|
||||
write_status(&self.world, id, &json!({"id": id, "version": version, "state": "created"}));
|
||||
if start {
|
||||
self.start_supervisor(id, version, true);
|
||||
}
|
||||
Ok(json!({"type": "spawned", "actor": id, "version": version, "tests": report}))
|
||||
async fn create_version(self: &Arc<Self>, id: &str, meta: &VersionMeta, wasm: &Path, source: Option<&Path>) -> Result<Value> {
|
||||
let (_, tests) = self.try_version(id, meta, wasm, source).await?;
|
||||
self.install(id, meta.version)?;
|
||||
self.start_supervisor(id, meta.version, true);
|
||||
Ok(json!({"type": "spawned", "actor": id, "version": meta.version, "tests": tests}))
|
||||
}
|
||||
|
||||
pub async fn upgrade_actor(self: &Arc<Self>, id: &str, wasm: &Path, source: Option<&Path>, meta: VersionMeta) -> Result<Value> {
|
||||
@@ -333,33 +296,19 @@ impl Broker {
|
||||
}
|
||||
|
||||
async fn upgrade_version(self: &Arc<Self>, id: &str, meta: &VersionMeta, wasm: &Path, source: Option<&Path>) -> Result<Value> {
|
||||
let version = meta.version;
|
||||
let tested = async {
|
||||
let component = self.store_version(id, meta, wasm, source)?;
|
||||
let source = self.world.version_dir(id, version).join("source");
|
||||
candidate::test(self, id, version, &component, &source, &self.world.data_dir(id), &self.world.public_dir(id)).await
|
||||
}
|
||||
.await;
|
||||
let report = match tested {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
self.reject(id, version, &e);
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
let requester = meta.requested_by.clone();
|
||||
let mut result = self.control(id, |reply| Control::Upgrade { version, requester, reply }).await?;
|
||||
result["tests"] = report;
|
||||
let (component, tests) = self.try_version(id, meta, wasm, source).await?;
|
||||
let (version, requester) = (meta.version, meta.requested_by.clone());
|
||||
let mut result = self.control(id, |reply| Control::Upgrade { version, component, requester, reply }).await?;
|
||||
result["tests"] = tests;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Boot the actors in `actor/`. Called at start and on `scan`.
|
||||
///
|
||||
/// - A known `actor.wasm` (equal to the installed version) starts as is.
|
||||
/// - A new or changed `actor.wasm` is first copied to `versions/<n>/`.
|
||||
/// A new actor is tested and installed. A changed one is an upgrade:
|
||||
/// if it fails, the installed files come back.
|
||||
/// - A file that equals an already rejected version is not tried again.
|
||||
/// - An `actor.wasm` equal to the installed version starts as is.
|
||||
/// - A new or changed `actor.wasm` is tested where it lies. A new actor
|
||||
/// is installed, a changed one is an upgrade. If its tests fail,
|
||||
/// nothing changes, and the file stays for the next scan.
|
||||
pub async fn scan(self: &Arc<Self>) -> Vec<Value> {
|
||||
let mut report = Vec::new();
|
||||
for id in self.world.actor_ids().unwrap_or_default() {
|
||||
@@ -377,8 +326,6 @@ impl Broker {
|
||||
|
||||
async fn scan_actor(self: &Arc<Self>, id: &str) -> Option<Value> {
|
||||
let meta = self.world.read_meta(id);
|
||||
let wasm = self.world.wasm_path(id);
|
||||
let same = |v: u64| same_file(&wasm, &self.world.version_dir(id, v).join("actor.wasm"));
|
||||
let installed = meta.version.filter(|v| self.world.version_dir(id, *v).join("actor.wasm").is_file());
|
||||
self.metas.lock().unwrap().insert(id.to_string(), meta.clone());
|
||||
if let Some(v) = installed
|
||||
@@ -386,59 +333,25 @@ impl Broker {
|
||||
{
|
||||
self.start_supervisor(id, v, meta.running);
|
||||
}
|
||||
if installed.is_some_and(same) {
|
||||
let wasm = self.world.wasm_path(id);
|
||||
if installed.is_some_and(|v| same_file(&wasm, &self.world.version_dir(id, v).join("actor.wasm"))) {
|
||||
return None;
|
||||
}
|
||||
let latest = self.world.versions(id).last().copied();
|
||||
if let Some(v) = latest
|
||||
&& same(v)
|
||||
&& self.version_rejected(id, v)
|
||||
{
|
||||
if let Some(current) = installed {
|
||||
let _ = self.restore_files(id, current);
|
||||
}
|
||||
self.log(id, &format!("actor.wasm equals the rejected version {v}; not installed"));
|
||||
return Some(json!({"actor": id, "skipped": "rejected version", "version": v}));
|
||||
}
|
||||
let version = self.world.next_version(id);
|
||||
let source = self.world.source_dir(id);
|
||||
let vmeta = VersionMeta {
|
||||
version,
|
||||
version: self.world.next_version(id),
|
||||
wasm: format!("/actor/{id}/actor.wasm"),
|
||||
source: Some(format!("/actor/{id}/source")),
|
||||
built_by: None,
|
||||
requested_by: "scan".into(),
|
||||
created_at: world::now(),
|
||||
rejected: None,
|
||||
};
|
||||
let source = source.is_dir().then_some(source.as_path());
|
||||
let result = match installed {
|
||||
None => self.create_version(id, version, &vmeta, &wasm, source, true).await,
|
||||
Some(_) => {
|
||||
let r = self.upgrade_version(id, &vmeta, &wasm, source).await;
|
||||
// Put the installed version back in place. Read it again: an
|
||||
// automatic rollback can have changed it meanwhile.
|
||||
if r.is_err()
|
||||
&& let Some(current) = self.meta(id).and_then(|m| m.version)
|
||||
{
|
||||
let _ = self.restore_files(id, current);
|
||||
}
|
||||
r
|
||||
}
|
||||
None => self.create_version(id, &vmeta, &wasm, source).await,
|
||||
Some(_) => self.upgrade_version(id, &vmeta, &wasm, source).await,
|
||||
};
|
||||
Some(match result {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
self.log(id, &format!("version {version} from disk rejected: {e:#}"));
|
||||
json!({"type": "error", "actor": id, "version": version, "reason": format!("{e:#}")})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn restore_files(&self, id: &str, version: u64) -> Result<()> {
|
||||
let dir = self.world.version_dir(id, version);
|
||||
world::write_atomic(&self.world.wasm_path(id), &std::fs::read(dir.join("actor.wasm"))?)?;
|
||||
world::replace_tree(&dir.join("source"), &self.world.source_dir(id))
|
||||
Some(result.unwrap_or_else(|e| json!({"type": "error", "actor": id, "reason": format!("{e:#}")})))
|
||||
}
|
||||
|
||||
/// Handle one message to `broker`.
|
||||
@@ -473,14 +386,13 @@ impl Broker {
|
||||
built_by: b["built_by"].as_str().map(str::to_string),
|
||||
requested_by: env.from.clone(),
|
||||
created_at: world::now(),
|
||||
rejected: None,
|
||||
};
|
||||
Ok((wasm, source, meta))
|
||||
};
|
||||
match ty {
|
||||
"spawn" => {
|
||||
let (wasm, source, meta) = version_meta()?;
|
||||
self.create_actor(&actor()?, &wasm, source.as_deref(), meta, true).await
|
||||
self.create_actor(&actor()?, &wasm, source.as_deref(), meta).await
|
||||
}
|
||||
"upgrade" => {
|
||||
let (wasm, source, meta) = version_meta()?;
|
||||
@@ -488,23 +400,13 @@ impl Broker {
|
||||
}
|
||||
"rollback" => {
|
||||
let id = actor()?;
|
||||
let target = b["version"].as_u64();
|
||||
let version = b["version"].as_u64().ok_or_else(|| anyhow!("missing \"version\""))?;
|
||||
let lock = self.op_lock(&id);
|
||||
let _guard = lock.lock().await;
|
||||
self.control(&id, |reply| Control::Rollback { target, reply }).await
|
||||
self.control(&id, |reply| Control::Rollback { version, reply }).await
|
||||
}
|
||||
"stop" => self.control(&actor()?, |reply| Control::Stop { reply }).await,
|
||||
"start" => self.control(&actor()?, |reply| Control::Start { reply }).await,
|
||||
"remove" => {
|
||||
let id = actor()?;
|
||||
let lock = self.op_lock(&id);
|
||||
let _guard = lock.lock().await;
|
||||
let r = self.control(&id, |reply| Control::Remove { reply }).await?;
|
||||
self.mailboxes.lock().unwrap().remove(&id);
|
||||
self.controls.lock().unwrap().remove(&id);
|
||||
self.metas.lock().unwrap().remove(&id);
|
||||
Ok(r)
|
||||
}
|
||||
"subscribe" | "unsubscribe" => {
|
||||
let publisher = actor()?;
|
||||
let on = ty == "subscribe";
|
||||
@@ -518,35 +420,21 @@ impl Broker {
|
||||
Ok(json!({"type": format!("{ty}d"), "actor": publisher}))
|
||||
}
|
||||
"scan" => Ok(json!({"type": "scanned", "changes": self.scan().await})),
|
||||
"list" => {
|
||||
let metas = self.metas.lock().unwrap().clone();
|
||||
let actors: Vec<Value> = metas
|
||||
.iter()
|
||||
.map(|(id, m)| {
|
||||
let status = std::fs::read_to_string(self.world.actor_dir(id).join("status.json"))
|
||||
.ok()
|
||||
.and_then(|s| serde_json::from_str::<Value>(&s).ok())
|
||||
.unwrap_or_default();
|
||||
json!({"id": id, "kind": if NATIVE.contains(&id.as_str()) { "native" } else { "wasm" },
|
||||
"version": m.version, "running": m.running, "state": status["state"],
|
||||
"subscriptions": m.subscriptions})
|
||||
})
|
||||
.collect();
|
||||
Ok(json!({"type": "actors", "actors": actors}))
|
||||
}
|
||||
"shutdown" => Ok(json!({"type": "ignored"})),
|
||||
other => bail!("unknown broker request {other:?}. Known: spawn, upgrade, rollback, stop, start, remove, subscribe, unsubscribe, scan, list"),
|
||||
other => bail!("unknown broker request {other:?}. Known: spawn, upgrade, rollback, stop, start, subscribe, unsubscribe, scan"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The native `broker` actor. Requests run concurrently. Aborting this
|
||||
/// task aborts the requests that still run.
|
||||
/// task aborts the requests that still run. Replies (messages with
|
||||
/// `reply_to`) get no answer, so that two actors never answer each other
|
||||
/// forever.
|
||||
pub async fn run_actor(self: Arc<Self>, mut mailbox: mpsc::Receiver<Envelope>) {
|
||||
let mut requests = tokio::task::JoinSet::new();
|
||||
loop {
|
||||
tokio::select! {
|
||||
env = mailbox.recv() => match env {
|
||||
Some(env) => { requests.spawn(self.clone().handle(env)); }
|
||||
Some(env) if env.reply_to.is_none() => { requests.spawn(self.clone().handle(env)); }
|
||||
Some(_) => {}
|
||||
None => break,
|
||||
},
|
||||
Some(_) = requests.join_next() => {}
|
||||
@@ -570,410 +458,3 @@ pub fn truncate(s: &str, max: usize) -> String {
|
||||
format!("{}… ({} bytes)", &s[..end], s.len())
|
||||
}
|
||||
|
||||
fn write_status(world: &World, id: &str, status: &Value) {
|
||||
let mut s = status.clone();
|
||||
s["updated_at"] = json!(world::now());
|
||||
let _ = world::write_atomic(
|
||||
&world.actor_dir(id).join("status.json"),
|
||||
serde_json::to_string_pretty(&s).unwrap_or_default().as_bytes(),
|
||||
);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Supervisor: owns one WASM actor and its instance.
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
struct Supervisor {
|
||||
broker: Arc<Broker>,
|
||||
id: String,
|
||||
version: u64,
|
||||
desired_running: bool,
|
||||
mailbox: mpsc::Receiver<Envelope>,
|
||||
control: mpsc::Receiver<Control>,
|
||||
component: Option<Component>,
|
||||
instance: Option<Instance>,
|
||||
state: &'static str,
|
||||
started_at: Instant,
|
||||
restarts: u32,
|
||||
restart_at: Option<Instant>,
|
||||
last_exit: Option<String>,
|
||||
last_delivered: Option<Envelope>,
|
||||
/// The message that the actor got just before it crashed. The broker
|
||||
/// delivers it again after the next upgrade.
|
||||
parked: Option<Envelope>,
|
||||
redeliver: Option<Envelope>,
|
||||
previous: Option<u64>,
|
||||
swapped_at: Option<Instant>,
|
||||
crashes_since_swap: usize,
|
||||
upgrade_requester: Option<String>,
|
||||
/// A control request that arrived during a blocked delivery.
|
||||
next_control: Option<Control>,
|
||||
}
|
||||
|
||||
async fn recv_line(instance: &mut Option<Instance>) -> Option<String> {
|
||||
match instance {
|
||||
Some(i) => i.stdout.recv().await,
|
||||
None => std::future::pending().await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn sleep_until_opt(t: Option<Instant>) {
|
||||
match t {
|
||||
Some(t) => tokio::time::sleep_until(t.into()).await,
|
||||
None => std::future::pending().await,
|
||||
}
|
||||
}
|
||||
|
||||
impl Supervisor {
|
||||
fn new(
|
||||
broker: Arc<Broker>,
|
||||
id: String,
|
||||
version: u64,
|
||||
running: bool,
|
||||
mailbox: mpsc::Receiver<Envelope>,
|
||||
control: mpsc::Receiver<Control>,
|
||||
) -> Self {
|
||||
Self {
|
||||
broker,
|
||||
id,
|
||||
version,
|
||||
desired_running: running,
|
||||
mailbox,
|
||||
control,
|
||||
component: None,
|
||||
instance: None,
|
||||
state: if running { "starting" } else { "stopped" },
|
||||
started_at: Instant::now(),
|
||||
restarts: 0,
|
||||
restart_at: None,
|
||||
last_exit: None,
|
||||
last_delivered: None,
|
||||
parked: None,
|
||||
redeliver: None,
|
||||
previous: None,
|
||||
swapped_at: None,
|
||||
crashes_since_swap: 0,
|
||||
upgrade_requester: None,
|
||||
next_control: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn world(&self) -> &World {
|
||||
&self.broker.world
|
||||
}
|
||||
|
||||
fn status(&self) {
|
||||
write_status(
|
||||
self.world(),
|
||||
&self.id,
|
||||
&json!({
|
||||
"id": self.id,
|
||||
"version": self.version,
|
||||
"state": self.state,
|
||||
"restarts": self.restarts,
|
||||
"last_exit": self.last_exit,
|
||||
"parked_message": self.parked.as_ref().map(|m| json!({"id": m.id, "from": m.from, "body": m.body})),
|
||||
"previous_version": self.previous,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async fn run(mut self) {
|
||||
self.status();
|
||||
loop {
|
||||
if let Some(ctl) = self.next_control.take() {
|
||||
if self.handle_control(ctl).await {
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if self.desired_running && self.instance.is_none() && self.restart_at.is_none_or(|t| Instant::now() >= t) {
|
||||
self.restart_at = None;
|
||||
let _ = self.start_instance().await;
|
||||
}
|
||||
let has_instance = self.instance.is_some();
|
||||
let waiting = !has_instance && self.desired_running;
|
||||
tokio::select! {
|
||||
ctl = self.control.recv() => {
|
||||
let Some(ctl) = ctl else { break };
|
||||
if self.handle_control(ctl).await {
|
||||
break;
|
||||
}
|
||||
}
|
||||
msg = self.mailbox.recv(), if has_instance => {
|
||||
let Some(msg) = msg else { break };
|
||||
self.deliver(msg).await;
|
||||
}
|
||||
line = recv_line(&mut self.instance) => {
|
||||
match line {
|
||||
Some(line) => self.broker.route_line(&self.id, &line),
|
||||
None => {
|
||||
let exit = self.instance.as_mut().unwrap().wait().await;
|
||||
self.instance = None;
|
||||
self.on_exit(exit).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = sleep_until_opt(self.restart_at), if waiting => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Write a message to the instance's stdin, and route its output
|
||||
/// meanwhile. An instance that does not read its stdin for
|
||||
/// `STDIN_WRITE_TIMEOUT`, or while a control request waits, is killed and
|
||||
/// restarted: a partly written line would corrupt the next message.
|
||||
async fn deliver(&mut self, msg: Envelope) {
|
||||
let Some(inst) = self.instance.as_mut() else { return };
|
||||
let (broker, id, line) = (self.broker.clone(), self.id.clone(), msg.to_line());
|
||||
let write = tokio::time::timeout(STDIN_WRITE_TIMEOUT, inst.send_line(&line, |l| broker.route_line(&id, &l)));
|
||||
let failure = tokio::select! {
|
||||
written = write => match written {
|
||||
Ok(Ok(())) => {
|
||||
self.last_delivered = Some(msg);
|
||||
return;
|
||||
}
|
||||
// The instance is ending. The main loop sees its closed stdout.
|
||||
Ok(Err(_)) => None,
|
||||
Err(_) => Some(format!("the actor did not read stdin for {STDIN_WRITE_TIMEOUT:?}")),
|
||||
},
|
||||
Some(ctl) = self.control.recv() => {
|
||||
self.next_control = Some(ctl);
|
||||
Some("the actor did not read stdin when the broker needed it".to_string())
|
||||
}
|
||||
};
|
||||
if msg.from != "broker" {
|
||||
let reason = failure.as_deref().unwrap_or("the actor stopped");
|
||||
let body = json!({"type": "undeliverable", "reason": reason, "to": self.id});
|
||||
self.broker.send("broker", Some(&msg.from), None, Some(&msg.id), body);
|
||||
}
|
||||
if let (Some(failure), Some(inst)) = (failure, &self.instance) {
|
||||
// The main loop sees the end of the instance and restarts it.
|
||||
self.broker.log(&self.id, &format!("killed: {failure}"));
|
||||
inst.kill();
|
||||
}
|
||||
}
|
||||
|
||||
/// Start an instance of the current version. On failure, the actor is
|
||||
/// "broken" until the next start, upgrade, or rollback.
|
||||
async fn start_instance(&mut self) -> Result<()> {
|
||||
let started = self.try_start();
|
||||
if let Err(e) = &started {
|
||||
self.broker.log(&self.id, &format!("cannot start version {}: {e:#}", self.version));
|
||||
self.state = "broken";
|
||||
self.desired_running = false;
|
||||
self.status();
|
||||
return started;
|
||||
}
|
||||
self.state = "running";
|
||||
self.started_at = Instant::now();
|
||||
self.status();
|
||||
if let Some(msg) = self.redeliver.take() {
|
||||
self.broker.log(&self.id, &format!("delivering parked message {} again", msg.id));
|
||||
self.deliver(msg).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn try_start(&mut self) -> Result<()> {
|
||||
if self.component.is_none() {
|
||||
self.component = Some(self.broker.load_version(&self.id, self.version)?);
|
||||
}
|
||||
let cfg = InstanceConfig {
|
||||
mounts: self.broker.mounts(&self.id),
|
||||
allow_http: true,
|
||||
stderr_log: self.world().actor_dir(&self.id).join("stderr.log"),
|
||||
env: vec![
|
||||
("SAUSAGE_ACTOR".into(), self.id.clone()),
|
||||
("SAUSAGE_VERSION".into(), self.version.to_string()),
|
||||
],
|
||||
};
|
||||
self.instance = Some(self.broker.runtime.start(self.component.as_ref().unwrap(), cfg)?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stop the instance gracefully. Its last messages still get routed.
|
||||
async fn stop_instance(&mut self) -> Option<Exit> {
|
||||
let inst = self.instance.take()?;
|
||||
let (broker, id) = (self.broker.clone(), self.id.clone());
|
||||
let exit = inst.stop(&shutdown_line(), SHUTDOWN_DEADLINE, |line| broker.route_line(&id, &line)).await;
|
||||
self.broker.log(&self.id, &format!("stopped: {exit}"));
|
||||
self.last_exit = Some(exit.to_string());
|
||||
Some(exit)
|
||||
}
|
||||
|
||||
async fn on_exit(&mut self, exit: Exit) {
|
||||
self.broker.log(&self.id, &format!("instance ended unexpectedly: {exit}"));
|
||||
self.last_exit = Some(exit.to_string());
|
||||
if !exit.is_clean()
|
||||
&& let Some(m) = self.last_delivered.take()
|
||||
{
|
||||
self.parked = Some(m);
|
||||
}
|
||||
if self.started_at.elapsed() > Duration::from_secs(60) {
|
||||
self.restarts = 0;
|
||||
}
|
||||
if let (Some(prev), Some(at)) = (self.previous, self.swapped_at)
|
||||
&& at.elapsed() < CRASH_WINDOW
|
||||
{
|
||||
self.crashes_since_swap += 1;
|
||||
if self.crashes_since_swap >= CRASH_LIMIT {
|
||||
let from = self.version;
|
||||
self.broker.log(&self.id, &format!("version {from} crashed {CRASH_LIMIT} times after the upgrade; rolling back to {prev}"));
|
||||
let result = self.rollback(Some(prev)).await;
|
||||
let body = match result {
|
||||
Ok(_) => json!({"type": "rolled_back", "actor": self.id, "from": from, "to": prev, "reason": "crash loop after upgrade", "last_exit": exit.to_string()}),
|
||||
Err(e) => json!({"type": "error", "actor": self.id, "reason": format!("automatic rollback failed: {e:#}")}),
|
||||
};
|
||||
if let Some(r) = self.upgrade_requester.clone() {
|
||||
self.broker.send("broker", Some(&r), None, None, body);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
let delay = Duration::from_millis(500 * 2u64.pow(self.restarts.min(6))).min(Duration::from_secs(30));
|
||||
self.restarts += 1;
|
||||
self.restart_at = Some(Instant::now() + delay);
|
||||
self.state = "restarting";
|
||||
self.status();
|
||||
}
|
||||
|
||||
fn snapshot_current(&self) -> Result<()> {
|
||||
let snap = self.world().version_dir(&self.id, self.version).join("snapshot");
|
||||
world::replace_tree(&self.world().data_dir(&self.id), &snap.join("data"))?;
|
||||
world::replace_tree(&self.world().public_dir(&self.id), &snap.join("public"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Make `version` current: compile it first, then write `actor.wasm`,
|
||||
/// `source/`, and `actor.json`. If compiling fails, nothing changes.
|
||||
fn install(&mut self, version: u64) -> Result<()> {
|
||||
let component = self.broker.load_version(&self.id, version)?;
|
||||
let dir = self.world().version_dir(&self.id, version);
|
||||
world::write_atomic(&self.world().wasm_path(&self.id), &std::fs::read(dir.join("actor.wasm"))?)?;
|
||||
world::replace_tree(&dir.join("source"), &self.world().source_dir(&self.id))?;
|
||||
self.broker.update_meta(&self.id, |m| {
|
||||
m.version = Some(version);
|
||||
m.running = true;
|
||||
})?;
|
||||
self.version = version;
|
||||
self.component = Some(component);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn set_running(&self, running: bool) -> Result<()> {
|
||||
self.broker.update_meta(&self.id, |m| m.running = running)
|
||||
}
|
||||
|
||||
/// Clear the restart and crash-loop state before a swap.
|
||||
fn reset_restarts(&mut self) {
|
||||
self.restarts = 0;
|
||||
self.restart_at = None;
|
||||
self.crashes_since_swap = 0;
|
||||
self.desired_running = true;
|
||||
}
|
||||
|
||||
async fn upgrade(&mut self, version: u64, requester: String) -> Result<Value> {
|
||||
self.stop_instance().await;
|
||||
self.snapshot_current()?;
|
||||
let previous = self.version;
|
||||
self.install(version)?;
|
||||
self.reset_restarts();
|
||||
self.previous = Some(previous);
|
||||
self.swapped_at = Some(Instant::now());
|
||||
self.upgrade_requester = Some(requester);
|
||||
self.redeliver = self.parked.take();
|
||||
if let Err(e) = self.start_instance().await {
|
||||
self.rollback(Some(previous)).await?;
|
||||
bail!("version {version} did not start ({e:#}); rolled back to {previous}");
|
||||
}
|
||||
Ok(json!({"type": "upgraded", "actor": self.id, "version": version, "previous": previous}))
|
||||
}
|
||||
|
||||
async fn rollback(&mut self, target: Option<u64>) -> Result<Value> {
|
||||
let target = match target {
|
||||
Some(t) => t,
|
||||
None => (1..self.version)
|
||||
.rev()
|
||||
.find(|v| self.world().version_dir(&self.id, *v).join("actor.wasm").is_file() && !self.broker.version_rejected(&self.id, *v))
|
||||
.ok_or_else(|| anyhow!("{} has no version before {}", self.id, self.version))?,
|
||||
};
|
||||
let dir = self.world().version_dir(&self.id, target);
|
||||
if !dir.join("actor.wasm").is_file() {
|
||||
bail!("{} has no version {target}", self.id);
|
||||
}
|
||||
self.broker.load_version(&self.id, target)?;
|
||||
self.stop_instance().await;
|
||||
self.snapshot_current()?;
|
||||
let from = self.version;
|
||||
let snap = dir.join("snapshot");
|
||||
if snap.is_dir() {
|
||||
world::replace_tree(&snap.join("data"), &self.world().data_dir(&self.id))?;
|
||||
world::replace_tree(&snap.join("public"), &self.world().public_dir(&self.id))?;
|
||||
}
|
||||
self.install(target)?;
|
||||
self.reset_restarts();
|
||||
self.previous = None;
|
||||
self.swapped_at = None;
|
||||
self.start_instance().await?;
|
||||
Ok(json!({"type": "rolled_back", "actor": self.id, "from": from, "to": target, "restored_state": snap.is_dir()}))
|
||||
}
|
||||
|
||||
/// Returns `true` when the supervisor must end.
|
||||
async fn handle_control(&mut self, ctl: Control) -> bool {
|
||||
match ctl {
|
||||
Control::Upgrade { version, requester, reply } => {
|
||||
let r = self.upgrade(version, requester).await;
|
||||
let _ = reply.send(r);
|
||||
}
|
||||
Control::Rollback { target, reply } => {
|
||||
let r = self.rollback(target).await;
|
||||
let _ = reply.send(r);
|
||||
}
|
||||
Control::Stop { reply } => {
|
||||
self.desired_running = false;
|
||||
self.restart_at = None;
|
||||
self.stop_instance().await;
|
||||
self.state = "stopped";
|
||||
let r = self.set_running(false);
|
||||
let _ = reply.send(r.map(|_| json!({"type": "stopped", "actor": self.id})));
|
||||
}
|
||||
Control::Start { reply } => {
|
||||
self.desired_running = true;
|
||||
self.restart_at = None;
|
||||
let r = self.set_running(true);
|
||||
let _ = reply.send(r.map(|_| json!({"type": "started", "actor": self.id})));
|
||||
}
|
||||
Control::Remove { reply } => {
|
||||
self.desired_running = false;
|
||||
self.stop_instance().await;
|
||||
let r = self.archive();
|
||||
let removed = r.is_ok();
|
||||
let _ = reply.send(r);
|
||||
if removed {
|
||||
return true;
|
||||
}
|
||||
self.state = "stopped";
|
||||
}
|
||||
Control::Shutdown { reply } => {
|
||||
self.stop_instance().await;
|
||||
self.state = "host stopped";
|
||||
self.status();
|
||||
let _ = reply.send(Ok(json!({"type": "shut_down"})));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
self.status();
|
||||
false
|
||||
}
|
||||
|
||||
/// Move the folder of the actor into `archive/`. Nothing is deleted.
|
||||
fn archive(&self) -> Result<Value> {
|
||||
let archive = self.world().root.join("archive");
|
||||
std::fs::create_dir_all(&archive)?;
|
||||
let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap_or_default().as_nanos();
|
||||
let dest = archive.join(format!("{}-{nanos}", self.id));
|
||||
std::fs::rename(self.world().actor_dir(&self.id), &dest)?;
|
||||
Ok(json!({"type": "removed", "actor": self.id, "archive": dest.display().to_string()}))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user