From 56d02befc8878434d6a2a9aa8acf3bf8c9c11dee Mon Sep 17 00:00:00 2001 From: Surma Date: Thu, 15 Oct 2026 05:06:24 +0100 Subject: [PATCH] Spike: console /await, DECISIONS.md, SPIKE.md --- DECISIONS.md | 154 +++++++++++++++++++++++++++++++++++++++++++++ README.md | 1 + SPIKE.md | 107 +++++++++++++++++++++++++++++++ cli/src/console.rs | 32 +++++++++- 4 files changed, 293 insertions(+), 1 deletion(-) create mode 100644 DECISIONS.md create mode 100644 SPIKE.md diff --git a/DECISIONS.md b/DECISIONS.md new file mode 100644 index 0000000..cea509d --- /dev/null +++ b/DECISIONS.md @@ -0,0 +1,154 @@ +# Decisions + +This file lists the design decisions of the spike. **(you)** marks a decision +that Surma made. **(spike)** marks a decision that Scout made alone during the +night and that needs review. + +## Architecture + +- **(you)** Actors are WASI commands (`wasm32-wasip2`). They use stdin, stdout, + files, and HTTP. No WIT interfaces, no component composition. +- **(you)** Actors exchange newline-delimited JSON through the host, which is + the broker. No built-in request/response. A reply is a message with + `reply_to`. Delivery is at most once. +- **(you)** The broker accepts only `.wasm` files. It knows no source language. +- **(you)** `cargo` is a built-in Rust compiler actor. **(spike)** It is native + (host code), because cargo cannot run inside WASM. +- **(you)** The agent is a normal WASM actor with the fixed ID `agent`. It can + read and upgrade its own source. +- **(spike)** Three native actors: `broker`, `console`, `cargo`. +- **(you)** One Cargo workspace with two crates: `cli` (binary `sausage`) and + `agent`. **(spike)** Example actors are in `examples/`, outside the workspace. +- **(you)** `sausage init` builds the agent and asks for the API shape, the + endpoint, the model, and the token. **(spike)** The token can also come from + `SAUSAGE_API_TOKEN`. The source contains no endpoint and no token. + +## Messages + +- **(spike)** Stdin line: `{"id", "from", "reply_to"?, "body"}`. + Stdout line: `{"to"?, "id"?, "reply_to"?, "body"}`. +- **(spike)** The broker sets `from`. An actor can choose its own `id`. The + broker then uses `"/"`, so IDs cannot collide or be faked. + Without an `id`, the broker assigns `m-`. +- **(you)** Without `to`, a message goes to the subscribers of the sender. + **(spike)** `subscribe` / `unsubscribe` messages to the broker. The registry + stores subscriptions, so they survive restarts and upgrades. `init` + subscribes `console` to `agent`. +- **(spike)** Errors from routing come back as + `{"type": "undeliverable", "reason", "to"}` from `broker`. +- **(spike)** Lifecycle: `{"type": "shutdown"}` from `broker`, then EOF on stdin. + After 5 seconds the broker kills the instance. +- **(spike)** Lines on stdout that are not JSON objects with `body` go to the + actor's `stderr.log` with a note. + +## Filesystem + +- **(you)** `/data` private, `/public` own public files, + `/actor//public` and `/actor//source` read-only for everyone. +- **(spike)** The broker also writes `/actor//status.json` and + `/actor//stderr.log`. `ls /actor` is the registry. +- **(you)** Commit on close: a write becomes visible only when the writer + closes the file. **(spike)** This also applies to `/data`. Implementation: a + wrapper around the `wasmtime-wasi` filesystem. An open for write goes to a + hidden `.sausage-tmp-*` file next to the target. The descriptor drop renames + it into place. Details: + - A normal exit commits every open file (process exit closes files). + - A trap or kill discards uncommitted writes. + - An append or an edit without truncate copies the old content first. + - Other descriptors see the old content until the close. + - Directory operations take effect immediately. + - The layer hides `.sausage-tmp-*` names from listings, `stat`, `open`, + `rename`, and `unlink`. + - Symlinks are disabled (they would break the path tracking). + - No `fsync`. A power loss can lose committed data. +- **(spike)** Raw sockets are not available (no network permission), but the + `wasi:sockets` interfaces are linked, so programs that import them still + start. + +## Broker requests + +- **(spike)** `spawn`, `upgrade`, `rollback`, `stop`, `start`, `remove`, + `subscribe`, `unsubscribe`, `list`. +- **(you)** `spawn` and `upgrade` take `wasm`, plus optional `source` and + `built_by` metadata. **(spike)** The broker copies the `.wasm` file and the + source folder into `broker/versions///`, with `meta.json` (paths, + builder, requester, time). +- **(spike)** Path resolution: `/actor/...` is global, and `/public/...` and + `/data/...` belong to the sender. `..` is rejected. Native senders (the + console) can give absolute host paths. +- **(you)** Anyone can upgrade anyone. No permissions. +- **(spike)** Actor IDs: `a-z`, `0-9`, `-`, `_`, at most 64 characters. +- **(spike)** `remove` archives every folder of the actor to `archive/`. Nothing + is deleted. + +## Upgrades, tests, rollback + +- **(spike)** The tests are message transcripts in `source/tests/*.jsonl`: + `{"send": body}` and `{"expect": subset-of-body, "to"?}`. Each file runs in + a fresh candidate on a copy of the actor's `/data` and `/public`. The + candidate has no HTTP access. An `expect` waits up to 10 s. Without tests, + the candidate must stay up for 2 s (smoke test). +- **(spike)** Swap: hold the mailbox, stop the old instance gracefully, + snapshot `/data` + `/public` into the old version's folder, install the new + source in `/actor//source`, start the new instance, release the mailbox. +- **(spike)** Restart after a crash: backoff 0.5 s × 2ⁿ, at most 30 s. The + counter resets after 60 s of uptime. +- **(spike)** Parked message: when an instance traps, the broker keeps the + last delivered message. It delivers that message again after the next + upgrade, not after a plain restart (that would loop). +- **(spike)** Crash loop: 3 crashes within 30 s after an upgrade → automatic + rollback to the previous version, with its data snapshot. The broker + informs the actor that requested the upgrade. +- **(spike)** `rollback` without `version` goes to the highest older version. + It restores that version's snapshot, and it first snapshots the current + state, so a roll-forward is possible. +- **(spike)** Versions are per actor, not global. + +## Limits + +- **(spike)** Mailbox: 1,000 messages. When it is full, the sender gets + `undeliverable`. +- **(spike)** The broker waits 30 s when it writes to an actor's stdin. Then it + drops the message and logs it. +- **(spike)** Time slices: the epoch advances every 10 ms, and each guest + yields at each epoch. +- **(spike)** One `sausage run` per world (file lock on `world/.lock`). +- **(spike)** HTTP for all running actors, with no allow-list yet. + +## Agent + +- **(you)** API shapes: OpenAI Responses and Anthropic Messages, chosen by + `"api"` in `/data/config.json`. +- **(spike)** Config fields: `api`, `base_url`, `model`, `token`, optional + `reasoning_effort` and `max_tokens` (16,000). OpenAI: `POST + {base_url}/v1/responses` with `store: false` and encrypted reasoning passed + back. Anthropic: `POST {base_url}/v1/messages` with `x-api-key`. +- **(spike)** One shared conversation in `/data/history.json`, in a neutral + format, so the API shape can change. The history is limited to 600 entries, + with the cut at a user message. There is no compaction yet. +- **(spike)** A turn starts only for `{"type": "text"}` messages *without* + `reply_to`. Replies never start turns, so two actors cannot keep the agent + in an endless loop. Other messages are collected and shown at the next + turn. +- **(spike)** Tools: `send_message` (with optional wait for the reply), + `wait_for_messages`, `read_file`, `list_dir`, `write_file`, `copy_dir`, + `remove_path`. At most 40 steps per turn. Tool output is cut at 30 KB. +- **(spike)** The final text of a turn goes to the sender, as a reply to the + message that started the turn. +- **(spike)** A shutdown during a turn saves the turn to `/data/turn.json`. The + next start (for example after a self-upgrade) resumes the turn and answers + the original message. + +## Console + +- **(spike)** `@ `, plain text to `agent`, `/deploy`, + `/list`, `/sleep`, `/await`, `/help`, `/quit`. `/sleep` and `/await` exist + for scripted tests. + +## Known risks (accepted for now) + +- Build scripts and proc macros run as native code in the host environment. +- Any actor can upgrade the agent with code that reads the token. +- Any actor can message, stop, upgrade, or remove any actor. +- A full copy of `/data` and `/public` at every upgrade is slow for large data. +- Mailboxes are in memory. A host crash loses queued messages. diff --git a/README.md b/README.md index 4a300d8..e164134 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ hello → text to the agent @agent what actors exist? → text to an actor /deploy notes examples/notes → build with cargo, then spawn or upgrade /list → list actors +/await agent 300 → wait for the next message (scripts) /quit ``` diff --git a/SPIKE.md b/SPIKE.md new file mode 100644 index 0000000..359fbfc --- /dev/null +++ b/SPIKE.md @@ -0,0 +1,107 @@ +# Spike results + +Date: 2026-10-15. Host: Scout container (8 cores), Rust 1.99, Wasmtime 49.0.3, +`wstd` 0.6.8. All numbers come from `sausage run` with release builds. + +## 1. Host, broker, routing, filesystem — works + +- `sausage init` builds the agent and registers it. `sausage run` starts the + broker, `cargo`, the console, and every registered actor. +- Routing, replies (`reply_to`), publish to subscribers, and `undeliverable` + errors work. +- **Read-only:** a write to `/actor//public/...` fails with "Operation + not permitted". A write to `/actor//public/...` fails too. A write to + `/public/...` works. +- **Commit on close:** while `fs-a` holds `/public/y.txt` open, `fs-b` gets + "No such file". `fs-a` itself also gets it through a second descriptor. The + temporary file does not appear in listings. After the close, `fs-b` reads + the content. +- **Append:** `append` to an existing file copies the old content. Readers + see the old version until the close. +- **Crash with an open file:** the write is discarded, and the old version + stays. +- **Escapes:** `/data/../../../../etc/hostname` and + `/actor/../private/agent/data/config.json` fail. +- **Hard kill of the host:** after `kill`, no `.sausage-tmp-*` files remained, + and the next `sausage run` started every actor again. + +## 2. Async stdin + HTTPS in one actor — works + +`examples/fetch` (`wstd`) handles `ping` at once while a 2-second HTTPS +request runs. HTTPS and plain HTTP both work through `wasi:http`, and the host +does TLS. Bodies go to `/public/fetch/`, and the reply contains the +global path. + +One server (`http://neverssl.com/`) never answered. `curl` also hangs on it, +so the server is the cause. But a request without a timeout keeps its task +alive, and the actor then did not exit on `shutdown`. The broker killed it +after 5 s, as designed. Actors should set HTTP timeouts. + +## 3. Upgrade cycle — works + +- **No loss during a swap:** `ticker` sent 200 messages to `notes` at 50 ms + intervals. `notes` was upgraded from v1 to v2 during the stream. The swap + happened after tick 68. The v1 snapshot has ticks 1–68, and v2 has ticks + 1–200, in order, with no gaps. +- **Parked message:** `{"type":"crash"}` crashed v1. The broker restarted v1 + and kept the message. After the upgrade to v2 (which handles `crash`), the + broker delivered it again, and v2 answered `not_crashing`. +- **Crash loop:** an upgrade to `examples/crasher` passed the smoke test (it + crashes after 3 s). It crashed 3 times in about 10 s. The broker rolled back + to v2 with its data and sent `rolled_back` to the console. +- **Explicit rollback** v2 → v1 restored the v1 snapshot. +- **Rejections:** a compiler error comes back as `build_failed` with the + rendered diagnostics. A failing transcript rejects the upgrade, and the old + version keeps running. Invalid IDs and `..` paths are rejected. + +## 4. Agent — works with both API shapes + +- **`gpt-6-luna`** (OpenAI Responses) through the vendors proxy: + - Asked "which actors exist", it called `broker` `list` and answered in + about 4 s (2 inferences). + - Asked for a `counter` actor, it took these steps without help: + 1. It wrote `Cargo.toml`, `src/main.rs`, and `tests/basic.jsonl`. + 2. It built the crate with `cargo` (1.9 s). + 3. It spawned the actor (the tests passed). + 4. It sent increment, increment, get (1, 2, 2). + 5. It read `/actor/counter/public/value.txt`. +- **Self-upgrade:** asked to change its own prompt, it took these steps: + 1. It copied its source and changed only `prompt.md` (a 3-line diff). + 2. It built the change and upgraded `agent`. + 3. The upgrade stopped it in the middle of the turn. The new version + resumed the turn from `/data/turn.json` and answered with the new + signature. +- **`claude-haiku-4-5`** (Anthropic Messages): the same conversation + continued, with tool calls. + +## 5. Measurements + +| What | Result | +| --- | --- | +| Round trip actor → actor → actor | p50 88 µs, p99 172 µs | +| Sequential round trips | ~10,400 per second | +| Warm `cargo` build, small actor | 0.5–1.9 s | +| Warm `cargo` build, agent | 5.5 s | +| First build of the agent (`init`) | 17 s | +| Component compile, small actor | ~70 ms | +| Component compile, agent (870 KB) | ~290 ms | +| Host start, 8 actors compiled in parallel | ~0.8 s each | +| Host RSS with 8 running actors | ~117 MB | +| First release build of `sausage` | ~9 min (Wasmtime) | + +Caveat: `~/.cargo/config.toml` in this container sets `kache` as the +`rustc-wrapper`. `kache` caches compiler outputs across projects, so the +first build of the agent (17 s) is probably faster than on a clean machine. + +## Next steps I suggest + +1. A `matrix` actor (WASM, `wstd`, long-polled `/sync`). The agent could write + it, with you checking the result. +2. Cache compiled components (`Component::serialize`, `.cwasm` files) to + remove the compile time at host start. +3. A network allow-list per actor, for example in a manifest in the source + folder. +4. A sandbox for builds (for example `bwrap`), because build scripts run + natively. +5. Context compaction for the agent's conversation. +6. A better console (line editing, history). diff --git a/cli/src/console.rs b/cli/src/console.rs index 95c871c..c739c56 100644 --- a/cli/src/console.rs +++ b/cli/src/console.rs @@ -28,12 +28,16 @@ const HELP: &str = "\ /deploy build a Rust folder with cargo, then spawn or upgrade /list list actors /sleep wait (for scripts) +/await wait for the next message from an actor (for scripts) /quit stop all actors and exit"; pub struct Console { broker: Arc, waiters: Mutex>>, seq: AtomicU64, + /// Number of messages received per sender, for `/await`. + seen: Mutex>, + arrived: tokio::sync::Notify, } fn show(env: &Envelope) { @@ -65,12 +69,20 @@ fn show(env: &Envelope) { impl Console { pub fn new(broker: Arc) -> Arc { - Arc::new(Self { broker, waiters: Mutex::new(HashMap::new()), seq: AtomicU64::new(1) }) + Arc::new(Self { + broker, + waiters: Mutex::new(HashMap::new()), + seq: AtomicU64::new(1), + seen: Mutex::new(HashMap::new()), + arrived: tokio::sync::Notify::new(), + }) } pub async fn run_mailbox(self: Arc, mut mailbox: mpsc::Receiver) { while let Some(env) = mailbox.recv().await { show(&env); + *self.seen.lock().unwrap().entry(env.from.clone()).or_default() += 1; + self.arrived.notify_waiters(); if let Some(r) = &env.reply_to && let Some(tx) = self.waiters.lock().unwrap().remove(r) { @@ -161,6 +173,24 @@ impl Console { let secs: f64 = parts.next().and_then(|s| s.parse().ok()).unwrap_or(1.0); tokio::time::sleep(Duration::from_secs_f64(secs)).await; } + Some("await") => { + let from = parts.next().unwrap_or("agent").to_string(); + let secs: f64 = parts.next().and_then(|s| s.parse().ok()).unwrap_or(60.0); + let count = |c: &Self| c.seen.lock().unwrap().get(&from).copied().unwrap_or(0); + let before = count(self); + let deadline = tokio::time::Instant::now() + Duration::from_secs_f64(secs); + loop { + let notified = self.arrived.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if count(self) != before { + break; + } + if tokio::time::timeout_at(deadline, notified).await.is_err() { + bail!("no message from {from} within {secs} s"); + } + } + } Some("deploy") => { let (Some(actor), Some(dir)) = (parts.next(), parts.next()) else { bail!("usage: /deploy ");