Watch the world folder: auto spawn/upgrade, broker watch/unwatch, retire scan

The host watches the whole world folder recursively with the notify crate
(any writer counts, .sausage-tmp-* skipped, ~100 ms coalescing per path).
A new or changed actor/<id>/actor.wasm (or new folder actor/<id>) is
booted as at start once quiet: new = spawn (tested first), changed =
upgrade. The broker `scan` request is gone; the boot pass stays, plus a
full pass when the OS watcher overflows.

New broker requests watch/unwatch {path} (saved in actor.json `watches`)
deliver fs_changed {path, op: write|create|remove|rename, to?} from `fs`,
not routed, dropped when the mailbox is full; overflow sends every watcher
one fs_changed {path:"/actor", op:"rescan"}.

Card #91 (multi-host step 1).

cli/tests/host.rs

@@ -108,15 +108,15 @@ async fn routes_replies_publishes_and_reports_undeliverable() {
 async fn broker_does_not_answer_messages_with_reply_to() {
     let world = create_world("broker-reply-to");
     let (host, mut mailbox) = start(&world).await;
-    let scanned = request(&host, &mut mailbox, json!({"type": "scan"})).await;
-    assert_eq!(scanned.body["type"], "scanned");
+    let untapped = request(&host, &mut mailbox, json!({"type": "untap"})).await;
+    assert_eq!(untapped.body["type"], "untapped");
 
     host.broker.send(
         "console",
         Some("broker"),
         None,
         Some("upstream-message"),
-        json!({"type": "scan"}),
+        json!({"type": "untap"}),
     );
     assert!(
         tokio::time::timeout(Duration::from_secs(1), mailbox.recv())
@@ -611,7 +611,10 @@ async fn unknown_broker_request_lists_the_known_ones() {
     let unknown = request(&host, &mut mailbox, json!({"type": "frobnicate"})).await;
     assert_eq!(unknown.body["type"], "error");
     let reason = unknown.body["reason"].as_str().unwrap();
-    assert!(reason.contains("remove") && reason.contains("tap"), "{reason}");
+    assert!(reason.contains("remove") && reason.contains("tap") && reason.contains("watch"), "{reason}");
+    // `scan` is retired: the host watches the world folder instead.
+    let scan = request(&host, &mut mailbox, json!({"type": "scan"})).await;
+    assert_eq!(scan.body["type"], "error");
     host.stop().await;
 }
 
@@ -974,3 +977,156 @@ async fn a_full_tapper_mailbox_drops_copies_without_blocking() {
     assert_eq!(ghost.try_recv().unwrap().body["type"], "tapped");
     host.stop().await;
 }
+
+/// Ask `actor` again and again (it may not exist yet) until `want` holds for
+/// the answer.
+async fn ask_until(
+    host: &Host,
+    mailbox: &mut mpsc::Receiver<Envelope>,
+    actor: &str,
+    body: Value,
+    want: impl Fn(&Value) -> bool,
+) -> Envelope {
+    tokio::time::timeout(Duration::from_secs(30), async {
+        loop {
+            let reply = ask(host, mailbox, actor, body.clone()).await;
+            if want(&reply.body) {
+                return reply;
+            }
+            tokio::time::sleep(Duration::from_millis(100)).await;
+        }
+    })
+    .await
+    .unwrap_or_else(|_| panic!("{actor} never gave the wanted answer"))
+}
+
+/// Every message that arrives until the mailbox is quiet for `quiet`.
+async fn drain(mailbox: &mut mpsc::Receiver<Envelope>, quiet: Duration) -> Vec<Envelope> {
+    let mut messages = Vec::new();
+    while let Ok(Some(message)) = tokio::time::timeout(quiet, mailbox.recv()).await {
+        messages.push(message);
+    }
+    messages
+}
+
+/// The `fs_changed` bodies among `messages` (all from `fs`).
+fn fs_events(messages: Vec<Envelope>) -> Vec<Value> {
+    messages
+        .into_iter()
+        .filter(|m| m.body["type"] == "fs_changed")
+        .map(|m| {
+            assert_eq!(m.from, "fs");
+            m.body
+        })
+        .collect()
+}
+
+#[tokio::test]
+async fn a_new_actor_wasm_on_disk_becomes_an_actor() {
+    let world = create_world("watch-spawn");
+    let (host, mut mailbox) = start(&world).await;
+    // Any writer counts: here the test itself, like a shell on the host.
+    let tests = world.source_dir("dropped").join("tests");
+    std::fs::create_dir_all(&tests).unwrap();
+    let test = json!({"steps": [{"send": {"type": "echo", "value": "candidate"}, "expect": {"type": "echo", "value": "candidate"}}]});
+    std::fs::write(tests.join("candidate.json"), test.to_string()).unwrap();
+    std::fs::copy(toolbox_wasm(), world.wasm_path("dropped")).unwrap();
+
+    let echo = json!({"type": "echo", "value": "hi"});
+    let reply = ask_until(&host, &mut mailbox, "dropped", echo, |b| b["type"] == "echo").await;
+    assert_eq!(reply.from, "dropped");
+    assert_eq!(world.read_meta("dropped").version, Some(1));
+    let meta = std::fs::read_to_string(world.version_dir("dropped", 1).join("meta.json")).unwrap();
+    assert_eq!(serde_json::from_str::<Value>(&meta).unwrap()["requested_by"], "scan");
+    host.stop().await;
+}
+
+#[tokio::test]
+async fn a_changed_actor_wasm_on_disk_is_an_upgrade() {
+    let world = create_world("watch-upgrade");
+    let wasm = toolbox_wasm().clone();
+    let (host, mut mailbox) = start(&world).await;
+    let source = passing_source(&world, "source");
+    spawn_type(&host, &mut mailbox, "changing", &wasm, &source).await;
+
+    // The same program plus a custom section: a different file.
+    let mut changed = std::fs::read(&wasm).unwrap();
+    changed.extend_from_slice(&[0, 4, 1, b'x', b'v', b'2']);
+    std::fs::write(world.wasm_path("changing"), changed).unwrap();
+
+    let version = json!({"type": "version"});
+    ask_until(&host, &mut mailbox, "changing", version, |b| b["version"] == "2").await;
+    assert_eq!(world.read_meta("changing").version, Some(2));
+    assert_eq!(world.versions("changing"), vec![1, 2]);
+    host.stop().await;
+}
+
+#[tokio::test]
+async fn watchers_get_one_event_per_change_from_any_writer() {
+    let (world, host, mut mailbox) = filesystem_world("watch-events").await;
+    let watch = json!({"type": "watch", "path": "/actor/writer/public/"});
+    let watching = request(&host, &mut mailbox, watch).await;
+    assert_eq!(watching.body, json!({"type": "watching", "path": "/actor/writer/public"}));
+    assert!(world.read_meta("console").watches.contains("/actor/writer/public"));
+    drain(&mut mailbox, Duration::from_millis(300)).await;
+
+    let quiet = Duration::from_millis(600);
+    let event = |path: &str, op: &str| json!({"type": "fs_changed", "path": format!("/actor/writer/public/{path}"), "op": op});
+
+    // An actor creates a file: one create, not create + write.
+    let write = json!({"type": "write", "path": "/public/a.txt", "text": "a"});
+    ask_type(&host, &mut mailbox, "writer", write, "ok").await;
+    assert_eq!(fs_events(drain(&mut mailbox, quiet).await), vec![event("a.txt", "create")]);
+
+    // The host (a shell, git, ...) changes it.
+    std::fs::write(world.public_dir("writer").join("a.txt"), "b").unwrap();
+    assert_eq!(fs_events(drain(&mut mailbox, quiet).await), vec![event("a.txt", "write")]);
+
+    // A rename is one event with `to`.
+    let rename = json!({"type": "rename", "path": "/public/a.txt", "to": "/public/b.txt"});
+    ask_type(&host, &mut mailbox, "writer", rename, "ok").await;
+    let mut renamed = event("a.txt", "rename");
+    renamed["to"] = json!("/actor/writer/public/b.txt");
+    assert_eq!(fs_events(drain(&mut mailbox, quiet).await), vec![renamed]);
+
+    std::fs::remove_file(world.public_dir("writer").join("b.txt")).unwrap();
+    assert_eq!(fs_events(drain(&mut mailbox, quiet).await), vec![event("b.txt", "remove")]);
+
+    // Nothing outside the watched path.
+    let outside = json!({"type": "write", "path": "/data/x", "text": "x"});
+    ask_type(&host, &mut mailbox, "writer", outside, "ok").await;
+    assert_eq!(fs_events(drain(&mut mailbox, quiet).await), Vec::<Value>::new());
+
+    let unwatch = json!({"type": "unwatch", "path": "/actor/writer/public"});
+    let unwatched = request(&host, &mut mailbox, unwatch).await;
+    assert_eq!(unwatched.body, json!({"type": "unwatched", "path": "/actor/writer/public"}));
+    assert!(world.read_meta("console").watches.is_empty());
+    let write = json!({"type": "write", "path": "/public/c.txt", "text": "c"});
+    ask_type(&host, &mut mailbox, "writer", write, "ok").await;
+    assert_eq!(fs_events(drain(&mut mailbox, quiet).await), Vec::<Value>::new());
+    host.stop().await;
+}
+
+#[tokio::test]
+async fn a_watcher_overflow_sends_one_rescan_and_runs_a_full_pass() {
+    let world = create_world("watch-overflow");
+    let wasm = toolbox_wasm().clone();
+    let (host, mut mailbox) = start(&world).await;
+    let mut bystander = host.mailbox("bystander");
+    let watch = json!({"type": "watch", "path": "/actor/elsewhere"});
+    assert_eq!(request(&host, &mut mailbox, watch).await.body["type"], "watching");
+
+    // A dropped actor that the full pass finds (before the watcher's quiet time).
+    let tests = world.source_dir("planted").join("tests");
+    std::fs::create_dir_all(&tests).unwrap();
+    let test = json!({"steps": [{"send": {"type": "echo", "value": "candidate"}, "expect": {"type": "echo", "value": "candidate"}}]});
+    std::fs::write(tests.join("candidate.json"), test.to_string()).unwrap();
+    std::fs::copy(&wasm, world.wasm_path("planted")).unwrap();
+    let report = host.broker.clone().overflowed().await;
+    assert!(report.iter().any(|r| r["type"] == "spawned" && r["actor"] == "planted"), "{report:?}");
+
+    let rescan = json!({"type": "fs_changed", "path": "/actor", "op": "rescan"});
+    assert_eq!(fs_events(drain(&mut mailbox, Duration::from_millis(600)).await), vec![rescan]);
+    assert!(bystander.try_recv().is_err(), "an actor without watches got the rescan");
+    host.stop().await;
+}