Supervise boot services with uniform five-second upgrade grace

cli/src/services.rs

@@ -1,12 +1,36 @@
-//! Ordered boot services and their identities for upgrade restarts (PROTOCOL.md section 8).
-use crate::Host;
+//! Ordered boot calls and the single restart owner for each service (PROTOCOL.md section 8).
+use crate::{Host, host::State};
 use serde_json::{Value, json};
-#[derive(Clone)]
+use std::{collections::VecDeque, sync::atomic::Ordering, time::Duration};
+use tokio::time::Instant;
+
+// Agreed systemd-derived values: /docs/design/plugin-handover/design.md,
+// https://www.man7.org/linux/man-pages/man5/systemd-system.conf.5.html.
+const CRASH_LIMIT: usize = 5;
+const CRASH_WINDOW: Duration = Duration::from_secs(10);
+
 pub(crate) struct Service {
     pub(crate) method: String,
     pub(crate) params: Value,
     pub(crate) request: Value,
-    pub(crate) starting: std::sync::Arc<std::sync::atomic::AtomicBool>,
+    pub(crate) owner: Option<String>,
+    pub(crate) pending: bool,
+    pub(crate) version: Option<String>,
+    pub(crate) enabled: bool,
+    crashes: VecDeque<Instant>,
+}
+impl Service {
+    fn crashed(&mut self, now: Instant) -> bool {
+        while self
+            .crashes
+            .front()
+            .is_some_and(|t| now.duration_since(*t) >= CRASH_WINDOW)
+        {
+            self.crashes.pop_front();
+        }
+        self.crashes.push_back(now);
+        self.crashes.len() <= CRASH_LIMIT
+    }
 }
 impl Host {
     /// Starts discovery in the background, then waits for each boot service in order.
@@ -24,30 +48,119 @@ impl Host {
         }
         Ok(())
     }
-    /// Records a unique request identity before the service can answer or exit.
+    /// Register once, before admission or readiness. Restarts reuse this registration.
     pub(crate) async fn start_service(&self, method: String, params: Value) {
-        if self.0.shutdown.load(std::sync::atomic::Ordering::SeqCst) {
-            return;
-        }
         let request = json!(uuid::Uuid::now_v7().to_string());
-        let starting = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true));
-        self.0.state.lock().unwrap().services.push(Service {
-            method: method.clone(),
-            params: params.clone(),
-            request: request.clone(),
-            starting: starting.clone(),
-        });
+        {
+            let mut state = self.0.state.lock().unwrap();
+            if self.0.shutdown.load(Ordering::SeqCst) {
+                return;
+            }
+            state.services.push(Service {
+                method,
+                params,
+                request: request.clone(),
+                owner: None,
+                enabled: true,
+                pending: true,
+                version: None,
+                crashes: VecDeque::new(),
+            });
+        }
+        self.run_service(request).await;
+    }
+    /// Await readiness only; process cleanup, not this response, decides whether to restart.
+    async fn run_service(&self, request: Value) {
+        let (method, params) = {
+            let state = self.0.state.lock().unwrap();
+            if self.0.shutdown.load(Ordering::SeqCst) {
+                return;
+            }
+            let Some(service) = state
+                .services
+                .iter()
+                .find(|s| s.request == request && s.enabled)
+            else {
+                return;
+            };
+            (service.method.clone(), service.params.clone())
+        };
         self.0
             .journal
             .write("core", "core", "core", &format!("service start {method}"));
-        if let Err(error) = self
-            .route("core".into(), Some(request), method, params, true)
-            .await
+        let result = self
+            .route("core".into(), Some(request.clone()), method, params, true)
+            .await;
         {
+            let mut state = self.0.state.lock().unwrap();
+            if let Some(service) = state.services.iter_mut().find(|s| s.request == request) {
+                service.pending = false;
+            }
+        }
+        if let Err(error) = result {
+            // Admission failures have no process lifetime to supervise. A process crash
+            // before readiness is handled by finish just like one after readiness.
             self.0
                 .journal
                 .write("core", "core", "core", &format!("service failed {error}"));
         }
-        starting.store(false, std::sync::atomic::Ordering::SeqCst);
+    }
+    /// Called under the state lock after the old owner and its files/locks are gone.
+    pub(crate) fn service_ended(
+        &self,
+        state: &mut State,
+        pid: &str,
+        status: &str,
+        stopped: bool,
+        superseded: bool,
+    ) {
+        let Some(service) = state
+            .services
+            .iter_mut()
+            .find(|s| s.owner.as_deref() == Some(pid))
+        else {
+            return;
+        };
+        service.owner = None;
+        if self.0.shutdown.load(Ordering::SeqCst) || !service.enabled {
+            return;
+        }
+        if !superseded {
+            if stopped {
+                service.enabled = false;
+                return;
+            }
+            if status == "exit 0" {
+                return;
+            }
+            if !service.crashed(Instant::now()) {
+                service.enabled = false;
+                self.0.journal.write(
+                    "core",
+                    "core",
+                    "core",
+                    &format!(
+                        "service gave up {}: more than {CRASH_LIMIT} crashes within {} seconds",
+                        service.method,
+                        CRASH_WINDOW.as_secs(),
+                    ),
+                );
+                return;
+            }
+        }
+        self.queue_service(service);
+    }
+    /// Reserve the next attempt before spawning it; upgrades and crashes share this owner.
+    pub(crate) fn queue_service(&self, service: &mut Service) {
+        service.request = json!(uuid::Uuid::now_v7().to_string());
+        service.pending = true;
+        let request = service.request.clone();
+        let host = self.clone();
+        tokio::spawn(async move {
+            host.run_service(request).await;
+        });
     }
 }
+
+#[cfg(test)]
+mod tests;