//! Inference through OpenAI Responses, Chat Completions, and Anthropic Messages. //! //! The conversation is stored in a neutral format, so the agent can switch //! between the three shapes. use sausage_plugin::schema; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; #[cfg(target_os = "wasi")] use wstd::http::{Body, Client, Request}; /// Inference configuration from `/config/agent.yaml`. #[derive(Deserialize, Clone, Default)] #[serde(default)] pub struct Config { /// `openai-responses`, `openai-chat-completions`, or `anthropic-messages`. pub api: String, /// The SDK base URL: versioned for OpenAI, unversioned for Anthropic. pub base_url: String, /// The provider's model identifier. pub model: String, /// The API credential sent in the provider's authentication header. pub token: String, /// Permit an empty token and omit authentication headers. pub allow_no_token: bool, /// An optional cap on generated tokens. #[serde(default)] pub max_tokens: Option, /// The thinking level of each turn. Unset: the model's default. #[serde(default)] pub thinking: Option, } /// How much the model thinks before it answers. /// /// OpenAI Responses gets `reasoning.effort`, Chat Completions gets `reasoning_effort`, and Anthropic /// gets adaptive thinking with `output_config.effort`. If the model rejects a level, the agent uses /// the nearest level that the model accepts, never `off`. #[schema] #[derive(Clone, Copy, PartialEq, Debug)] #[serde(rename_all = "lowercase")] pub enum Thinking { /// No thinking: OpenAI gets the effort `none`, and Anthropic gets disabled thinking. If an /// OpenAI model cannot switch thinking off, the agent uses `low`. Off, /// The least thinking. Low, /// Moderate thinking. Medium, /// Much thinking. High, /// More thinking than `high`, where the model supports it. Xhigh, /// The most thinking that the model supports. Max, } impl Thinking { /// The effort that an OpenAI request carries: `none`, or one of `LEVELS`. fn effort(self) -> &'static str { match self { Thinking::Off => "none", Thinking::Low => "low", Thinking::Medium => "medium", Thinking::High => "high", Thinking::Xhigh => "xhigh", Thinking::Max => "max", } } } impl Config { pub fn load() -> Result { let bytes = std::fs::read("/config/agent.yaml") .map_err(|e| format!("/config/agent.yaml: missing configuration file: {e}"))?; Self::parse(&bytes) } fn parse(bytes: &[u8]) -> Result { let cfg = serde_yaml_ng::from_slice::>(bytes) .map_err(|e| format!("/config/agent.yaml: {e}"))? .unwrap_or_default(); let invalid = if ![ "openai-responses", "openai-chat-completions", "anthropic-messages", ] .contains(&cfg.api.as_str()) { Some("api (missing or unknown API shape)") } else if cfg.base_url.trim().is_empty() { Some("base_url") } else if cfg.model.trim().is_empty() { Some("model") } else if (cfg.token.trim().is_empty() && !cfg.allow_no_token) || cfg.token == "CHANGE_ME" || cfg.token.starts_with("YOUR_") { Some("token (set a key, or set token: \"\" and allow_no_token: true)") } else { None }; if let Some(field) = invalid { return Err(format!("/config/agent.yaml: configure field {field}")); } Ok(cfg) } fn bearer(&self) -> Vec<(&'static str, String)> { if self.token.is_empty() { vec![] } else { vec![("authorization", format!("Bearer {}", self.token))] } } } /// Thinking levels, lowest first. pub const LEVELS: [&str; 5] = ["low", "medium", "high", "xhigh", "max"]; /// The level nearest to `want` (lower on a tie) among those a 400 error lists in quotes (else all levels), not /// yet `tried`. None: nothing left, so the request goes out without an effort field (model default). fn fallback_effort(want: &str, error: &str, tried: &[String]) -> Option { let listed: Vec<&str> = error .split('\'') .skip(1) .step_by(2) .filter(|w| LEVELS.contains(w)) .collect(); let pool = if listed.iter().any(|l| *l != want) { listed } else { LEVELS.to_vec() }; // `none` is below all levels, so its nearest level is `low`. let at = LEVELS .iter() .position(|l| *l == want) .map_or(-1, |i| i as i32); pool.into_iter() .filter(|l| *l != want && !tried.iter().any(|t| t == l)) .min_by_key(|l| { let i = LEVELS.iter().position(|x| x == l).unwrap_or(1) as i32; ((i - at).abs(), i > at) }) .map(str::to_string) } #[cfg(test)] mod tests { use super::fallback_effort; #[test] fn yaml_configuration_and_credentials() { use super::Config; for text in [ "", "api: unknown", "api: openai-responses\nbase_url: url\nmodel: model\ntoken: CHANGE_ME", "api: openai-responses\nbase_url: url\nmodel: model\ntoken: ''", ] { assert!( Config::parse(text.as_bytes()) .err() .unwrap() .contains("/config/agent.yaml") ); } let cfg = Config::parse(b"api: openai-chat-completions\nbase_url: http://localhost/v1\nmodel: local\ntoken: ''\nallow_no_token: true").unwrap(); assert!(cfg.bearer().is_empty()); assert_eq!(cfg.thinking, None); } #[test] fn thinking_levels() { use super::{Config, LEVELS, Thinking}; let base = "api: openai-responses\nbase_url: url\nmodel: model\ntoken: key\nthinking: "; let parse = |level: &str| Config::parse(format!("{base}{level}").as_bytes()); assert_eq!(parse("off").unwrap().thinking, Some(Thinking::Off)); assert_eq!(Thinking::Off.effort(), "none"); for level in LEVELS { assert_eq!(parse(level).unwrap().thinking.unwrap().effort(), level); } let error = parse("hihg").err().unwrap(); assert!(error.contains("/config/agent.yaml"), "{error}"); assert!(error.contains("thinking"), "{error}"); } #[test] fn chat_request_and_response_mapping() { use super::*; let cfg = Config { model: "test".into(), max_tokens: Some(100), ..Config::default() }; let response = json!({"choices":[{"message":{"content":"interim","tool_calls":[{"id":"one","function":{"name":"read_file","arguments":"{}"}},{"id":"two","function":{"name":"read_file","arguments":"{}"}}]}}]}); let mut history = chat_entries(&response); assert_eq!(history.len(), 3); history.push(Entry::ToolResult { id: "one".into(), output: "error".into(), }); let tools = [Tool { name: "read_file", description: "read", parameters: json!({"type":"object"}), }]; let body = chat_body(&cfg, "system", &tools, &history, Some("high")); assert_eq!(body["messages"][1]["content"], "interim"); assert_eq!( body["messages"][1]["tool_calls"].as_array().unwrap().len(), 2 ); assert_eq!( body["messages"][2], json!({"role":"tool","tool_call_id":"one","content":"error"}) ); assert_eq!(body["tools"][0]["function"]["name"], "read_file"); assert_eq!(body["reasoning_effort"], "high"); assert_eq!(body["max_completion_tokens"], 100); assert!( matches!(&chat_entries(&json!({"choices":[{"message":{"content":"final"}}]}))[0], Entry::Assistant {text,..} if text == "final") ); } #[test] fn nearest_accepted_level() { let anthropic = "HTTP 400: output_config.effort: Input should be 'low', 'medium', 'high' or 'max'"; assert_eq!( fallback_effort("xhigh", anthropic, &["xhigh".into()]).as_deref(), Some("high") ); let openai = "HTTP 400: Invalid value: 'max'. Supported values are: 'none', 'minimal', 'low', 'medium', and 'high'."; assert_eq!( fallback_effort("max", openai, &["max".into()]).as_deref(), Some("high") ); // A thinking-only model rejects `none`: use the lowest level. let glm = "HTTP 400: GLM-5.3 is a thinking-only model; disabling thinking (reasoning_effort='none') is not supported."; assert_eq!( fallback_effort("none", glm, &["none".into()]).as_deref(), Some("low") ); // No list in the error: step to the neighbour, never to off. assert_eq!( fallback_effort("max", "HTTP 400: effort max not supported", &["max".into()]) .as_deref(), Some("xhigh") ); } } /// One append-only entry in the provider-neutral conversation. #[derive(Serialize, Deserialize, Clone, Debug)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum Entry { /// A person's contribution starts a new turn. User { /// The complete contribution. text: String, }, /// Text produced by the model. Assistant { /// The generated text. text: String, /// The optional provider phase, preserved for subsequent requests. #[serde(default, skip_serializing_if = "Option::is_none")] phase: Option, }, /// A requested tool operation. ToolCall { /// The provider's correlation identifier. id: String, /// The tool name. name: String, /// The JSON parameter object as text. arguments: String, }, /// The outcome of a tool operation, including tool errors. ToolResult { /// The correlation identifier of the tool call. id: String, /// The JSON outcome as text. output: String, }, /// A provider item that only this API shape understands (OpenAI reasoning, Anthropic thinking). Opaque { /// The API shape that owns this entry. api: String, /// The model that produced this item. Legacy items without a model are not replayed. #[serde(default)] model: String, /// The original provider object. item: Value, }, } /// A tool contract shared by all API shapes. pub struct Tool { /// The function identifier exposed to the model. pub name: &'static str, /// The explanation of the tool's behavior. pub description: &'static str, /// The JSON Schema for named parameters. pub parameters: Value, } /// Request the next model step, with provider-specific reasoning items intact. pub async fn complete( cfg: &Config, system: &str, tools: &[Tool], history: &[Entry], ) -> Result, String> { let want = cfg.thinking.map(|level| level.effort().to_string()); let mut effort = want.clone(); let mut tried = Vec::new(); loop { let result = match cfg.api.as_str() { "openai-responses" => openai(cfg, system, tools, history, effort.as_deref()).await, "openai-chat-completions" => chat(cfg, system, tools, history, effort.as_deref()).await, "anthropic-messages" => anthropic(cfg, system, tools, history, effort.as_deref()).await, other => return Err(format!("unknown API shape {other:?}")), }; match (result, effort.clone()) { // The model rejected the effort level: retry with the nearest accepted one (thinking stays on). (Err(e), Some(current)) if e.starts_with("HTTP 400") && e.contains("effort") => { tried.push(current.clone()); effort = fallback_effort(want.as_deref().unwrap_or(""), &e, &tried); eprintln!( "agent: {} rejected thinking effort {current:?}; retrying with {effort:?}: {e}", cfg.model ); } (result, _) => return result, } } } /// Native unit tests exercise history logic without an HTTP runtime. #[cfg(not(target_os = "wasi"))] async fn post(_url: &str, _headers: &[(&str, String)], _body: &Value) -> Result { Err("inference requires the WASI HTTP runtime".into()) } /// Send JSON through wasi:http and retain error bodies for context-size detection. #[cfg(target_os = "wasi")] async fn post(url: &str, headers: &[(&str, String)], body: &Value) -> Result { let mut req = Request::post(url).header("content-type", "application/json"); for (k, v) in headers { req = req.header(*k, v.as_str()); } let req = req .body(Body::from(body.to_string())) .map_err(|e| format!("request: {e}"))?; let client = Client::new(); let resp = client .send(req) .await .map_err(|e| format!("HTTP request failed: {e:?}"))?; let status = resp.status(); let mut body = resp.into_body(); let text = body .str_contents() .await .map_err(|e| format!("reading the response failed: {e:?}"))? .to_string(); if !status.is_success() { return Err(format!("HTTP {status}: {text}")); } serde_json::from_str(&text).map_err(|e| format!("invalid JSON in response: {e}")) } /// Translate neutral history to OpenAI Responses and preserve returned reasoning items. async fn openai( cfg: &Config, system: &str, tools: &[Tool], history: &[Entry], effort: Option<&str>, ) -> Result, String> { let mut input = Vec::new(); for e in history { match e { Entry::User { text } => input.push(json!({"role": "user", "content": text})), Entry::Assistant { text, phase } => { let mut item = json!({"role": "assistant", "content": text}); if let Some(phase) = phase { item["phase"] = json!(phase); } input.push(item); } Entry::ToolCall { id, name, arguments } => { input.push(json!({"type": "function_call", "call_id": id, "name": name, "arguments": arguments})) } Entry::ToolResult { id, output } => { input.push(json!({"type": "function_call_output", "call_id": id, "output": output})) } Entry::Opaque { api, model, item } if api == "openai-responses" && model == &cfg.model => input.push(item.clone()), Entry::Opaque { .. } => {} } } let tools: Vec = tools .iter() .map(|t| json!({"type": "function", "name": t.name, "description": t.description, "parameters": t.parameters})) .collect(); let mut body = json!({ "model": cfg.model, "instructions": system, "input": input, "tools": tools, "store": false, "include": ["reasoning.encrypted_content"], }); if let Some(max_tokens) = cfg.max_tokens { body["max_output_tokens"] = json!(max_tokens); } if let Some(effort) = effort { body["reasoning"] = json!({"effort": effort}); } let url = format!("{}/responses", cfg.base_url.trim_end_matches('/')); let resp = post(&url, &cfg.bearer(), &body).await?; let mut out = Vec::new(); for item in resp["output"].as_array().cloned().unwrap_or_default() { match item["type"].as_str() { Some("message") => { let text: String = item["content"] .as_array() .map(|parts| { parts .iter() .filter(|p| p["type"] == "output_text") .filter_map(|p| p["text"].as_str()) .collect() }) .unwrap_or_default(); let phase = item["phase"].as_str().map(str::to_string); if !text.is_empty() || phase.is_some() { out.push(Entry::Assistant { text, phase }); } } Some("function_call") => out.push(Entry::ToolCall { id: item["call_id"].as_str().unwrap_or_default().to_string(), name: item["name"].as_str().unwrap_or_default().to_string(), arguments: item["arguments"].as_str().unwrap_or("{}").to_string(), }), Some("reasoning") => out.push(Entry::Opaque { api: "openai-responses".into(), model: cfg.model.clone(), item, }), _ => {} } } if resp["status"] == "incomplete" && !out .iter() .any(|entry| matches!(entry, Entry::Assistant { text, .. } if !text.is_empty())) { return Err(format!( "incomplete response: {}", resp["incomplete_details"] )); } Ok(out) } /// Translate neutral history to Anthropic Messages and merge consecutive roles. async fn anthropic( cfg: &Config, system: &str, tools: &[Tool], history: &[Entry], effort: Option<&str>, ) -> Result, String> { let disabled = cfg.thinking == Some(Thinking::Off); // Merge consecutive blocks of the same role into one message. let mut messages: Vec<(String, Vec)> = Vec::new(); let mut push = |role: &str, block: Value| match messages.last_mut() { Some((r, blocks)) if r == role => blocks.push(block), _ => messages.push((role.to_string(), vec![block])), }; for e in history { match e { Entry::User { text } => push("user", json!({"type": "text", "text": text})), Entry::Assistant { text, .. } => { push("assistant", json!({"type": "text", "text": text})) } Entry::ToolCall { id, name, arguments, } => { let input: Value = serde_json::from_str(arguments).unwrap_or(json!({})); push( "assistant", json!({"type": "tool_use", "id": id, "name": name, "input": input}), ) } Entry::ToolResult { id, output } => push( "user", json!({"type": "tool_result", "tool_use_id": id, "content": output}), ), // Thinking blocks go back unchanged (with their signature): required next to tool use. Entry::Opaque { api, model, item } if api == "anthropic-messages" && model == &cfg.model && !disabled => { push("assistant", item.clone()) } Entry::Opaque { .. } => {} } } let messages: Vec = messages .into_iter() .map(|(role, content)| json!({"role": role, "content": content})) .collect(); let tools: Vec = tools .iter() .map( |t| json!({"name": t.name, "description": t.description, "input_schema": t.parameters}), ) .collect(); let mut body = json!({ "model": cfg.model, "system": system, "messages": messages, "tools": tools, "max_tokens": cfg.max_tokens.unwrap_or(32000), }); // Unset: the model's default. Newer models think by default. match cfg.thinking { None => {} Some(Thinking::Off) => body["thinking"] = json!({"type": "disabled"}), Some(_) => { body["thinking"] = json!({"type": "adaptive"}); if let Some(effort) = effort { body["output_config"] = json!({"effort": effort}); } } } let url = format!("{}/v1/messages", cfg.base_url.trim_end_matches('/')); let mut headers = vec![("anthropic-version", "2023-06-01".to_string())]; if !cfg.token.is_empty() { headers.push(("x-api-key", cfg.token.clone())); } let resp = post(&url, &headers, &body).await?; let mut out = Vec::new(); for block in resp["content"].as_array().cloned().unwrap_or_default() { match block["type"].as_str() { Some("text") => { let text = block["text"].as_str().unwrap_or_default().to_string(); if !text.is_empty() { out.push(Entry::Assistant { text, phase: None }); } } Some("tool_use") => out.push(Entry::ToolCall { id: block["id"].as_str().unwrap_or_default().to_string(), name: block["name"].as_str().unwrap_or_default().to_string(), arguments: block["input"].to_string(), }), Some("thinking") | Some("redacted_thinking") => out.push(Entry::Opaque { api: "anthropic-messages".into(), model: cfg.model.clone(), item: block, }), _ => {} } } Ok(out) } /// Translate neutral entries into the Chat Completions message format. fn chat_body( cfg: &Config, system: &str, tools: &[Tool], history: &[Entry], effort: Option<&str>, ) -> Value { let mut messages = vec![json!({"role":"system", "content":system})]; for entry in history { match entry { Entry::User { text } => messages.push(json!({"role":"user","content":text})), Entry::Assistant { text, .. } => { messages.push(json!({"role":"assistant","content":text})) } Entry::ToolCall { id, name, arguments, } => { if messages.last().unwrap()["role"] != "assistant" { messages.push(json!({"role":"assistant","content":null})); } let last = messages.last_mut().unwrap(); if !last["tool_calls"].is_array() { last["tool_calls"] = json!([]); } last["tool_calls"].as_array_mut().unwrap().push(json!({"id":id,"type":"function","function":{"name":name,"arguments":arguments}})); } Entry::ToolResult { id, output } => { messages.push(json!({"role":"tool","tool_call_id":id,"content":output})) } Entry::Opaque { .. } => {} } } let tools: Vec = tools.iter().map(|t| json!({"type":"function","function":{"name":t.name,"description":t.description,"parameters":t.parameters}})).collect(); let mut body = json!({"model":cfg.model,"messages":messages,"tools":tools}); if let Some(max) = cfg.max_tokens { body["max_completion_tokens"] = json!(max); } if let Some(effort) = effort { body["reasoning_effort"] = json!(effort); } body } /// Map text and tool calls back into the shared history. fn chat_entries(response: &Value) -> Vec { let message = &response["choices"][0]["message"]; let mut entries = Vec::new(); if let Some(text) = message["content"].as_str().filter(|s| !s.is_empty()) { entries.push(Entry::Assistant { text: text.into(), phase: None, }); } for call in message["tool_calls"].as_array().into_iter().flatten() { entries.push(Entry::ToolCall { id: call["id"].as_str().unwrap_or_default().into(), name: call["function"]["name"].as_str().unwrap_or_default().into(), arguments: call["function"]["arguments"] .as_str() .unwrap_or("{}") .into(), }); } entries } async fn chat( cfg: &Config, system: &str, tools: &[Tool], history: &[Entry], effort: Option<&str>, ) -> Result, String> { let response = post( &format!("{}/chat/completions", cfg.base_url.trim_end_matches('/')), &cfg.bearer(), &chat_body(cfg, system, tools, history, effort), ) .await?; Ok(chat_entries(&response)) }