surma/sausage
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Match real-check replies to their submitted inputs

Browse Source
This commit is contained in:
surma
2026-11-12 19:55:51 +00:00
parent 3ba35d3ce6
commit d59c94a0af
1 changed files with 11 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
host/tests/e2e-real.nu
+11 -4
View File
@@ -27,7 +27,7 @@ def entries [base: string, id: int, after: int] {
api $base $'conversations/($id)/entries?after=($after)' | get entries
}
def answer [base: string, id: int, after: int, directory: path] {
def answer [base: string, id: int, after: int, directory: path, expected: string] {
let deadline = (date now) + 180sec
loop {
let found = (entries $base $id $after | where conversationId == $id)
@@ -38,7 +38,12 @@ def answer [base: string, id: int, after: int, directory: path] {
error make {msg: $'model_error: ($errors | first | get errorMessage)'}
}
let conversation = (api $base conversations | get conversations | where id == $id | first)
let answered = ($messages | any {|message| ($message.stopReason? | default '') == stop and ($message.content | any {|part| $part.type == text and ($part.text? | default '' | str trim | is-not-empty) }) })
let inputs = ($found | where kind == 'pi.user' | where {|entry| $entry.model | any {|message| $message.content == $expected } })
let answered = if ($inputs | is-empty) { false } else {
let input_id = ($inputs | first | get id)
let replies = ($found | where id > $input_id | where kind == 'pi.assistant' | get model | flatten)
$replies | any {|message| ($message.stopReason? | default '') == stop and ($message.content | any {|part| $part.type == text and ($part.text? | default '' | str trim | is-not-empty) }) }
}
if not $conversation.running and $answered {
return $found
}
@@ -50,7 +55,7 @@ def submit [base: string, id: int, content: string, directory: path] {
let previous = (entries $base $id 0)
let after = ($previous | get id | append 0 | math max)
api $base $'conversations/($id)/submit' {content: $content, requestId: (random uuid)} | ignore
answer $base $id $after $directory
answer $base $id $after $directory $content
}
# The caller supplies one config file. Its provider values never enter the report.
@@ -126,7 +131,9 @@ def main [config: path] {
let others = ($roster | where model != $current.model | uniq-by model)
let request = if ($others | is-empty) { 'Reply with E2E_OK only. Do not use tools.' } else {
let names = ($others | get name | str join ', ')
$'This is an authorized end-to-end smoke check. Use branch with repo /src/sausage and isolated false. Ask the coordinator to use its subagent tool to start exactly one agent for each of these roster names: ($names). Each must reply E2E_OK only without tools. Do not edit files or run checks. Report completion briefly.'
$'This is an authorized end-to-end smoke check. Use branch with repo /src/sausage and isolated false. Ask the coordinator to use its subagent tool to start exactly one agent for each of these roster names: ($names). Each must reply E2E_OK only without tools. Use isolate none for every subagent.
Do not inspect files. Do not diagnose failures. Do not change configuration. Do not run checks. Forward these restrictions to every child. Report completion briefly.'
}
let result = (submit $base $main.conversationId $request $directory)
{id: $main.conversationId, others: $others, entries: $result}