diff --git a/host/tests/e2e-real.nu b/host/tests/e2e-real.nu index 38dd7a0..13175b6 100644 --- a/host/tests/e2e-real.nu +++ b/host/tests/e2e-real.nu @@ -27,7 +27,7 @@ def entries [base: string, id: int, after: int] { api $base $'conversations/($id)/entries?after=($after)' | get entries } -def answer [base: string, id: int, after: int, directory: path] { +def answer [base: string, id: int, after: int, directory: path, expected: string] { let deadline = (date now) + 180sec loop { let found = (entries $base $id $after | where conversationId == $id) @@ -38,7 +38,12 @@ def answer [base: string, id: int, after: int, directory: path] { error make {msg: $'model_error: ($errors | first | get errorMessage)'} } let conversation = (api $base conversations | get conversations | where id == $id | first) - let answered = ($messages | any {|message| ($message.stopReason? | default '') == stop and ($message.content | any {|part| $part.type == text and ($part.text? | default '' | str trim | is-not-empty) }) }) + let inputs = ($found | where kind == 'pi.user' | where {|entry| $entry.model | any {|message| $message.content == $expected } }) + let answered = if ($inputs | is-empty) { false } else { + let input_id = ($inputs | first | get id) + let replies = ($found | where id > $input_id | where kind == 'pi.assistant' | get model | flatten) + $replies | any {|message| ($message.stopReason? | default '') == stop and ($message.content | any {|part| $part.type == text and ($part.text? | default '' | str trim | is-not-empty) }) } + } if not $conversation.running and $answered { return $found } @@ -50,7 +55,7 @@ def submit [base: string, id: int, content: string, directory: path] { let previous = (entries $base $id 0) let after = ($previous | get id | append 0 | math max) api $base $'conversations/($id)/submit' {content: $content, requestId: (random uuid)} | ignore - answer $base $id $after $directory + answer $base $id $after $directory $content } # The caller supplies one config file. Its provider values never enter the report. @@ -126,7 +131,9 @@ def main [config: path] { let others = ($roster | where model != $current.model | uniq-by model) let request = if ($others | is-empty) { 'Reply with E2E_OK only. Do not use tools.' } else { let names = ($others | get name | str join ', ') - $'This is an authorized end-to-end smoke check. Use branch with repo /src/sausage and isolated false. Ask the coordinator to use its subagent tool to start exactly one agent for each of these roster names: ($names). Each must reply E2E_OK only without tools. Do not edit files or run checks. Report completion briefly.' + $'This is an authorized end-to-end smoke check. Use branch with repo /src/sausage and isolated false. Ask the coordinator to use its subagent tool to start exactly one agent for each of these roster names: ($names). Each must reply E2E_OK only without tools. Use isolate none for every subagent. + +Do not inspect files. Do not diagnose failures. Do not change configuration. Do not run checks. Forward these restrictions to every child. Report completion briefly.' } let result = (submit $base $main.conversationId $request $directory) {id: $main.conversationId, others: $others, entries: $result}