Complete the P3 landing exit test across three SIGKILL boundaries

This commit is contained in:
2026-11-07 19:39:16 +00:00
parent c3c5b36d09
commit 0e81e16f5c
5 changed files with 257 additions and 148 deletions
+7 -2
View File
@@ -1,7 +1,7 @@
/** Stateless role/input rules for deterministic faux-model replay in integration tests. */
import { fauxProvider, fauxAssistantMessage, fauxToolCall } from '@earendil-works/pi-ai/providers/faux';
export type Answer = { text?: string; tools?: { name: string; args: any }[] };
export type Rule = { id: string; role: string; match: { kind?: string; text: string }; answer: Answer };
export type Rule = { id: string; role: string; match: { kind?: string; text: string }; answer: Answer | ((transcript: any) => Answer) };
export const textOf = (content: any): string => typeof content === 'string' ? content : (content ?? []).filter((part: any) => part.type === 'text').map((part: any) => part.text).join('\n');
export function answerFor(rules: Rule[], transcript: any) {
const system = JSON.stringify(transcript.messages.filter((message: any) => message.role === 'system'));
@@ -9,7 +9,7 @@ export function answerFor(rules: Rule[], transcript: any) {
const last = transcript.messages.findLast((message: any) => message.role !== 'system');
const rule = rules.find(rule => rule.role === role && (!rule.match.kind || rule.match.kind === last?.role) && textOf(last?.content).includes(rule.match.text));
if (!rule) throw new Error(`No scripted rule for ${role}: ${JSON.stringify(last)}`);
return { rule: rule.id, answer: rule.answer };
return { rule: rule.id, answer: typeof rule.answer === 'function' ? rule.answer(transcript) : rule.answer };
}
export function scriptedModel(rules: Rule[], report: (rule: string) => void) {
const faux = fauxProvider({ tokensPerSecond: 100000 });
@@ -33,6 +33,11 @@ export function testReplay() {
const first = answerFor(rules, { messages: [system, input] });
const replay = answerFor(JSON.parse(JSON.stringify(rules)), { messages: [system, { role: 'assistant', content: [] }, input] });
if (JSON.stringify(first) !== JSON.stringify(replay)) throw new Error('Scripted replay changed the answer');
const dynamic: Rule[] = [{ id: 'land', role: 'worker', match: { text: 'build' }, answer: transcript => ({ tools: [{ name: 'land', args: { id: /Started subagent (\d+)/.exec(JSON.stringify(transcript.messages))?.[1] } }] }) }];
const history = { role: 'toolResult', content: 'Started subagent 42.' };
const selected = answerFor(dynamic, { messages: [system, history, input] });
const restarted = answerFor(dynamic, { messages: [system, { role: 'assistant', content: [] }, history, input] });
if (JSON.stringify(selected) !== JSON.stringify(restarted) || selected.answer.tools?.[0].args.id !== '42') throw new Error('Dynamic tool identity changed on replay');
const toolRules: Rule[] = [{ id: 'worker-done', role: 'worker', match: { text: 'committed' }, answer: { text: 'done' } }];
if (answerFor(toolRules, { messages: [system, { role: 'toolResult', content: [{ type: 'text', text: 'committed' }] }] }).rule !== 'worker-done') throw new Error('Tool result rule did not match');
}