44 lines
3.6 KiB
TypeScript
44 lines
3.6 KiB
TypeScript
/** Stateless role/input rules for deterministic faux-model replay in integration tests. */
|
|
import { fauxProvider, fauxAssistantMessage, fauxToolCall } from '@earendil-works/pi-ai/providers/faux';
|
|
export type Answer = { text?: string; tools?: { name: string; args: any }[] };
|
|
export type Rule = { id: string; role: string; match: { kind?: string; text: string }; answer: Answer | ((transcript: any) => Answer) };
|
|
export const textOf = (content: any): string => typeof content === 'string' ? content : (content ?? []).filter((part: any) => part.type === 'text').map((part: any) => part.text).join('\n');
|
|
export function answerFor(rules: Rule[], transcript: any) {
|
|
const system = JSON.stringify(transcript.messages.filter((message: any) => message.role === 'system'));
|
|
const role = /P3 role:([a-z]+)/.exec(system)?.[1];
|
|
const last = transcript.messages.findLast((message: any) => message.role !== 'system');
|
|
const rule = rules.find(rule => rule.role === role && (!rule.match.kind || rule.match.kind === last?.role) && textOf(last?.content).includes(rule.match.text));
|
|
if (!rule) throw new Error(`No scripted rule for ${role}: ${JSON.stringify(last)}`);
|
|
return { rule: rule.id, answer: typeof rule.answer === 'function' ? rule.answer(transcript) : rule.answer };
|
|
}
|
|
export function scriptedModel(rules: Rule[], report: (rule: string) => void) {
|
|
const faux = fauxProvider({ tokensPerSecond: 100000 });
|
|
const respond = (transcript: any): any => {
|
|
faux.appendResponses([respond]);
|
|
const selected = answerFor(rules, transcript);
|
|
report(selected.rule);
|
|
const calls = selected.answer.tools ?? [];
|
|
return fauxAssistantMessage([
|
|
...(selected.answer.text ? [{ type: 'text' as const, text: selected.answer.text }] : []),
|
|
...calls.map((call, index) => fauxToolCall(call.name, call.args, { id: `${selected.rule}-${index}` })),
|
|
], { stopReason: calls.length ? 'toolUse' : 'stop', timestamp: 0 });
|
|
};
|
|
faux.setResponses([respond]);
|
|
return faux.provider;
|
|
}
|
|
/** Unit test: history length and a provider restart must not change rule selection or the answer. */
|
|
export function testReplay() {
|
|
const rules: Rule[] = [{ id: 'worker-write', role: 'worker', match: { kind: 'user', text: 'build' }, answer: { tools: [{ name: 'write', args: { path: 'feature.txt' } }] } }];
|
|
const system = { role: 'system', content: 'P3 role:worker' }, input = { role: 'user', content: 'build' };
|
|
const first = answerFor(rules, { messages: [system, input] });
|
|
const replay = answerFor(JSON.parse(JSON.stringify(rules)), { messages: [system, { role: 'assistant', content: [] }, input] });
|
|
if (JSON.stringify(first) !== JSON.stringify(replay)) throw new Error('Scripted replay changed the answer');
|
|
const dynamic: Rule[] = [{ id: 'land', role: 'worker', match: { text: 'build' }, answer: transcript => ({ tools: [{ name: 'land', args: { id: /Started subagent (\d+)/.exec(JSON.stringify(transcript.messages))?.[1] } }] }) }];
|
|
const history = { role: 'toolResult', content: 'Started subagent 42.' };
|
|
const selected = answerFor(dynamic, { messages: [system, history, input] });
|
|
const restarted = answerFor(dynamic, { messages: [system, { role: 'assistant', content: [] }, history, input] });
|
|
if (JSON.stringify(selected) !== JSON.stringify(restarted) || selected.answer.tools?.[0].args.id !== '42') throw new Error('Dynamic tool identity changed on replay');
|
|
const toolRules: Rule[] = [{ id: 'worker-done', role: 'worker', match: { text: 'committed' }, answer: { text: 'done' } }];
|
|
if (answerFor(toolRules, { messages: [system, { role: 'toolResult', content: [{ type: 'text', text: 'committed' }] }] }).rule !== 'worker-done') throw new Error('Tool result rule did not match');
|
|
}
|