surma
0dc97c7682
Rewrite Ultimate roles for the WASI world and explicit repositories. Add approved proxy model families, ordered overlays, and executable parser and asset checks.
149 lines
12 KiB
TypeScript
149 lines
12 KiB
TypeScript
/** Node-only test runner. The implementation imports no Node APIs. Run from the repository root. */
|
|
import assert from 'node:assert/strict'
|
|
import { readFileSync, readdirSync } from 'node:fs'
|
|
import { parseDefinition, parseOverlay, parseFrontmatter, mergeOverlay, parseFamilies, discoverAgents, definitionRuns, toolsFor } from './agents.ts'
|
|
let count = 0
|
|
function test(name: string, run: () => void) { run(); console.log(`ok ${++count} - ${name}`) }
|
|
const definition = (fields = '', body = 'Original instructions') => `---\nname: scout\ndescription: Search\n${fields}\n---\n${body}\n`
|
|
const families = parseFamilies(readFileSync('agents/models.json', 'utf8'))
|
|
const models = [...new Set(Object.values(families).map(family => family.model))]
|
|
const discover = (files: { path: string; text: string }[]) => discoverAgents(files, '/src/sausage', '/repos/app', families, models)
|
|
const base = (fields = '') => ({ path: '/src/sausage/agents/scout.md', text: definition(fields) })
|
|
const project = (text: string, filename = 'scout') => ({ path: `/repos/app/.sausage/agents/${filename}.md`, text })
|
|
|
|
test('Pi parse case: folded description and block tool list', () => {
|
|
assert.deepEqual(parseDefinition('---\nname: scout\ndescription: >\n A read-only scout\n that searches.\nmodel: openai/gpt-6.1-sol\nthinking: max\ntools:\n - read\n - bash\n---\nSearch\n', 'scout.md'), {
|
|
name: 'scout', description: 'A read-only scout that searches.', path: 'scout.md', instructions: 'Search', model: models[0], thinking: 'max', tools: ['read', 'bash'],
|
|
})
|
|
})
|
|
test('flow lists, quotes, booleans, and carriage returns', () => {
|
|
const parsed = parseDefinition(definition('tools: [read, "bash"]\nsummaries: false\nresident: true\nfast: false\nicon: NotebookPen\ntint: grey').replaceAll('\n', '\r\n'), 'test')
|
|
assert.deepEqual(parsed.tools, ['read', 'bash']); assert.equal(parsed.summaries, false)
|
|
assert.equal(parsed.resident, true); assert.equal(parsed.fast, false)
|
|
assert.equal(parseFrontmatter("---\nname: 'it''s'\n---\nBody").frontmatter.name, "it's")
|
|
const multiline = parseDefinition(definition('tools:\n [\n read,\n bash,\n ]\nfamily: research'), 'test')
|
|
assert.deepEqual(multiline.tools, ['read', 'bash']); assert.equal(multiline.family, 'research')
|
|
assert.deepEqual(parseDefinition(definition('tools: [read, "unknown,tool"]'), 'test').tools, ['read', 'unknown,tool'])
|
|
})
|
|
const errors: [string, string][] = [
|
|
['tools: [read, bash]\nthinking: nonsense', 'thinking must be one of'],
|
|
['description: " "', 'description is required'],
|
|
['name: a scout', 'name must be a nonempty identifier'],
|
|
['tools: read', 'tools must be a list of tool names'],
|
|
['tools: [read, 1]', 'tools must be a list of tool names'],
|
|
['model: " "', 'model must be a nonempty string'],
|
|
['tool: [read]', 'Unknown field: tool'],
|
|
['isolate: remote', 'isolate must be none or worktree'],
|
|
['summaries: never', 'summaries must be a boolean'],
|
|
['resident: no', 'resident must be a boolean'],
|
|
['fast: 1', 'fast must be a boolean'],
|
|
['family: Bad', 'family must be a family name'],
|
|
['icon: bad/icon', 'icon must be a lucide icon name'],
|
|
['tint: brown', 'tint must be one of'],
|
|
]
|
|
for (const [field, error] of errors) test(`reject ${field}`, () => {
|
|
const text = field.startsWith('description:') ? definition().replace('description: Search', field) : field.startsWith('name:') ? definition().replace('name: scout', field) : definition(field)
|
|
assert.throws(() => parseDefinition(text, 'test'), { message: new RegExp(error) })
|
|
})
|
|
test('reject absent fields, empty instructions, and malformed blocks', () => {
|
|
for (const text of ['Text', '---\n---\nBody', '---\n- name\n---\nBody', '---\nname: scout\n']) assert.throws(() => parseDefinition(text, 'test'), /Expected frontmatter/)
|
|
assert.throws(() => parseDefinition(definition().replace('description: Search\n', ''), 'test'), /description is required/)
|
|
assert.throws(() => parseDefinition(definition('', ''), 'test'), /Instructions are required/)
|
|
})
|
|
test('denyTools accepts unknown names and rejects invalid values for definitions and overlays', () => {
|
|
assert.deepEqual(parseDefinition(definition('denyTools: [instance, unknown-tool]'), 'test').denyTools, ['instance', 'unknown-tool'])
|
|
assert.deepEqual(parseDefinition(definition('denyTools: []'), 'test').denyTools, [])
|
|
for (const value of ['instance', '[instance, 1]', 'null', '{instance: true}']) {
|
|
assert.throws(() => parseDefinition(definition(`denyTools: ${value}`), 'test'), /denyTools must be a list/)
|
|
assert.throws(() => parseOverlay(definition(`addTools: []\ndenyTools: ${value}`), 'test'), /denyTools must be a list/)
|
|
}
|
|
})
|
|
test('ordered overlays add tools and append bodies without changing source', () => {
|
|
const found = discover([project('---\nname: scout\naddTools: [write, read]\n---\nFirst', '01'), base('tools: [read, bash]'), project('---\nname: scout\naddTools: [message]\n---\nSecond', '02')])
|
|
assert.deepEqual(found.errors, []); assert.deepEqual(found.replaced, [])
|
|
const agent = found.definitions[0]
|
|
assert.equal(agent.source, 'builtin'); assert.equal(agent.path, base().path)
|
|
assert.deepEqual(agent.tools, ['read', 'bash', 'write', 'message'])
|
|
assert.equal(agent.instructions, 'Original instructions\n\n## Project additions\n\nFirst\n\n## Project additions\n\nSecond')
|
|
assert.deepEqual(agent.overlays, [project('', '01').path, project('', '02').path])
|
|
})
|
|
test('overlay settings replace while denials accumulate through empty overlay', () => {
|
|
const found = discover([base('tools: [read]\ndenyTools: [unknown-tool]\nthinking: low'), project('---\nname: scout\naddTools: []\ndenyTools: [unknown-tool, bash]\nmodel: anthropic/claude-opus-5-5\nthinking: high\nfast: true\ndescription: Our scout\n---', '01'), project('---\nname: scout\naddTools: []\ndenyTools: []\n---', '02')])
|
|
assert.deepEqual(found.errors, [])
|
|
assert.deepEqual(found.definitions[0].denyTools, ['unknown-tool', 'bash'])
|
|
assert.equal(found.definitions[0].thinking, 'high'); assert.equal(found.definitions[0].fast, true)
|
|
assert.equal(found.definitions[0].description, 'Our scout')
|
|
assert.deepEqual(found.definitions[0].tools, ['read'])
|
|
})
|
|
test('inherited tools retain deferred additions and do not mutate parent', () => {
|
|
const parent = ['read', 'bash', 'write', 'subagent']
|
|
const child = mergeOverlay(parseDefinition(definition('denyTools: [write, subagent]'), 'test'), parseOverlay('---\nname: scout\naddTools: [message]\n---', 'overlay'))
|
|
assert.equal(child.tools, undefined); assert.deepEqual(child.addTools, ['message'])
|
|
assert.deepEqual(toolsFor(child, parent), ['read', 'bash', 'message'])
|
|
assert.deepEqual(toolsFor(parseDefinition(definition('tools: [read, write]\ndenyTools: [write]'), 'test'), parent), ['read'])
|
|
assert.deepEqual(parent, ['read', 'bash', 'write', 'subagent'])
|
|
})
|
|
test('overlay errors do not replace a valid base', () => {
|
|
const found = discover([base(), project('---\nname: scout\naddTools: [x]\ntools: [read]\n---', 'listed'), project('---\nname: scout\naddTools: []\ncolour: red\n---', 'unknown'), project('---\nname: nobody\naddTools: [x]\n---', 'orphan')])
|
|
assert.deepEqual(Object.fromEntries(found.errors.map(error => [error.name, error.error])), {
|
|
listed: 'Error: An overlay adds tools with addTools, not tools', unknown: 'Error: Unknown field: colour', orphan: 'Error: addTools needs an earlier definition named nobody',
|
|
})
|
|
assert.equal(found.definitions[0].instructions, 'Original instructions')
|
|
assert.throws(() => parseOverlay('---\nname: scout\naddTools: read\n---', 'test'), /addTools must be a list/)
|
|
})
|
|
test('replacement uses name, not filename, and respects exact roots', () => {
|
|
const found = discover([base(), project(definition('family: research', 'Replacement'), 'different'), { path: '/repos/other/.sausage/agents/scout.md', text: definition() }, { path: '/src/sausage/agents/nested/no.md', text: definition() }])
|
|
assert.deepEqual(found.errors, []); assert.equal(found.definitions.length, 1)
|
|
assert.equal(found.definitions[0].source, 'project'); assert.equal(found.definitions[0].instructions, 'Replacement')
|
|
assert.deepEqual(found.replaced, [{ name: 'scout', path: project('', 'different').path, source: 'project' }])
|
|
})
|
|
test('family fallback, thinking override, model suffix, and unknown models', () => {
|
|
const run = (fields: string) => definitionRuns(parseDefinition(definition(fields), 'test'), families, models)
|
|
assert.equal(run('family: missing').family, 'default')
|
|
assert.equal(run('family: constructor').family, 'default')
|
|
assert.equal(run('family: research').thinking, 'high')
|
|
assert.equal(run('family: research\nthinking: low').thinking, 'low')
|
|
assert.equal(run('model: openai/gpt-6.1-sol').thinking, undefined)
|
|
assert.equal(run('model: openai/gpt-6.1-sol:high').thinking, 'high')
|
|
assert.equal(run('model: openai/gpt-6.1-sol:high\nthinking: low').thinking, 'low')
|
|
for (const model of ['anthropic/unknown', 'unknown']) assert.match(discover([base(`model: ${model}`)]).errors[0].error, /model: Model .* not found/)
|
|
const found = discover([base('model: openai/gpt-6.1-sol'), project('---\nname: scout\nfamily: lead\naddTools: []\n---')])
|
|
assert.equal(found.definitions[0].model, undefined); assert.equal(found.definitions[0].runs.family, 'lead')
|
|
const suffix = discover([base('thinking: low'), project('---\nname: scout\nmodel: openai/gpt-6.1-sol:high\naddTools: []\n---')])
|
|
assert.equal(suffix.definitions[0].runs.thinking, 'high')
|
|
})
|
|
test('family world file requires valid refs and default', () => {
|
|
for (const text of ['null', '[]', '{}', '{"default":{"model":"bad"}}', '{"Default":{"model":"openai/a"}}', '{"default":{"model":"openai/a","thinking":"bad"}}']) assert.throws(() => parseFamilies(text))
|
|
assert.deepEqual(Object.keys(families), ['default', 'fast', 'review', 'research', 'design', 'lead'])
|
|
assert.equal(families.review.thinking, 'xhigh')
|
|
assert.equal(families.design.thinking, 'high')
|
|
})
|
|
test('all shipped definitions and overlays parse and discover', () => {
|
|
const files = readdirSync('agents').filter(name => name.endsWith('.md')).map(name => ({ path: `/src/sausage/agents/${name}`, text: readFileSync(`agents/${name}`, 'utf8') }))
|
|
assert.equal(files.length, 12)
|
|
const available = ['read', 'bash', 'edit', 'write', 'message', 'note', 'notify', 'automation', 'branch', 'compact', 'remember', 'memory', 'todo', 'goal', 'subagent', 'plan', 'workspace']
|
|
for (const file of files) {
|
|
const agent = 'addTools' in parseFrontmatter(file.text).frontmatter ? parseOverlay(file.text, file.path) : parseDefinition(file.text, file.path)
|
|
assert.ok(!/\b(pnpm|vitest|clone|instance|restart|web_search|mcp|setup|codemode|debug)\b|~\/\.ultimate/.test(agent.instructions), file.path)
|
|
const fields = 'settings' in agent ? agent.settings : agent
|
|
for (const tool of [...(fields.tools ?? []), ...(fields.denyTools ?? []), ...('addTools' in agent ? agent.addTools ?? [] : [])]) assert.ok(available.includes(tool), `${file.path}: ${tool}`)
|
|
}
|
|
const found = discover(files)
|
|
assert.deepEqual(found.errors, []); assert.equal(found.definitions.length, 11)
|
|
assert.equal(found.definitions.find(agent => agent.name === 'coordinator')!.runs.family, 'lead')
|
|
for (const agent of found.definitions.filter(agent => agent.name.startsWith('reviewer'))) assert.equal(agent.runs.thinking, 'xhigh')
|
|
})
|
|
test('all applicable skills parse and avoid unavailable tools', () => {
|
|
let skills = 0
|
|
for (const name of readdirSync('skills')) {
|
|
if (name === 'README.md') continue
|
|
const parsed = parseFrontmatter(readFileSync(`skills/${name}/SKILL.md`, 'utf8'))
|
|
assert.equal(parsed.frontmatter.name, name)
|
|
assert.equal(typeof parsed.frontmatter.description, 'string'); assert.ok(parsed.body)
|
|
assert.ok(!/\b(pnpm|vitest|clone|instance|restart|web_search|mcp|setup|codemode|debug)\b|~\/\.ultimate/.test(parsed.body), name)
|
|
skills++
|
|
}
|
|
assert.equal(skills, 4)
|
|
})
|
|
console.log(`${count} checks passed`)
|