Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 110 additions & 0 deletions hub/src/overseer/converse.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -175,4 +175,114 @@ describe('runOverseerConverse', () => {
expect((e as BrainUnavailableError).reachable).toBe(true)
}
})

it('marks ping_session ok:false in the tool trace when relay fails', async () => {
const overseer = {
...fakeOverseer,
pingSession: async () => ({
ok: false,
sessionId: 'sess-1',
sessionName: null,
project: null,
resumed: false,
tombstone: 'Failed to relay: session_not_found',
error: 'session_not_found'
})
} as unknown as OverseerEntity
const fetchMock = vi.fn()
.mockResolvedValueOnce(chatResponse({
role: 'assistant',
content: '',
tool_calls: [{
id: 'c1',
type: 'function',
function: { name: 'ping_session', arguments: '{"sessionId":"sess-1","message":"hi"}' }
}]
}))
.mockResolvedValueOnce(chatResponse({ role: 'assistant', content: 'Could not reach that session.' }))
setFetch(fetchMock)

const { toolTrace } = await runOverseerConverse({
overseer,
config,
messages: [{ role: 'operator', content: 'ping session sess-1: "hi"' }]
})

expect(toolTrace[0]).toMatchObject({
tool: 'ping_session',
ok: false,
error: 'session_not_found'
})
})

it('keeps successful relay in the tool trace when the follow-up brain call fails', async () => {
const overseer = {
...fakeOverseer,
pingSession: async () => ({
ok: true,
sessionId: 'new-id',
sessionName: 'Worker',
project: 'hapi',
resumed: true,
tombstone: 'Relayed to Worker (new-id00) [resumed]: "please continue"'
})
} as unknown as OverseerEntity
const fetchMock = vi.fn()
.mockResolvedValueOnce(chatResponse({
role: 'assistant',
content: '',
tool_calls: [{
id: 'c1',
type: 'function',
function: { name: 'ping_session', arguments: '{"sessionId":"old-id","message":"please continue"}' }
}]
}))
.mockRejectedValueOnce(new Error('ECONNREFUSED'))
setFetch(fetchMock)

const { reply, toolTrace } = await runOverseerConverse({
overseer,
config,
messages: [{ role: 'operator', content: 'ping session old-id: "please continue"' }]
})

expect(toolTrace).toHaveLength(1)
expect(toolTrace[0]).toMatchObject({ tool: 'ping_session', ok: true })
expect(reply).toContain('already succeeded')
expect(reply).toContain('Relayed to Worker')
expect(reply).toContain('Do not retry')
})

it('refuses ping_session when the operator message has no write intent', async () => {
const pingSession = vi.fn()
const overseer = {
...fakeOverseer,
pingSession
} as unknown as OverseerEntity
const fetchMock = vi.fn()
.mockResolvedValueOnce(chatResponse({
role: 'assistant',
content: '',
tool_calls: [{
id: 'c1',
type: 'function',
function: { name: 'ping_session', arguments: '{"sessionId":"sess-1","message":"hi"}' }
}]
}))
.mockResolvedValueOnce(chatResponse({
role: 'assistant',
content: 'I cannot relay without an explicit operator request.'
}))
setFetch(fetchMock)

const { toolTrace } = await runOverseerConverse({
overseer,
config,
messages: [{ role: 'operator', content: 'what needs my attention?' }]
})

expect(pingSession).not.toHaveBeenCalled()
expect(toolTrace[0]).toMatchObject({ tool: 'ping_session', ok: false })
expect(toolTrace[0]?.error).toMatch(/not authorized/i)
})
})
139 changes: 126 additions & 13 deletions hub/src/overseer/converse.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,15 @@
import {
buildOverseerOpenAiTools,
buildOverseerSystemPrompt,
fingerprintWriteToolCall,
isOverseerWriteTool,
isWriteToolAuthorized,
isWriteToolCallAuthorized,
resolveOverseerWriteAuthorization,
type OverseerConverseMessage,
type OverseerToolTraceEntry
type OverseerToolName,
type OverseerToolTraceEntry,
type OverseerWriteAuthorization
} from '@hapi/protocol'
import type { OverseerEntity } from '../sync/overseerEntity'
import { isOverseerToolName, runOverseerTool } from './runOverseerTool'
Expand Down Expand Up @@ -62,25 +69,78 @@ function parseToolArgs(raw: string): Record<string, unknown> {
}
}

/** Write tools return `{ ok: boolean, … }`; propagate that into the converse audit trail. */
export function toolResultOk(result: unknown): boolean {
if (result && typeof result === 'object' && 'ok' in result) {
return (result as { ok: unknown }).ok !== false
}
return true
}

function toolResultError(result: unknown): string | undefined {
if (!result || typeof result !== 'object') return undefined
const record = result as { error?: unknown; tombstone?: unknown }
if (typeof record.error === 'string' && record.error.trim()) return record.error
if (typeof record.tombstone === 'string' && record.tombstone.trim()) return record.tombstone
return 'tool returned ok:false'
}

function writeResultTombstone(result: unknown): string | null {
if (!result || typeof result !== 'object') return null
const tombstone = (result as { tombstone?: unknown }).tombstone
return typeof tombstone === 'string' && tombstone.trim() ? tombstone.trim() : null
}

function fallbackReplyAfterWriteSuccess(confirmations: string[]): string {
if (confirmations.length === 0) {
return 'A follow-up brain call failed after tools ran. Check the tool trace before retrying.'
}
return [
'Write tool(s) already succeeded; the brain failed while composing the confirmation.',
'Do not retry the same write unless you intend a duplicate.',
...confirmations.map((line) => `- ${line}`)
].join('\n')
}

function hasSuccessfulWrite(toolTrace: OverseerToolTraceEntry[]): boolean {
return toolTrace.some((entry) => entry.ok && isOverseerWriteTool(entry.tool as OverseerToolName))
}

export async function runOverseerConverse(params: {
overseer: OverseerEntity
config: BrainConfig
messages: OverseerConverseMessage[]
maxIterations?: number
signal?: AbortSignal
/** Explicit client opt-in for write tools (admin/voice confirm). */
allowWrites?: boolean
}): Promise<{ reply: string; toolTrace: OverseerToolTraceEntry[] }> {
const { overseer, config, messages, maxIterations = 6, signal } = params
const { overseer, config, messages, maxIterations = 6, signal, allowWrites } = params

const tools = buildOverseerOpenAiTools() as OverseerOpenAiToolLike[]
const latestOperatorText = [...messages].reverse().find((m) => m.role === 'operator')?.content ?? ''
const writeAuth: OverseerWriteAuthorization = resolveOverseerWriteAuthorization({
latestOperatorText,
allowWrites
})

const tools = (buildOverseerOpenAiTools() as OverseerOpenAiToolLike[]).filter((tool) => {
const name = tool.function?.name ?? ''
return isWriteToolAuthorized(name, writeAuth)
})
const clockLine = `Server time now: ${new Date().toISOString()} (epoch ms ${Date.now()}, timezone ${Intl.DateTimeFormat().resolvedOptions().timeZone}). Relative snoozes must use absolute snoozedUntil epoch ms from this clock.`
const convo: OpenAiChatMessage[] = [
{ role: 'system', content: `${buildOverseerSystemPrompt()}\n\n${GROUNDING_DIRECTIVE}` },
{ role: 'system', content: `${buildOverseerSystemPrompt()}\n\n${GROUNDING_DIRECTIVE}\n\n# Clock\n\n${clockLine}` },
...messages.map((m): OpenAiChatMessage => ({
role: m.role === 'operator' ? 'user' : 'assistant',
content: m.content
}))
]

const toolTrace: OverseerToolTraceEntry[] = []
/** Tombstones from successful write tools — used if a later brain call fails. */
const writeConfirmations: string[] = []
/** Successful irreversible call fingerprints — reject duplicates in this turn. */
const consumedWriteFingerprints = new Set<string>()
// The brain (llama-server) does not honor tool_choice:'required', so it will
// sometimes answer a fleet question from nothing (e.g. "the inbox is empty"
// when it never called query_inbox). Guardrail: if the very first answer
Expand All @@ -89,7 +149,17 @@ export async function runOverseerConverse(params: {
let nudged = false

for (let iter = 0; iter < maxIterations; iter++) {
const message = await callBrain({ config, messages: convo, tools, signal })
let message: OpenAiChatMessage
try {
message = await callBrain({ config, messages: convo, tools, signal })
} catch (error) {
// Irreversible writes already landed — return their audit trail so the
// route can record the turn and the operator does not duplicate-retry.
if (hasSuccessfulWrite(toolTrace)) {
return { reply: fallbackReplyAfterWriteSuccess(writeConfirmations), toolTrace }
}
throw error
}
const calls = message.tool_calls ?? []

if (calls.length === 0) {
Expand All @@ -112,6 +182,10 @@ export async function runOverseerConverse(params: {
// user/assistant path that all templates render. We also drop the raw
// assistant tool-call message from history for the same reason.
const resultLines: string[] = []
const batchHasRead = calls.some((call) => {
const name = call.function?.name ?? ''
return isOverseerToolName(name) && !isOverseerWriteTool(name)
})
for (const call of calls) {
const name = call.function?.name ?? ''
const argsRaw = call.function?.arguments ?? ''
Expand All @@ -121,11 +195,43 @@ export async function runOverseerConverse(params: {
resultLines.push(`${name || 'unknown'}(${argsRaw}) => ${JSON.stringify({ error: `unknown tool: ${name}` })}`)
continue
}
if (batchHasRead && isOverseerWriteTool(name)) {
const deferred = 'write deferred: resolve identifying read tools first, then call the write in a later turn'
toolTrace.push({ tool: name, args, ok: false, error: deferred })
resultLines.push(`${name}(${argsRaw}) => ${JSON.stringify({ error: deferred })}`)
continue
}
const authz = isWriteToolCallAuthorized(name, args, writeAuth)
if (!authz.ok) {
toolTrace.push({ tool: name, args, ok: false, error: authz.error })
resultLines.push(`${name}(${argsRaw}) => ${JSON.stringify({ error: authz.error })}`)
continue
}
if (isOverseerWriteTool(name)) {
const fp = fingerprintWriteToolCall(name, args)
if (consumedWriteFingerprints.has(fp)) {
const dup = 'duplicate irreversible tool call rejected (already executed this turn)'
toolTrace.push({ tool: name, args, ok: false, error: dup })
resultLines.push(`${name}(${argsRaw}) => ${JSON.stringify({ error: dup })}`)
continue
}
}
try {
// The conversational surface is the operator-directed write-path, so dispositions
// are allowed here (gated off on the raw HTTP tool-dispatch endpoint).
const result = runOverseerTool(overseer, name, args, true)
toolTrace.push({ tool: name, args, ok: true })
const result = await runOverseerTool(overseer, name, args, true)
Comment thread
heavygee marked this conversation as resolved.
Comment thread
heavygee marked this conversation as resolved.
const ok = toolResultOk(result)
toolTrace.push({
tool: name,
args,
ok,
...(ok ? {} : { error: toolResultError(result) })
})
if (ok && isOverseerWriteTool(name)) {
consumedWriteFingerprints.add(fingerprintWriteToolCall(name, args))
const tombstone = writeResultTombstone(result)
writeConfirmations.push(tombstone ?? `${name} succeeded`)
}
// The brain opts into 'full' per call when it needs depth; default lean.
const detail = args.detail === 'full' ? 'full' : 'lean'
const projected = projectToolResultForBrain(name, result, detail)
Expand All @@ -143,10 +249,17 @@ export async function runOverseerConverse(params: {
}

// Iteration cap hit while still calling tools — ask once more for a plain answer.
const finalMsg = await callBrain({
config,
messages: [...convo, { role: 'user', content: 'Answer now in plain text, no more tools.' }],
signal
})
return { reply: (finalMsg.content ?? '').trim() || 'I gathered the data but could not compose an answer.', toolTrace }
try {
const finalMsg = await callBrain({
config,
messages: [...convo, { role: 'user', content: 'Answer now in plain text, no more tools.' }],
signal
})
return { reply: (finalMsg.content ?? '').trim() || 'I gathered the data but could not compose an answer.', toolTrace }
} catch (error) {
if (hasSuccessfulWrite(toolTrace)) {
return { reply: fallbackReplyAfterWriteSuccess(writeConfirmations), toolTrace }
}
throw error
}
}
13 changes: 8 additions & 5 deletions hub/src/overseer/runOverseerTool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import {
} from '@hapi/protocol'
import type { OverseerEntity } from '../sync/overseerEntity'

/** Thrown when a write tool (`record_disposition`) is dispatched on a read-only surface (R2 gate). */
/** Thrown when a write tool is dispatched on a read-only surface (R2 gate). */
export class OverseerWriteNotAllowedError extends Error {
constructor(tool: string) {
super(`Tool "${tool}" writes and is not allowed on this surface`)
Expand All @@ -17,15 +17,16 @@ export class OverseerWriteNotAllowedError extends Error {
/**
* Execute one Overseer tool by name against the entity. Shared by the HTTP tool-dispatch route and
* the converse tool-calling loop so both go through exactly one place. Throws `ZodError` on invalid
* args. Every tool is read-only EXCEPT `record_disposition`; writes are gated behind `allowWrites`
* (the conversational path sets it; the raw HTTP dispatch does not).
* args. Write tools (`record_disposition`, `ping_session`) are gated behind `allowWrites`
* (the conversational path sets it; the raw HTTP dispatch does not). Async because `ping_session`
* may resume a worker before enqueueing.
*/
export function runOverseerTool(
export async function runOverseerTool(
overseer: OverseerEntity,
tool: OverseerToolName,
args: unknown,
allowWrites = false
): unknown {
): Promise<unknown> {
if (isOverseerWriteTool(tool) && !allowWrites) {
throw new OverseerWriteNotAllowedError(tool)
}
Expand Down Expand Up @@ -58,6 +59,8 @@ export function runOverseerTool(
return overseer.queryDispositions(overseerToolArgsSchemas.query_dispositions.parse(args))
case 'record_disposition':
return overseer.recordDisposition(overseerToolArgsSchemas.record_disposition.parse(args))
case 'ping_session':
return overseer.pingSession(overseerToolArgsSchemas.ping_session.parse(args))
Comment thread
heavygee marked this conversation as resolved.
Comment thread
heavygee marked this conversation as resolved.
default: {
const exhaustive: never = tool
throw new Error(`Unknown overseer tool: ${String(exhaustive)}`)
Expand Down
15 changes: 12 additions & 3 deletions hub/src/overseer/toolProjection.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,13 @@ describe('projectToolResultForBrain', () => {
expect(lean.counts).toEqual({ candidates: 1, surfaced: 1, held: 0 })
})

it('thins inbox items to id/what/status/priority and drops the fat', () => {
it('thins inbox items to id/what/summary/category/session/status/priority and drops the fat', () => {
const full = {
items: [
{
id: 7, title: 'CI auth blocking 3 workers', status: 'surfaced', priority: 90,
category: 'blocker', summary: 'long summary…', reasonForPriority: 'shared root cause',
relatedSessionId: '96f67085-5dd3-4a10-aa7c-785f72a227c2',
sourceEventIds: [1, 2, 3], artifactRefs: ['a'.repeat(400)], createdAt: 1, updatedAt: 2
},
{ id: 8, title: 'needs a decision', status: 'new', priority: 40, artifactRefs: ['x'.repeat(400)] }
Expand All @@ -28,8 +29,16 @@ describe('projectToolResultForBrain', () => {
const lean = projectToolResultForBrain('query_inbox', full) as { total: number; items: unknown[] }
expect(lean.total).toBe(2)
expect(lean.items).toEqual([
{ id: 7, what: 'CI auth blocking 3 workers', status: 'surfaced', priority: 90 },
{ id: 8, what: 'needs a decision', status: 'new', priority: 40 }
{
id: 7,
what: 'CI auth blocking 3 workers',
summary: 'long summary…',
category: 'blocker',
session: '96f67085-5dd3-4a10-aa7c-785f72a227c2',
status: 'surfaced',
priority: 90
},
{ id: 8, what: 'needs a decision', summary: undefined, category: undefined, session: undefined, status: 'new', priority: 40 }
])
// the fat is gone
expect(JSON.stringify(lean)).not.toContain('artifactRefs')
Expand Down
Loading
Loading