-
Notifications
You must be signed in to change notification settings - Fork 0
feat(overseer): deterministic per-turn fallback when no summary line (Half B, piece 2) #87
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: feat/overseer-summary-emit
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,142 @@ | ||
| import { describe, expect, it } from 'bun:test' | ||
| import type { Session } from '@hapi/protocol/types' | ||
| import { Store } from '../store' | ||
| import { OverseerEventRecorder, toSessionSnapshot } from './overseerEventRecorder' | ||
|
|
||
| function makeSession(id: string, flavor: string, overrides?: Partial<Session>): Session { | ||
| return { | ||
| id, | ||
| namespace: 'default', | ||
| seq: 0, | ||
| createdAt: Date.now(), | ||
| updatedAt: Date.now(), | ||
| active: true, | ||
| activeAt: Date.now(), | ||
| metadata: { flavor, path: '/tmp', host: 'local' }, | ||
| metadataVersion: 1, | ||
| agentState: null, | ||
| agentStateVersion: 1, | ||
| thinking: false, | ||
| thinkingAt: 0, | ||
| model: null, | ||
| modelReasoningEffort: null, | ||
| effort: null, | ||
| serviceTier: null, | ||
| ...overrides | ||
| } | ||
| } | ||
|
|
||
| function agentText(message: string) { | ||
| return { | ||
| role: 'agent', | ||
| content: { type: 'codex', data: { type: 'message', message } } | ||
| } | ||
| } | ||
|
|
||
| describe('OverseerEventRecorder turn fallback', () => { | ||
| it('synthesizes a session-log-only progress event when no summary line', () => { | ||
| const store = new Store(':memory:') | ||
| const recorder = new OverseerEventRecorder(store.events, store.inbox) | ||
| const session = store.sessions.getOrCreateSession('cur', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') | ||
|
|
||
| const event = recorder.onAgentMessage( | ||
| toSessionSnapshot(makeSession(session.id, 'cursor'), session.tag), | ||
| 'msg-fb', | ||
| agentText('Refactored the parser and added tests.\n\nMore detail here.'), | ||
| Date.now() | ||
| ) | ||
|
|
||
| expect(event).not.toBeNull() | ||
| expect(event?.eventType).toBe('progress') | ||
| expect(event?.attentionCandidate).toBe(0) | ||
| expect(event?.operatorActionRequired).toBe(0) | ||
| expect(event?.summary).toBe('Refactored the parser and added tests.') | ||
| expect(event?.provenance).toContain('hub-synthesized') | ||
|
|
||
| const payload = JSON.parse(event!.payloadJson!) as { synthesized?: boolean } | ||
| expect(payload.synthesized).toBe(true) | ||
|
|
||
| // Session log gets it; the attention inbox stays empty. | ||
| expect(store.events.count()).toBe(1) | ||
| expect(store.inbox.count()).toBe(0) | ||
| }) | ||
|
|
||
| it('does not synthesize when a real AGENT_NOTIFY_SUMMARY is present', () => { | ||
| const store = new Store(':memory:') | ||
| const recorder = new OverseerEventRecorder(store.events, store.inbox) | ||
| const session = store.sessions.getOrCreateSession('cur2', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') | ||
|
|
||
| const event = recorder.onAgentMessage( | ||
| toSessionSnapshot(makeSession(session.id, 'cursor'), session.tag), | ||
| 'msg-real', | ||
| agentText('All done.\nAGENT_NOTIFY_SUMMARY {"version":1,"status":"done","action":"Review PR","summary":"Shipped"}'), | ||
| Date.now() | ||
| ) | ||
|
|
||
| expect(event?.provenance).toBe('AGENT_NOTIFY_SUMMARY') | ||
| expect(store.events.list({ eventType: 'progress' })).toHaveLength(0) | ||
| expect(store.events.count()).toBe(1) | ||
| }) | ||
|
|
||
| it('does not synthesize when the summary line is malformed (validation_error wins)', () => { | ||
| const store = new Store(':memory:') | ||
| const recorder = new OverseerEventRecorder(store.events, store.inbox) | ||
| const session = store.sessions.getOrCreateSession('cur3', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') | ||
|
|
||
| const event = recorder.onAgentMessage( | ||
| toSessionSnapshot(makeSession(session.id, 'cursor'), session.tag), | ||
| 'msg-bad', | ||
| agentText('Working.\nAGENT_NOTIFY_SUMMARY {not valid json'), | ||
| Date.now() | ||
| ) | ||
|
|
||
| expect(event?.eventType).toBe('validation_error') | ||
| expect(store.events.list({ eventType: 'progress' })).toHaveLength(0) | ||
| }) | ||
|
|
||
| it('is idempotent for a redelivered message id', () => { | ||
| const store = new Store(':memory:') | ||
| const recorder = new OverseerEventRecorder(store.events, store.inbox) | ||
| const session = store.sessions.getOrCreateSession('cur4', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') | ||
| const snapshot = toSessionSnapshot(makeSession(session.id, 'cursor'), session.tag) | ||
|
|
||
| recorder.onAgentMessage(snapshot, 'msg-dup', agentText('Progress update.'), Date.now()) | ||
| recorder.onAgentMessage(snapshot, 'msg-dup', agentText('Progress update.'), Date.now()) | ||
|
|
||
| expect(store.events.list({ eventType: 'progress' })).toHaveLength(1) | ||
| }) | ||
|
|
||
| it('caps a long first line with an ellipsis', () => { | ||
| const store = new Store(':memory:') | ||
| const recorder = new OverseerEventRecorder(store.events, store.inbox) | ||
| const session = store.sessions.getOrCreateSession('cur5', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') | ||
|
|
||
| const longLine = 'x'.repeat(500) | ||
| const event = recorder.onAgentMessage( | ||
| toSessionSnapshot(makeSession(session.id, 'cursor'), session.tag), | ||
| 'msg-long', | ||
| agentText(longLine), | ||
| Date.now() | ||
| ) | ||
|
|
||
| expect(event?.summary?.length).toBe(200) | ||
| expect(event?.summary?.endsWith('\u2026')).toBe(true) | ||
| }) | ||
|
|
||
| it('ignores user messages', () => { | ||
| const store = new Store(':memory:') | ||
| const recorder = new OverseerEventRecorder(store.events, store.inbox) | ||
| const session = store.sessions.getOrCreateSession('cur6', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') | ||
|
|
||
| const userContent = { role: 'user', content: { type: 'text', text: 'do the thing' } } | ||
| const event = recorder.onAgentMessage( | ||
| toSessionSnapshot(makeSession(session.id, 'cursor'), session.tag), | ||
| 'msg-user', | ||
| userContent, | ||
| Date.now() | ||
| ) | ||
|
|
||
| expect(event).toBeNull() | ||
| expect(store.events.list({ eventType: 'progress' })).toHaveLength(0) | ||
| }) | ||
| }) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -130,6 +130,20 @@ function extractToolFailureSummary(content: unknown): string | null { | |
| return null | ||
| } | ||
|
|
||
| const TURN_FALLBACK_SUMMARY_MAX = 200 | ||
|
|
||
| /** First non-empty, trimmed line of text, capped for a one-line summary. */ | ||
| function firstNonEmptyLine(text: string): string | null { | ||
| for (const rawLine of text.split('\n')) { | ||
| const line = rawLine.trim() | ||
| if (line.length === 0) continue | ||
| return line.length > TURN_FALLBACK_SUMMARY_MAX | ||
| ? `${line.slice(0, TURN_FALLBACK_SUMMARY_MAX - 1)}\u2026` | ||
| : line | ||
| } | ||
| return null | ||
| } | ||
|
|
||
| function buildTags(notify: NotifySummary | null, flavor: string): string | null { | ||
| const parts: string[] = [] | ||
| if (notify?.agent) parts.push(`agent:${notify.agent}`) | ||
|
|
@@ -220,6 +234,16 @@ export class OverseerEventRecorder { | |
| }) | ||
| } | ||
| } | ||
|
|
||
| // Deterministic backstop: an agent produced visible text but no | ||
| // AGENT_NOTIFY_SUMMARY (rule compliance can never be 100%). Synthesize | ||
| // a minimal, session-log-only capture so the overseer never has a | ||
| // fully blind agent turn. No LLM; attention stays 0 so the inbox is | ||
| // untouched. Marked hub-synthesized so it is never mistaken for a | ||
| // real self-report. | ||
| if (!primary && plainText) { | ||
| primary = this.synthesizeTurnFallback(session, messageId, plainText, ts) | ||
|
Comment on lines
+244
to
+245
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When Cursor applies the known Useful? React with 👍 / 👎. |
||
| } | ||
| } | ||
|
|
||
| // Always scoop URLs from any ingestible message text (agent or user). | ||
|
|
@@ -321,6 +345,42 @@ export class OverseerEventRecorder { | |
| return emitted | ||
| } | ||
|
|
||
| /** | ||
| * Minimal per-turn fallback event when no AGENT_NOTIFY_SUMMARY was emitted. | ||
| * | ||
| * Summary is the first non-empty line of the assistant text (deterministic, | ||
| * no LLM). Status defaults to `progress` via the empty-status mapping, and | ||
| * attention stays 0, so these land in the Session Log only — never the | ||
| * attention inbox. This is the safety net under the Cursor rule overlay: | ||
| * even a dropped summary line yields a captured turn. | ||
| */ | ||
| private synthesizeTurnFallback( | ||
| session: SessionSnapshot, | ||
| messageId: string, | ||
| plainText: string, | ||
| ts: number | ||
| ): StoredSystemEvent | null { | ||
| const summary = firstNonEmptyLine(plainText) | ||
| if (!summary) return null | ||
|
|
||
| const eventType = mapNotifyStatusToEventType(undefined) | ||
| return this.insertSystemEvent(session, { | ||
| ts, | ||
| sourceKind: 'system', | ||
| sourceRef: session.id, | ||
| eventType, | ||
| attentionCandidate: 0, | ||
| operatorActionRequired: 0, | ||
| summary, | ||
| relatedSessionId: session.id, | ||
| provenance: 'hub-synthesized from assistant text (no AGENT_NOTIFY_SUMMARY)', | ||
| idempotencyKey: `session:${session.id}:message:${messageId}:turn_fallback`, | ||
| payloadFields: { messageId, synthesized: true }, | ||
| severity: deriveSeverity(eventType), | ||
| tags: buildTags(null, session.flavor) | ||
| }) | ||
| } | ||
|
|
||
| private recordNotifySummary( | ||
| session: SessionSnapshot, | ||
| messageId: string, | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
When an ACP-backed agent emits text before a tool call,
AcpMessageHandlerflushes that text as its own message before the final turn text (cli/src/agent/backends/acp/AcpMessageHandler.ts:613-620). This branch records a syntheticprogressevent immediately for that pre-tool segment, so even a compliant turn that later ends with a realAGENT_NOTIFY_SUMMARYstill gets extra hub-synthesized progress rows in the session log/overseer memory. Please defer the fallback until a turn boundary or otherwise suppress it for non-final text segments.Useful? React with 👍 / 👎.