diff --git a/android/app/src/main/kotlin/app/hapi/companion/fcm/NotificationChannels.kt b/android/app/src/main/kotlin/app/hapi/companion/fcm/NotificationChannels.kt index ea971bdc10..0930296461 100644 --- a/android/app/src/main/kotlin/app/hapi/companion/fcm/NotificationChannels.kt +++ b/android/app/src/main/kotlin/app/hapi/companion/fcm/NotificationChannels.kt @@ -13,6 +13,9 @@ import app.hapi.data.push.PushPayload * * - `permission_requests` — HIGH: an agent is blocked on the operator; the * heads-up + sound interruption is the point. + * - `model_errors` — HIGH: the agent hit a model-side failure (rate limit, + * quota, crash); same urgency as a blocked permission, event-tagged so + * distinct errors do not overwrite each other. * - `ready` — DEFAULT: the agent finished and is waiting for input. * - `task_notifications` — DEFAULT: task completed/failed; also the bucket * for unknown types / contract versions (never heads-up for those). @@ -34,6 +37,13 @@ object NotificationChannels { ).apply { description = context.getString(R.string.channel_permission_requests_desc) }, + NotificationChannel( + PushPayload.CHANNEL_MODEL_ERROR, + context.getString(R.string.channel_model_errors), + NotificationManager.IMPORTANCE_HIGH, + ).apply { + description = context.getString(R.string.channel_model_errors_desc) + }, NotificationChannel( PushPayload.CHANNEL_READY, context.getString(R.string.channel_ready), diff --git a/android/app/src/main/kotlin/app/hapi/companion/fcm/PushNotifications.kt b/android/app/src/main/kotlin/app/hapi/companion/fcm/PushNotifications.kt index 853e1c9250..f3b6ba2833 100644 --- a/android/app/src/main/kotlin/app/hapi/companion/fcm/PushNotifications.kt +++ b/android/app/src/main/kotlin/app/hapi/companion/fcm/PushNotifications.kt @@ -64,7 +64,7 @@ object PushNotifications { when (payload.type) { PushType.PERMISSION_REQUEST -> addPermissionActions(context, builder, payload) PushType.READY, PushType.TASK_NOTIFICATION -> addReplyActions(context, builder, payload) - null -> Unit + PushType.MODEL_ERROR, null -> Unit } } diff --git a/android/app/src/main/res/values-zh-rCN/strings.xml b/android/app/src/main/res/values-zh-rCN/strings.xml index 926edf7b76..5bd072d580 100644 --- a/android/app/src/main/res/values-zh-rCN/strings.xml +++ b/android/app/src/main/res/values-zh-rCN/strings.xml @@ -172,6 +172,8 @@ 权限请求 智能体正在等待您允许或拒绝一次工具调用 + 模型错误 + 智能体遇到需要处理的模型侧失败 代理就绪 智能体已完成并正在等待输入 任务通知 diff --git a/android/app/src/main/res/values/strings.xml b/android/app/src/main/res/values/strings.xml index 8000898697..48c6fd03be 100644 --- a/android/app/src/main/res/values/strings.xml +++ b/android/app/src/main/res/values/strings.xml @@ -165,6 +165,8 @@ Permission requests An agent is waiting for you to allow or deny a tool call + Model errors + An agent hit a model-side failure that needs attention Agent ready An agent finished and is waiting for input Task notifications diff --git a/android/core/data/src/main/kotlin/app/hapi/data/push/PushPayload.kt b/android/core/data/src/main/kotlin/app/hapi/data/push/PushPayload.kt index 7cc54b93b5..eaaa2e506f 100644 --- a/android/core/data/src/main/kotlin/app/hapi/data/push/PushPayload.kt +++ b/android/core/data/src/main/kotlin/app/hapi/data/push/PushPayload.kt @@ -12,7 +12,8 @@ import kotlinx.serialization.Serializable enum class PushType(val wire: String) { READY("ready"), PERMISSION_REQUEST("permission-request"), - TASK_NOTIFICATION("task-notification"); + TASK_NOTIFICATION("task-notification"), + MODEL_ERROR("model-error"); companion object { fun fromWire(value: String): PushType? = entries.firstOrNull { it.wire == value } @@ -75,6 +76,8 @@ data class PushPayload( val severity: PushSeverity?, val contractVersion: String?, val notifySummary: PushNotifySummary?, + /** Hub coalescing tag (`FcmSendPayload.tag`), when present on the data map. */ + val tag: String?, ) { /** @@ -95,30 +98,31 @@ data class PushPayload( get() = isKnownContractVersion && when (type) { PushType.PERMISSION_REQUEST -> requestId != null PushType.READY, PushType.TASK_NOTIFICATION -> true - null -> false + PushType.MODEL_ERROR, null -> false } /** * Notification channel routing: `permission_requests` (HIGH) / - * `ready` (DEFAULT) / `task_notifications` (DEFAULT). Unknown types and - * unknown contract versions land in the default-importance - * `task_notifications` bucket — never in the heads-up channel. + * `model_errors` (HIGH) / `ready` (DEFAULT) / `task_notifications` + * (DEFAULT). Unknown types and unknown contract versions land in the + * default-importance `task_notifications` bucket — never heads-up. */ val channelId: String get() = when { !isKnownContractVersion -> CHANNEL_TASK_NOTIFICATIONS type == PushType.PERMISSION_REQUEST -> CHANNEL_PERMISSION_REQUESTS + type == PushType.MODEL_ERROR -> CHANNEL_MODEL_ERROR type == PushType.READY -> CHANNEL_READY else -> CHANNEL_TASK_NOTIFICATIONS } /** - * Coalescing tag `type-`: a newer push of the same type for - * the same session replaces the previous notification instead of - * stacking (mirrors the hub-side `tag` scheme it uses for Web Push). + * Coalescing tag: prefer the hub-supplied `data.tag` (event-specific + * for `model-error`) before falling back to `type-`. */ val notificationTag: String - get() = "${rawType.ifBlank { "unknown" }}-$sessionId" + get() = tag?.takeIf { it.isNotBlank() } + ?: "${rawType.ifBlank { "unknown" }}-$sessionId" /** Title to render; falls back to the session name, then a constant. */ val displayTitle: String @@ -152,6 +156,7 @@ data class PushPayload( const val CHANNEL_PERMISSION_REQUESTS = "permission_requests" const val CHANNEL_READY = "ready" const val CHANNEL_TASK_NOTIFICATIONS = "task_notifications" + const val CHANNEL_MODEL_ERROR = "model_errors" private const val DEFAULT_TITLE = "HAPI" @@ -175,6 +180,7 @@ data class PushPayload( severity = PushSeverity.fromWire(data["severity"]), contractVersion = data["contractVersion"]?.takeIf { it.isNotBlank() }, notifySummary = data["notifySummary"]?.let(::parseNotifySummary), + tag = data["tag"]?.takeIf { it.isNotBlank() }, ) } diff --git a/android/core/data/src/test/kotlin/app/hapi/data/push/PushPayloadTest.kt b/android/core/data/src/test/kotlin/app/hapi/data/push/PushPayloadTest.kt index 37dcdca2fc..e2ee50f6d4 100644 --- a/android/core/data/src/test/kotlin/app/hapi/data/push/PushPayloadTest.kt +++ b/android/core/data/src/test/kotlin/app/hapi/data/push/PushPayloadTest.kt @@ -82,6 +82,16 @@ class PushPayloadTest { PushPayload.CHANNEL_TASK_NOTIFICATIONS, PushPayload.parse(permissionData("type" to "task-notification"))!!.channelId, ) + assertEquals( + PushPayload.CHANNEL_MODEL_ERROR, + PushPayload.parse( + permissionData( + "type" to "model-error", + "severity" to "error", + "tag" to "model-error-11111111-2222-3333-4444-555555555555-evt-1", + ) + )!!.channelId, + ) } @Test @@ -127,6 +137,51 @@ class PushPayloadTest { ) } + @Test + fun `model-error is a known type and prefers the hub event tag`() { + val eventTag = "model-error-11111111-2222-3333-4444-555555555555-evt-1710000000000" + val payload = PushPayload.parse( + permissionData( + "type" to "model-error", + "severity" to "error", + "title" to "Rate limited", + "body" to "status 429", + "tag" to eventTag, + ).minus("requestId"), + )!! + + assertEquals(PushType.MODEL_ERROR, payload.type) + assertEquals("model-error", payload.rawType) + assertEquals(PushPayload.CHANNEL_MODEL_ERROR, payload.channelId) + assertEquals(eventTag, payload.notificationTag) + assertEquals(PushSeverity.ERROR, payload.severity) + assertFalse(payload.supportsActions) + } + + @Test + fun `model-error without a hub tag falls back to type-sessionId`() { + val payload = PushPayload.parse( + permissionData("type" to "model-error").minus("requestId"), + )!! + assertEquals( + "model-error-11111111-2222-3333-4444-555555555555", + payload.notificationTag, + ) + } + + @Test + fun `distinct model-error tags do not collapse`() { + val session = "11111111-2222-3333-4444-555555555555" + val first = PushPayload.parse( + permissionData("type" to "model-error", "tag" to "model-error-$session-evt-1"), + )!!.notificationTag + val second = PushPayload.parse( + permissionData("type" to "model-error", "tag" to "model-error-$session-evt-2"), + )!!.notificationTag + assertEquals("model-error-$session-evt-1", first) + assertEquals("model-error-$session-evt-2", second) + } + // ---------------------------------------------------------- ready bodies -- @Test diff --git a/cli/src/agent/backends/acp/AcpSdkBackend.test.ts b/cli/src/agent/backends/acp/AcpSdkBackend.test.ts index 4d9036018b..d20fe65ff8 100644 --- a/cli/src/agent/backends/acp/AcpSdkBackend.test.ts +++ b/cli/src/agent/backends/acp/AcpSdkBackend.test.ts @@ -735,6 +735,39 @@ describe('AcpSdkBackend', () => { expect(turnCompleteIdx).toBeGreaterThan(lateIdx); }); + it('skips session/prompt when shouldSend is false after pre-prompt drain', async () => { + backendStatics.PRE_PROMPT_UPDATE_QUIET_PERIOD_MS = 1; + backendStatics.PRE_PROMPT_UPDATE_DRAIN_TIMEOUT_MS = 50; + backendStatics.UPDATE_QUIET_PERIOD_MS = 1; + backendStatics.UPDATE_DRAIN_TIMEOUT_MS = 50; + backendStatics.LATE_FLUSH_WINDOW_MS = 1; + + const backend = new AcpSdkBackend({ command: 'opencode' }); + const backendInternal = backend as unknown as { + transport: { + sendRequest: (...args: unknown[]) => Promise; + close: () => Promise; + } | null; + }; + let sendCalls = 0; + backendInternal.transport = { + sendRequest: async () => { + sendCalls += 1; + return { stopReason: 'end_turn' }; + }, + close: async () => {} + }; + + const sent = await backend.prompt( + 'session-1', + [{ type: 'text', text: 'hi' }], + () => {}, + { shouldSend: () => false } + ); + expect(sent).toBe(false); + expect(sendCalls).toBe(0); + }); + it('attributes pre-prompt straggler chunks to the previous turn\'s onUpdate', async () => { backendStatics.UPDATE_QUIET_PERIOD_MS = 25; backendStatics.UPDATE_DRAIN_TIMEOUT_MS = 200; diff --git a/cli/src/agent/backends/acp/AcpSdkBackend.ts b/cli/src/agent/backends/acp/AcpSdkBackend.ts index dca431371f..df01425137 100644 --- a/cli/src/agent/backends/acp/AcpSdkBackend.ts +++ b/cli/src/agent/backends/acp/AcpSdkBackend.ts @@ -542,8 +542,9 @@ export class AcpSdkBackend implements AgentBackend { async prompt( sessionId: string, content: PromptContent[], - onUpdate: (msg: AgentMessage) => void - ): Promise { + onUpdate: (msg: AgentMessage) => void, + options?: { shouldSend?: () => boolean } + ): Promise { if (!this.transport) { throw new Error('ACP transport not initialized'); } @@ -561,6 +562,9 @@ export class AcpSdkBackend implements AgentBackend { ); await this.sessionUpdateQueue; this.messageHandler?.drainBuffers(); + if (options?.shouldSend && !options.shouldSend()) { + return false; + } this.messageHandler = new AcpMessageHandler(onUpdate, { textChunkMode: this.options.textChunkMode, flavor: this.options.flavor, @@ -655,6 +659,7 @@ export class AcpSdkBackend implements AgentBackend { } } } + return true; } async cancelPrompt(sessionId: string): Promise { diff --git a/cli/src/agent/types.ts b/cli/src/agent/types.ts index 6a39494665..fafc1f258b 100644 --- a/cli/src/agent/types.ts +++ b/cli/src/agent/types.ts @@ -105,7 +105,12 @@ export interface AgentBackend { setConfigOption?(sessionId: string, configId: string, value: string): Promise; getSessionModelsMetadata?(sessionId: string): AgentSessionModelsMetadata | undefined; getThoughtLevelConfigOption?(sessionId: string): AgentSessionConfigOptionDescriptor | undefined; - prompt(sessionId: string, content: PromptContent[], onUpdate: (msg: AgentMessage) => void): Promise; + prompt( + sessionId: string, + content: PromptContent[], + onUpdate: (msg: AgentMessage) => void, + options?: { shouldSend?: () => boolean } + ): Promise; cancelPrompt(sessionId: string): Promise; respondToPermission(sessionId: string, request: PermissionRequest, response: PermissionResponse): Promise; onPermissionRequest(handler: (request: PermissionRequest) => void): void; diff --git a/cli/src/api/api.ts b/cli/src/api/api.ts index 5b3acf23b8..9f5d3ef6c9 100644 --- a/cli/src/api/api.ts +++ b/cli/src/api/api.ts @@ -1,6 +1,7 @@ import axios from 'axios' import type { AgentState, ClearOpencodeSessionCallbackRequest, ClearOpencodeSessionResponse, CreateMachineResponse, CreateSessionResponse, RunnerState, Machine, MachineMetadata, Metadata, Session } from '@/api/types' import { applyHubSessionSummaryContract } from '@/modules/common/sessionSummaryInstruction' +import { setAutoBridgeTransientModelErrors } from '@/cursor/cursorModelErrorBridgePrefs' import type { LocalResumeTarget, ResumableSession } from '@hapi/protocol' import { AgentStateSchema, @@ -88,6 +89,9 @@ export class ApiClient { if (typeof parsed.data.sessionSummaryContract === 'boolean') { applyHubSessionSummaryContract(parsed.data.sessionSummaryContract) } + if (typeof parsed.data.autoBridgeTransientModelErrors === 'boolean') { + setAutoBridgeTransientModelErrors(parsed.data.autoBridgeTransientModelErrors) + } const raw = parsed.data.session @@ -144,6 +148,9 @@ export class ApiClient { if (typeof parsed.data.sessionSummaryContract === 'boolean') { applyHubSessionSummaryContract(parsed.data.sessionSummaryContract) } + if (typeof parsed.data.autoBridgeTransientModelErrors === 'boolean') { + setAutoBridgeTransientModelErrors(parsed.data.autoBridgeTransientModelErrors) + } const raw = parsed.data.session const metadata = (() => { diff --git a/cli/src/api/apiSession.ts b/cli/src/api/apiSession.ts index 64766f172a..c6bc537b0a 100644 --- a/cli/src/api/apiSession.ts +++ b/cli/src/api/apiSession.ts @@ -1088,6 +1088,17 @@ export class ApiSessionClient extends EventEmitter { summary: string tokensBefore?: number estimatedTokensAfter?: number + } | { + type: 'modelError' + kind: string + transient: boolean + rawSnippet: string + priorAssistantClaimsDone: boolean + } | { + type: 'modelErrorBridged' + kind: string + auto: boolean + eventId: string }, id?: string): void { const content = { role: 'agent', diff --git a/cli/src/cursor/cursorAcpRemoteLauncher.test.ts b/cli/src/cursor/cursorAcpRemoteLauncher.test.ts index 010b4bd04a..d1233a6b90 100644 --- a/cli/src/cursor/cursorAcpRemoteLauncher.test.ts +++ b/cli/src/cursor/cursorAcpRemoteLauncher.test.ts @@ -16,7 +16,6 @@ const harness = vi.hoisted(() => ({ newSessionAttempts: 0, promptCalls: 0, prompts: [] as unknown[][], - deferPrompt: null as Promise | null, deferSoftSteer: null as Promise | null, softSteerDispatchError: null as Error | null, deferSoftSteerDispatch: null as Promise | null, @@ -24,14 +23,41 @@ const harness = vi.hoisted(() => ({ promptMessages: [] as AgentMessage[], promptMessageBatches: [] as AgentMessage[][], promptStderrErrors: [] as Array<{ type: string; message: string; raw: string }>, - releasePrompt: null as (() => void) | null, backendArgs: null as { command: string; args?: string[] } | null, setConfigOptionCalls: [] as Array<{ sessionId: string; configId: string; value: string }>, deferSetConfigOption: null as Promise | null, releaseSetConfigOption: null as (() => void) | null, deferLoadSession: null as Promise | null, releaseLoadSession: null as (() => void) | null, - stderrErrorHandler: null as ((error: { type: string; message: string; raw?: string }) => void) | null, + stderrErrorHandler: null as ((error: { + type: string + message: string + raw: string + }) => void) | null, + emitStderrOnPrompt: null as { + type: 'rate_limit' | 'model_not_found' | 'authentication' | 'quota_exceeded' | 'unknown' + message: string + raw: string + } | null, + emitStderrOnInitialize: null as { + type: 'rate_limit' | 'model_not_found' | 'authentication' | 'quota_exceeded' | 'unknown' + message: string + raw: string + } | null, + emitStderrOnLoadSession: null as { + type: 'rate_limit' | 'model_not_found' | 'authentication' | 'quota_exceeded' | 'unknown' + message: string + raw: string + } | null, + emitTextOnPrompt: null as string | null, + promptReject: null as Error | null, + deferPrompt: null as Promise | null, + releasePrompt: null as (() => void) | null, + deferBeforeSend: null as Promise | null, + releaseBeforeSend: null as (() => void) | null, + promptSends: 0, + /** When cancelPrompt runs, reject the deferred prompt with this error. */ + rejectPromptOnCancel: null as Error | null, disconnectError: null as Error | null, overlayCleanup: null as ReturnType | null, agentActivityListener: null as ((thinking: boolean) => void) | null @@ -55,12 +81,19 @@ vi.mock('./utils/cursorAcpBackend', () => ({ return { initialize: vi.fn(async () => { harness.initializeAttempts += 1; + if (harness.emitStderrOnInitialize && harness.stderrErrorHandler) { + harness.stderrErrorHandler(harness.emitStderrOnInitialize); + } + // Remap path (#1430 / stale spawn): fail only the first initialize + // so the launcher can retry after rewriting --model. if (harness.initializeError && harness.initializeAttempts === 1) { - harness.stderrErrorHandler?.({ - type: 'model_not_found', - message: harness.initializeError.message, - raw: harness.initializeError.message - }); + if (!harness.emitStderrOnInitialize) { + harness.stderrErrorHandler?.({ + type: 'model_not_found', + message: harness.initializeError.message, + raw: harness.initializeError.message + }); + } throw harness.initializeError; } }), @@ -71,6 +104,9 @@ vi.mock('./utils/cursorAcpBackend', () => ({ if (harness.deferLoadSession) { await harness.deferLoadSession; } + if (harness.emitStderrOnLoadSession && harness.stderrErrorHandler) { + harness.stderrErrorHandler(harness.emitStderrOnLoadSession); + } if (harness.loadSessionError) throw harness.loadSessionError; return 'loaded-acp-session'; }), @@ -137,18 +173,46 @@ vi.mock('./utils/cursorAcpBackend', () => ({ } return undefined; }), - prompt: vi.fn(async (_sessionId: string, content: unknown[], onMessage?: (message: AgentMessage) => void) => { + prompt: vi.fn(async ( + _sessionId: string, + content: unknown[], + onMessage?: (message: AgentMessage) => void, + options?: { shouldSend?: () => boolean } + ) => { harness.promptCalls++; + if (harness.deferBeforeSend) { + await harness.deferBeforeSend; + if (options?.shouldSend && !options.shouldSend()) { + return false; + } + } + harness.promptSends++; harness.prompts.push(content); const messages = harness.promptMessageBatches.shift() ?? harness.promptMessages.splice(0, 1); for (const message of messages) onMessage?.(message); + if (harness.emitTextOnPrompt && onMessage) { + onMessage({ type: 'text', text: harness.emitTextOnPrompt }); + } const stderrError = harness.promptStderrErrors.shift(); if (stderrError) harness.stderrErrorHandler?.(stderrError); + if (harness.emitStderrOnPrompt && harness.stderrErrorHandler) { + harness.stderrErrorHandler(harness.emitStderrOnPrompt); + } if (harness.deferPrompt) await harness.deferPrompt; const error = harness.promptErrors.shift(); if (error) throw error; + if (harness.promptReject) { + throw harness.promptReject; + } + return true; + }), + cancelPrompt: vi.fn(async () => { + // Settlement of a deferred prompt is owned by the test so + // userAbortRequested is visible before classifyAcpRpcRejection. + if (harness.rejectPromptOnCancel) { + harness.promptReject = harness.rejectPromptOnCancel; + } }), - cancelPrompt: vi.fn(async () => {}), getPromptGeneration: vi.fn(() => 1), beginSoftSteerPrompt: vi.fn(() => ({ dispatched: harness.softSteerDispatchError @@ -160,8 +224,8 @@ vi.mock('./utils/cursorAcpBackend', () => ({ abortSoftSteers: vi.fn(), waitForResponseComplete: vi.fn(async () => {}), respondToPermission: vi.fn(async () => {}), - onStderrError: vi.fn((handler) => { - harness.stderrErrorHandler = handler ?? null; + onStderrError: vi.fn((handler: typeof harness.stderrErrorHandler) => { + harness.stderrErrorHandler = handler; }), setUsageUpdateListener: vi.fn(), setAgentActivityListener: vi.fn((listener: ((thinking: boolean) => void) | null) => { @@ -227,8 +291,9 @@ import { _resetSharedCursorModelsCacheForTests, writeSharedCursorModelsCache } from '@/modules/common/cursorModelsSharedCache'; +import { setAutoBridgeTransientModelErrors } from './cursorModelErrorBridgePrefs'; -function makeSession(sessionId: string | null, closeQueue = true): CursorSession { +function makeSession(sessionId: string | null, opts?: { keepQueueOpen?: boolean }): CursorSession { const queue = new MessageQueue2(() => 'mode'); const client = makeClient(); @@ -247,7 +312,7 @@ function makeSession(sessionId: string | null, closeQueue = true): CursorSession }); session.onSessionFoundWithProtocol = vi.fn(); - if (closeQueue) { + if (!opts?.keepQueueOpen) { queue.close(); } @@ -265,6 +330,7 @@ function makeClient() { }), unregisterHandler: vi.fn() }, + getMetadata: vi.fn(() => null), updateMetadata: vi.fn(), flushMetadata: vi.fn(async () => true), sendSessionEvent: vi.fn(), @@ -287,8 +353,8 @@ describe('cursorAcpRemoteLauncher', () => { harness.failSetConfigOption = false; harness.supportsLoadSession = true; harness.loadSessionCalled = false; - harness.newSessionCalled = false; harness.newSessionAttempts = 0; + harness.newSessionCalled = false; harness.promptCalls = 0; harness.prompts = []; harness.deferPrompt = null; @@ -306,16 +372,29 @@ describe('cursorAcpRemoteLauncher', () => { harness.deferLoadSession = null; harness.releaseLoadSession = null; harness.stderrErrorHandler = null; + harness.emitStderrOnPrompt = null; + harness.emitStderrOnInitialize = null; + harness.emitStderrOnLoadSession = null; + harness.emitTextOnPrompt = null; + harness.promptReject = null; + harness.deferPrompt = null; + harness.releasePrompt = null; + harness.deferBeforeSend = null; + harness.releaseBeforeSend = null; + harness.promptSends = 0; + harness.rejectPromptOnCancel = null; harness.disconnectError = null; harness.overlayCleanup = null; harness.agentActivityListener = null; legacyLauncher.mockClear(); + setAutoBridgeTransientModelErrors(false); process.stdin.isTTY = false; process.stdout.isTTY = false; }); afterEach(() => { vi.clearAllMocks(); + setAutoBridgeTransientModelErrors(false); _resetSharedCursorModelsCacheForTests(); }); @@ -325,7 +404,7 @@ describe('cursorAcpRemoteLauncher', () => { // Soft-steer completion never settles — simulates Cursor keeping the // concurrent request open after an ordinary Abort. harness.deferSoftSteer = new Promise(() => {}); - const session = makeSession(null, false); + const session = makeSession(null, { keepQueueOpen: true }); const mode = { permissionMode: 'default' } as EnhancedMode; session.queue.push('first', mode, 'first'); @@ -352,7 +431,7 @@ describe('cursorAcpRemoteLauncher', () => { let releasePrompt!: () => void; harness.deferPrompt = new Promise((resolve) => { releasePrompt = resolve; }); harness.softSteerDispatchError = new Error('stdin closed'); - const session = makeSession(null, false); + const session = makeSession(null, { keepQueueOpen: true }); const mode = { permissionMode: 'default' } as EnhancedMode; session.queue.push('first', mode, 'first'); @@ -379,7 +458,7 @@ describe('cursorAcpRemoteLauncher', () => { let rejectSoftSteer!: (error: Error) => void; harness.deferPrompt = new Promise((resolve) => { releasePrompt = resolve; }); harness.deferSoftSteer = new Promise((_, reject) => { rejectSoftSteer = reject; }); - const session = makeSession(null, false); + const session = makeSession(null, { keepQueueOpen: true }); const mode = { permissionMode: 'default' } as EnhancedMode; session.queue.push('first', mode, 'first'); @@ -411,7 +490,7 @@ describe('cursorAcpRemoteLauncher', () => { harness.deferSoftSteer = new Promise((_, reject) => { rejectSoftSteer = reject; }); const indeterminate = new Error('ACP transport closed'); Object.defineProperty(indeterminate, ACP_INDETERMINATE_SYMBOL, { value: true }); - const session = makeSession(null, false); + const session = makeSession(null, { keepQueueOpen: true }); const mode = { permissionMode: 'default' } as EnhancedMode; session.queue.push('first', mode, 'first'); @@ -441,7 +520,7 @@ describe('cursorAcpRemoteLauncher', () => { const indeterminate = new Error('ACP write callback failed'); Object.defineProperty(indeterminate, ACP_INDETERMINATE_SYMBOL, { value: true }); harness.softSteerDispatchError = indeterminate; - const session = makeSession(null, false); + const session = makeSession(null, { keepQueueOpen: true }); const mode = { permissionMode: 'default' } as EnhancedMode; session.queue.push('first', mode, 'first'); @@ -468,7 +547,7 @@ describe('cursorAcpRemoteLauncher', () => { let releaseDispatch!: () => void; harness.deferPrompt = new Promise((resolve) => { releasePrompt = resolve; }); harness.deferSoftSteerDispatch = new Promise((resolve) => { releaseDispatch = resolve; }); - const session = makeSession(null, false); + const session = makeSession(null, { keepQueueOpen: true }); const mode = { permissionMode: 'default' } as EnhancedMode; session.queue.push('first', mode, 'first'); @@ -499,7 +578,7 @@ describe('cursorAcpRemoteLauncher', () => { harness.deferPrompt = new Promise((resolve) => { releasePrompt = resolve; }); harness.deferSoftSteerDispatch = new Promise((resolve) => { releaseDispatch = resolve; }); harness.deferSoftSteer = new Promise((resolve) => { releaseSoftSteer = resolve; }); - const session = makeSession(null, false); + const session = makeSession(null, { keepQueueOpen: true }); const mode = { permissionMode: 'default' } as EnhancedMode; session.queue.push('first', mode, 'first'); @@ -532,7 +611,7 @@ describe('cursorAcpRemoteLauncher', () => { let releaseDispatch!: () => void; harness.deferPrompt = new Promise((resolve) => { releasePrompt = resolve; }); harness.deferSoftSteerDispatch = new Promise((resolve) => { releaseDispatch = resolve; }); - const session = makeSession(null, false); + const session = makeSession(null, { keepQueueOpen: true }); const mode = { permissionMode: 'default' } as EnhancedMode; session.queue.push('first', mode, 'first'); @@ -565,7 +644,7 @@ describe('cursorAcpRemoteLauncher', () => { // Completion never resolves — simulates Cursor keeping the concurrent // request open past Exit/Switch. harness.deferSoftSteer = new Promise(() => {}); - const session = makeSession(null, false); + const session = makeSession(null, { keepQueueOpen: true }); const mode = { permissionMode: 'default' } as EnhancedMode; session.queue.push('first', mode, 'first'); @@ -883,6 +962,7 @@ describe('cursorAcpRemoteLauncher', () => { const queue = new MessageQueue2(() => 'mode'); const client = makeClient() as unknown as ApiSessionClient & { sendAgentMessage: ReturnType; + sendSessionEvent: ReturnType; }; const session = new CursorSession({ api: {} as never, @@ -908,6 +988,12 @@ describe('cursorAcpRemoteLauncher', () => { type: 'error', message: expect.stringContaining('not retried') })); + expect(client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'modelError' + )).toBe(true); + expect(client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'modelErrorBridged' + )).toBe(false); }); it('removes the Cursor MCP overlay even when backend.disconnect rejects', async () => { @@ -1763,4 +1849,1121 @@ describe('cursorAcpRemoteLauncher', () => { expect(JSON.stringify(harness.prompts[1])).toContain('second'); expect(JSON.stringify(harness.prompts[1])).not.toContain('skill_lookup'); }); + + it('keeps generic unknown stderr status-only and still emits ready', async () => { + // Bot Major: type:unknown comes from any stderr with error/failed/exception. + // Must not set turnHasModelError / suppress ready / write lastModelError. + harness.emitStderrOnPrompt = { + type: 'unknown', + message: 'Some plugin failed to load: exception during init', + raw: 'Some plugin failed to load: exception during init' + }; + + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + sendSessionEvent: ReturnType + updateMetadata: ReturnType + sendAgentMessage: ReturnType + }; + + session.queue.push('hello', { permissionMode: 'default' }); + session.queue.close(); + + await cursorAcpRemoteLauncher(session); + + expect(harness.promptCalls).toBe(1); + expect(client.sendSessionEvent).toHaveBeenCalledWith({ type: 'ready' }); + expect(client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'modelError' + )).toBe(false); + const wroteLastModelError = client.updateMetadata.mock.calls.some((call) => { + const updater = call[0] as (m: Record) => Record; + if (typeof updater !== 'function') return false; + return Boolean(updater({}).lastModelError); + }); + expect(wroteLastModelError).toBe(false); + }); + + it('keeps weak typed authentication stderr status-only and still emits ready', async () => { + // Transport types "authentication provider initialized" as authentication + // via bare substring; strong-signature gate must keep it status-only. + harness.emitStderrOnPrompt = { + type: 'authentication', + message: 'authentication provider initialized', + raw: 'authentication provider initialized' + }; + + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + sendSessionEvent: ReturnType + }; + + session.queue.push('hello', { permissionMode: 'default' }); + session.queue.close(); + + await cursorAcpRemoteLauncher(session); + + expect(client.sendSessionEvent).toHaveBeenCalledWith({ type: 'ready' }); + expect(client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'modelError' + )).toBe(false); + }); + + it('records modelError for model_not_found stderr during prompt and suppresses ready', async () => { + harness.emitStderrOnPrompt = { + type: 'model_not_found', + message: 'Cannot use this model: cursor-bad-id. Available models: auto', + raw: 'Cannot use this model: cursor-bad-id. Available models: auto' + }; + + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + sendSessionEvent: ReturnType + }; + + session.queue.push('hello', { permissionMode: 'default' }); + session.queue.close(); + + await cursorAcpRemoteLauncher(session); + + expect(client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'modelError' && call[0]?.kind === 'model_not_found' + )).toBe(true); + expect(client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'ready' + )).toBe(false); + }); + + it('ignores Cannot use this model stderr during initialize/load so remap can succeed', async () => { + // Setup/load remap rejects a stale spawn model on stderr, then continues. + // Must not persist lastModelError / suppress later ready. + const stale = { + type: 'model_not_found' as const, + message: 'Cannot use this model: grok-4.5[fast=true]. Available models: auto', + raw: 'Cannot use this model: grok-4.5[fast=true]. Available models: auto' + }; + harness.emitStderrOnInitialize = stale; + harness.emitStderrOnLoadSession = stale; + + const session = makeSession('resume-remap-ok', { keepQueueOpen: true }); + const client = session.client as unknown as { + sendSessionEvent: ReturnType + updateMetadata: ReturnType + }; + + session.queue.push('hello', { permissionMode: 'default' }); + session.queue.close(); + + await cursorAcpRemoteLauncher(session); + + expect(harness.loadSessionCalled).toBe(true); + expect(client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'modelError' + )).toBe(false); + expect(client.updateMetadata.mock.calls.some((call) => { + const updater = call[0] as (m: Record) => Record; + if (typeof updater !== 'function') return false; + return Boolean(updater({}).lastModelError); + })).toBe(false); + expect(client.sendSessionEvent).toHaveBeenCalledWith({ type: 'ready' }); + }); + + it('still records modelError for typed rate_limit stderr and suppresses ready', async () => { + harness.emitStderrOnPrompt = { + type: 'rate_limit', + message: 'Rate limit exceeded.', + raw: 'status 429 ratelimitexceeded' + }; + + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + sendSessionEvent: ReturnType + updateMetadata: ReturnType + }; + + session.queue.push('hello', { permissionMode: 'default' }); + session.queue.close(); + + await cursorAcpRemoteLauncher(session); + + expect(client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'modelError' && call[0]?.kind === 'rate_limited' + )).toBe(true); + expect(client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'ready' + )).toBe(false); + const wroteLastModelError = client.updateMetadata.mock.calls.some((call) => { + const updater = call[0] as (m: Record) => Record; + if (typeof updater !== 'function') return false; + const next = updater({}); + return (next.lastModelError as { kind?: string } | undefined)?.kind === 'rate_limited'; + }); + expect(wroteLastModelError).toBe(true); + }); + + it('prefers structural RPC classification over text fallback when both fire', async () => { + // Prompt callback emits wire text first (unknown_t_prefix / non-transient), + // then the promise rejects with WritableIterable (transport_closed). + harness.emitTextOnPrompt = '\n\nError: T: WritableIterable is closed'; + harness.promptReject = new Error('WritableIterable is closed'); + + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + sendSessionEvent: ReturnType + updateMetadata: ReturnType + }; + + session.queue.push('hello', { permissionMode: 'default' }); + session.queue.close(); + + await cursorAcpRemoteLauncher(session); + + const modelErrors = client.sendSessionEvent.mock.calls + .map((call) => call[0]) + .filter((event) => event?.type === 'modelError'); + expect(modelErrors).toHaveLength(1); + expect(modelErrors[0]?.kind).toBe('transport_closed'); + expect(modelErrors[0]?.transient).toBe(false); + expect(client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'ready' + )).toBe(false); + }); + + it('prefers specific deferred stderr over generic transport_closed RPC', async () => { + // Strong quota stderr first, then generic transport close — keep the + // non-transient cause so retry copy is not “safe to retry”. + harness.emitStderrOnPrompt = { + type: 'quota_exceeded', + message: 'Quota exceeded.', + raw: 'resource exhausted' + }; + harness.promptReject = new Error('WritableIterable is closed'); + + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + sendSessionEvent: ReturnType + }; + + session.queue.push('hello', { permissionMode: 'default' }); + session.queue.close(); + + await cursorAcpRemoteLauncher(session); + + const modelErrors = client.sendSessionEvent.mock.calls + .map((call) => call[0]) + .filter((event) => event?.type === 'modelError'); + expect(modelErrors).toHaveLength(1); + expect(modelErrors[0]?.kind).toBe('quota_exhausted'); + expect(modelErrors[0]?.transient).toBe(false); + }); + + it('still records modelError for canceled RPC rejection without user abort', async () => { + harness.promptReject = new Error('Error: T: [canceled] Operation aborted'); + + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + sendSessionEvent: ReturnType + }; + + session.queue.push('hello', { permissionMode: 'default' }); + session.queue.close(); + await cursorAcpRemoteLauncher(session); + + expect(client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'modelError' && call[0]?.kind === 'canceled' + )).toBe(true); + }); + + it('does not promote user Abort cancel rejection to modelError', async () => { + // Cursor rejects session/prompt after session/cancel with this wire shape; + // classifier maps it to kind=canceled, but Abort must not page/notify. + // Switch → requestExit → handleAbort sets shouldExit + userAbortRequested + // before we settle the deferred prompt rejection (ordering matches + // cancel-then-reject on the wire; avoids queue.reset hang in tests). + harness.deferPrompt = new Promise((resolve) => { + harness.releasePrompt = resolve; + }); + harness.rejectPromptOnCancel = new Error('Error: T: [canceled] Operation aborted'); + + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + rpcHandlerManager: { registerHandler: ReturnType } + sendSessionEvent: ReturnType + updateMetadata: ReturnType + }; + + session.queue.push('hello', { permissionMode: 'default' }); + + const launchPromise = cursorAcpRemoteLauncher(session); + await vi.waitFor(() => expect(harness.promptCalls).toBe(1)); + + const switchHandler = client.rpcHandlerManager.registerHandler.mock.calls.find( + (call) => call[0] === 'switch' + )?.[1] as (() => Promise) | undefined; + expect(switchHandler).toBeTypeOf('function'); + await switchHandler!(); + harness.releasePrompt?.(); + await launchPromise; + + expect(client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'modelError' + )).toBe(false); + const wroteLastModelError = client.updateMetadata.mock.calls.some((call) => { + const updater = call[0] as (m: Record) => Record; + if (typeof updater !== 'function') return false; + return Boolean(updater({}).lastModelError); + }); + expect(wroteLastModelError).toBe(false); + }); + + it('does not mark an in-flight bridge recovered when Abort wins the race', async () => { + // Enqueue bridge while idle, then hold the bridge prompt and abort before + // it settles — must NOT emit recovered. + const session = makeSession('acp-session', { keepQueueOpen: true }); + const client = session.client as unknown as { + rpcHandlerManager: { registerHandler: ReturnType } + sendSessionEvent: ReturnType + updateMetadata: ReturnType + }; + + let waitCount = 0; + const nextWait = { release: null as (() => void) | null }; + const originalWait = session.queue.waitForMessagesAndGetAsString.bind(session.queue); + session.queue.waitForMessagesAndGetAsString = async (signal) => { + waitCount += 1; + // Park from the second wait (post-first-turn idle) so bridge stays queued. + if (waitCount >= 2 && nextWait.release === null) { + await new Promise((resolve) => { + nextWait.release = resolve; + }); + } + return originalWait(signal); + }; + + session.queue.push('hello', { permissionMode: 'default' }); + const launchPromise = cursorAcpRemoteLauncher(session); + await vi.waitFor(() => expect(harness.promptCalls).toBe(1)); + await vi.waitFor(() => nextWait.release !== null); + + const bridgeHandler = client.rpcHandlerManager.registerHandler.mock.calls.find( + (call) => call[0] === RPC_METHODS.BridgeModelError + )?.[1] as ((payload: unknown) => Promise<{ ok: boolean; reason?: string }>) | undefined; + const eventId = '55555555-5555-4555-8555-555555555555'; + expect(await bridgeHandler!({ + eventId, + kind: 'rate_limited', + transient: true, + rawSnippet: 'status 429', + lastUserMessage: 'hello', + priorAssistantClaimsDone: false + })).toEqual({ ok: true }); + + const bridgeGate = { release: null as (() => void) | null }; + harness.deferPrompt = new Promise((resolve) => { + bridgeGate.release = resolve; + }); + nextWait.release?.(); + await vi.waitFor(() => expect(harness.promptCalls).toBe(2)); + + const abortHandler = client.rpcHandlerManager.registerHandler.mock.calls.find( + (call) => call[0] === RPC_METHODS.Abort + )?.[1] as (() => Promise) | undefined; + expect(abortHandler).toBeTypeOf('function'); + await abortHandler!(); + bridgeGate.release?.(); + session.queue.close(); + await launchPromise; + + expect(client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'modelErrorBridged' + )).toBe(false); + const wroteBridgedForEventId = client.updateMetadata.mock.calls.some((call) => { + const updater = call[0] as (m: Record) => Record; + if (typeof updater !== 'function') return false; + const err = updater({}).lastModelError as { bridgedForEventId?: string } | undefined; + return err?.bridgedForEventId === eventId; + }); + expect(wroteBridgedForEventId).toBe(false); + }); + + it('does not promote ACP process-exit rejection after deliberate abort to modelError', async () => { + harness.deferPrompt = new Promise((resolve) => { + harness.releasePrompt = resolve; + }); + + // Non-null Cursor session id so handleAbort reaches backend.cancelPrompt. + const session = makeSession('acp-session', { keepQueueOpen: true }); + const client = session.client as unknown as { + rpcHandlerManager: { registerHandler: ReturnType } + sendSessionEvent: ReturnType + updateMetadata: ReturnType + }; + + session.queue.push('hello', { permissionMode: 'default' }); + const launchPromise = cursorAcpRemoteLauncher(session); + await vi.waitFor(() => expect(harness.promptCalls).toBe(1)); + + harness.rejectPromptOnCancel = new Error('ACP process exited (code=143, signal=null)'); + const abortHandler = client.rpcHandlerManager.registerHandler.mock.calls.find( + (call) => call[0] === RPC_METHODS.Abort + )?.[1] as (() => Promise) | undefined; + expect(abortHandler).toBeTypeOf('function'); + await abortHandler!(); + harness.releasePrompt?.(); + session.queue.close(); + await launchPromise; + + expect(client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'modelError' + )).toBe(false); + const wroteLastModelError = client.updateMetadata.mock.calls.some((call) => { + const updater = call[0] as (m: Record) => Record; + if (typeof updater !== 'function') return false; + return Boolean(updater({}).lastModelError); + }); + expect(wroteLastModelError).toBe(false); + expect(session.queue.pendingLocalIds().some((id) => id.startsWith('bridge:'))).toBe(false); + }); + + it('rejects manual bridge while a normal prompt is in flight', async () => { + harness.deferPrompt = new Promise((resolve) => { + harness.releasePrompt = resolve; + }); + + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + rpcHandlerManager: { registerHandler: ReturnType } + }; + + session.queue.push('hello', { permissionMode: 'default' }); + const launchPromise = cursorAcpRemoteLauncher(session); + await vi.waitFor(() => expect(harness.promptCalls).toBe(1)); + + const bridgeHandler = client.rpcHandlerManager.registerHandler.mock.calls.find( + (call) => call[0] === RPC_METHODS.BridgeModelError + )?.[1] as ((payload: unknown) => Promise<{ ok: boolean; reason?: string }>) | undefined; + + const eventId = '66666666-6666-4666-8666-666666666666'; + expect(await bridgeHandler!({ + eventId, + kind: 'rate_limited', + transient: true, + rawSnippet: 'status 429', + lastUserMessage: 'older failed turn', + priorAssistantClaimsDone: false + })).toEqual({ ok: false, reason: 'prompt_in_flight' }); + expect(session.queue.pendingLocalIds().some((id) => id.startsWith('bridge:'))).toBe(false); + + session.queue.close(); + harness.releasePrompt?.(); + await launchPromise; + }); + + it('clears pending bridge on abort so the same event can be bridged again', async () => { + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + rpcHandlerManager: { registerHandler: ReturnType } + }; + + // Park the post-turn wait so a manual bridge stays queued (idle, not mid-prompt). + let waitCount = 0; + const nextWait = { release: null as (() => void) | null }; + const originalWait = session.queue.waitForMessagesAndGetAsString.bind(session.queue); + session.queue.waitForMessagesAndGetAsString = async (signal) => { + waitCount += 1; + if (waitCount >= 2 && nextWait.release === null) { + await new Promise((resolve) => { + nextWait.release = resolve; + }); + } + return originalWait(signal); + }; + + session.queue.push('hello', { permissionMode: 'default' }); + const launchPromise = cursorAcpRemoteLauncher(session); + await vi.waitFor(() => expect(harness.promptCalls).toBe(1)); + await vi.waitFor(() => nextWait.release !== null); + + const bridgeHandler = client.rpcHandlerManager.registerHandler.mock.calls.find( + (call) => call[0] === RPC_METHODS.BridgeModelError + )?.[1] as ((payload: unknown) => Promise<{ ok: boolean; reason?: string }>) | undefined; + expect(bridgeHandler).toBeTypeOf('function'); + + const eventId = '11111111-1111-4111-8111-111111111111'; + const bridgePayload = { + eventId, + kind: 'rate_limited', + transient: true, + rawSnippet: 'status 429', + lastUserMessage: 'hello', + priorAssistantClaimsDone: false + }; + + expect(await bridgeHandler!(bridgePayload)).toEqual({ ok: true }); + expect(session.queue.queue.some((item) => item.internal?.kind === 'model-error-bridge' && item.internal.eventId === eventId)).toBe(true); + expect(session.queue.pendingLocalIds().some((id) => id.startsWith('bridge:'))).toBe(false); + expect(await bridgeHandler!(bridgePayload)).toEqual({ + ok: false, + reason: 'not_bridgeable' + }); + + const abortHandler = client.rpcHandlerManager.registerHandler.mock.calls.find( + (call) => call[0] === RPC_METHODS.Abort + )?.[1] as (() => Promise) | undefined; + expect(abortHandler).toBeTypeOf('function'); + await abortHandler!(); + + expect(session.queue.pendingLocalIds().some((id) => id.startsWith('bridge:'))).toBe(false); + expect(await bridgeHandler!(bridgePayload)).toEqual({ ok: true }); + expect(session.queue.queue.some((item) => item.internal?.kind === 'model-error-bridge' && item.internal.eventId === eventId)).toBe(true); + expect(session.queue.pendingLocalIds().some((id) => id.startsWith('bridge:'))).toBe(false); + + session.queue.close(); + nextWait.release?.(); + await launchPromise; + }); + + it('treats a forged bridge: localId user turn as normal and refuses Bridge overtake', async () => { + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + rpcHandlerManager: { registerHandler: ReturnType } + }; + + let waitCount = 0; + const nextWait = { release: null as (() => void) | null }; + const originalWait = session.queue.waitForMessagesAndGetAsString.bind(session.queue); + session.queue.waitForMessagesAndGetAsString = async (signal) => { + waitCount += 1; + if (waitCount >= 2 && nextWait.release === null) { + await new Promise((resolve) => { + nextWait.release = resolve; + }); + } + return originalWait(signal); + }; + + session.queue.push('first', { permissionMode: 'default' }); + const launchPromise = cursorAcpRemoteLauncher(session); + await vi.waitFor(() => expect(harness.promptCalls).toBe(1)); + await vi.waitFor(() => nextWait.release !== null); + + // Forged localId must not count as queue-owned Bridge provenance. + session.queue.push('forged', { permissionMode: 'default' }, 'bridge:evt-forged'); + expect(session.queue.hasPendingNonBridgeTurn()).toBe(true); + + const bridgeHandler = client.rpcHandlerManager.registerHandler.mock.calls.find( + (call) => call[0] === RPC_METHODS.BridgeModelError + )?.[1] as ((payload: unknown) => Promise<{ ok: boolean; reason?: string }>) | undefined; + + expect(await bridgeHandler!({ + eventId: '55555555-5555-4555-8555-555555555555', + kind: 'rate_limited', + transient: true, + rawSnippet: 'status 429', + lastUserMessage: 'first', + priorAssistantClaimsDone: false + })).toEqual({ ok: false, reason: 'superseded_by_newer_turn' }); + expect(session.queue.queue.some((item) => item.internal?.kind === 'model-error-bridge')).toBe(false); + + session.queue.close(); + nextWait.release?.(); + await launchPromise; + }); + + it('rejects Bridge when the last user prompt exceeds the exact-replay limit', async () => { + const { MAX_LAST_USER_MESSAGE_CHARS } = await import('./cursorModelErrorBridge'); + harness.emitStderrOnPrompt = { + type: 'rate_limit', + message: 'Rate limit exceeded.', + raw: 'status 429 ratelimitexceeded' + }; + + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + rpcHandlerManager: { registerHandler: ReturnType } + sendSessionEvent: ReturnType + updateMetadata: ReturnType + }; + + const longPrompt = 'x'.repeat(MAX_LAST_USER_MESSAGE_CHARS + 1); + session.queue.push(longPrompt, { permissionMode: 'default' }); + session.queue.close(); + await cursorAcpRemoteLauncher(session); + + await vi.waitFor(() => client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'modelError' + )); + + const recorded = client.updateMetadata.mock.calls + .map((c) => { + const u = c[0] as (m: Record) => Record; + if (typeof u !== 'function') return null; + return u({}).lastModelError as { + eventId?: string + bridgeable?: boolean + lastUserMessage?: string + } | undefined; + }) + .find((err) => typeof err?.eventId === 'string'); + expect(recorded?.bridgeable).toBe(false); + expect(recorded?.lastUserMessage).toBe(''); + + const bridgeHandler = client.rpcHandlerManager.registerHandler.mock.calls.find( + (call) => call[0] === RPC_METHODS.BridgeModelError + )?.[1] as ((payload: unknown) => Promise<{ ok: boolean; reason?: string }>) | undefined; + expect(await bridgeHandler!({ + eventId: recorded?.eventId, + kind: 'rate_limited', + transient: true, + rawSnippet: 'status 429', + lastUserMessage: longPrompt, + priorAssistantClaimsDone: false + })).toEqual({ ok: false, reason: 'not_bridgeable' }); + }); + + it('does not wrap pass-through slash commands as Bridge prompts', async () => { + harness.emitStderrOnPrompt = { + type: 'rate_limit', + message: 'Rate limit exceeded.', + raw: 'status 429 ratelimitexceeded' + }; + + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + rpcHandlerManager: { registerHandler: ReturnType } + sendSessionEvent: ReturnType + updateMetadata: ReturnType + }; + + session.queue.push('/compact keep recap', { permissionMode: 'default' }); + session.queue.close(); + await cursorAcpRemoteLauncher(session); + + await vi.waitFor(() => client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'modelError' + )); + + const recorded = client.updateMetadata.mock.calls + .map((c) => { + const u = c[0] as (m: Record) => Record; + if (typeof u !== 'function') return null; + return u({}).lastModelError as { + eventId?: string + bridgeable?: boolean + lastUserMessage?: string + } | undefined; + }) + .find((err) => typeof err?.eventId === 'string'); + expect(recorded?.bridgeable).toBe(false); + expect(recorded?.lastUserMessage).toBe(''); + + const bridgeHandler = client.rpcHandlerManager.registerHandler.mock.calls.find( + (call) => call[0] === RPC_METHODS.BridgeModelError + )?.[1] as ((payload: unknown) => Promise<{ ok: boolean; reason?: string }>) | undefined; + expect(await bridgeHandler!({ + eventId: recorded?.eventId, + kind: 'rate_limited', + transient: true, + rawSnippet: 'status 429', + lastUserMessage: '/compact keep recap', + priorAssistantClaimsDone: false + })).toEqual({ ok: false, reason: 'not_bridgeable' }); + }); + + it('rejects manual bridge when a newer user turn is already queued', async () => { + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + rpcHandlerManager: { registerHandler: ReturnType } + updateMetadata: ReturnType + }; + + let waitCount = 0; + const nextWait = { release: null as (() => void) | null }; + const originalWait = session.queue.waitForMessagesAndGetAsString.bind(session.queue); + session.queue.waitForMessagesAndGetAsString = async (signal) => { + waitCount += 1; + if (waitCount >= 2 && nextWait.release === null) { + await new Promise((resolve) => { + nextWait.release = resolve; + }); + } + return originalWait(signal); + }; + + session.queue.push('first', { permissionMode: 'default' }); + const launchPromise = cursorAcpRemoteLauncher(session); + await vi.waitFor(() => expect(harness.promptCalls).toBe(1)); + await vi.waitFor(() => nextWait.release !== null); + + const bridgeHandler = client.rpcHandlerManager.registerHandler.mock.calls.find( + (call) => call[0] === RPC_METHODS.BridgeModelError + )?.[1] as ((payload: unknown) => Promise<{ ok: boolean; reason?: string }>) | undefined; + + session.queue.push('correction instead of retry', { permissionMode: 'default' }); + expect(await bridgeHandler!({ + eventId: '44444444-4444-4444-8444-444444444444', + kind: 'rate_limited', + transient: true, + rawSnippet: 'status 429', + lastUserMessage: 'first', + priorAssistantClaimsDone: false + })).toEqual({ ok: false, reason: 'superseded_by_newer_turn' }); + + expect(session.queue.pendingLocalIds().some((id) => id.startsWith('bridge:'))).toBe(false); + expect(session.queue.queue.some((item) => item.message === 'correction instead of retry')).toBe(true); + + session.queue.close(); + nextWait.release?.(); + await launchPromise; + }); + + it('does not auto-bridge ahead of a newer queued user turn', async () => { + const { setAutoBridgeTransientModelErrors } = await import('./cursorModelErrorBridgePrefs'); + setAutoBridgeTransientModelErrors(true); + + harness.deferPrompt = new Promise((resolve) => { + harness.releasePrompt = resolve; + }); + harness.promptReject = new Error('status 429 ratelimitexceeded'); + + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + sendSessionEvent: ReturnType + }; + + session.queue.push('first', { permissionMode: 'default' }); + const launchPromise = cursorAcpRemoteLauncher(session); + await vi.waitFor(() => expect(harness.promptCalls).toBe(1)); + + // User queues a replacement while the failing turn is still settling. + session.queue.push('do something else', { permissionMode: 'default' }); + harness.releasePrompt?.(); + + await vi.waitFor(() => client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'modelError' + )); + expect(session.queue.pendingLocalIds().some((id) => id.startsWith('bridge:'))).toBe(false); + expect(session.queue.queue[0]?.message).toBe('do something else'); + + setAutoBridgeTransientModelErrors(false); + session.queue.close(); + await launchPromise; + }); + + it('drops an already-queued Bridge when a newer user turn arrives before dequeue', async () => { + harness.emitStderrOnPrompt = { + type: 'rate_limit', + message: 'Rate limit exceeded.', + raw: 'status 429 ratelimitexceeded' + }; + + let metadata: Record = { + path: '/tmp/project', + host: 'localhost' + }; + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + rpcHandlerManager: { registerHandler: ReturnType } + sendSessionEvent: ReturnType + getMetadata: ReturnType + updateMetadata: ReturnType + }; + client.getMetadata.mockImplementation(() => metadata); + client.updateMetadata.mockImplementation((updater: unknown) => { + if (typeof updater === 'function') { + metadata = (updater as (m: Record) => Record)(metadata); + } + }); + + let waitCount = 0; + const nextWait = { release: null as (() => void) | null }; + const originalWait = session.queue.waitForMessagesAndGetAsString.bind(session.queue); + session.queue.waitForMessagesAndGetAsString = async (signal) => { + waitCount += 1; + if (waitCount >= 2 && nextWait.release === null) { + await new Promise((resolve) => { + nextWait.release = resolve; + }); + } + return originalWait(signal); + }; + + session.queue.push('first', { permissionMode: 'default' }); + const launchPromise = cursorAcpRemoteLauncher(session); + await vi.waitFor(() => expect(harness.promptCalls).toBe(1)); + await vi.waitFor(() => client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'modelError' + )); + await vi.waitFor(() => nextWait.release !== null); + + const recorded = metadata.lastModelError as { + eventId?: string + kind?: string + rawSnippet?: string + } | undefined; + expect(recorded?.eventId).toBeTypeOf('string'); + + const bridgeHandler = client.rpcHandlerManager.registerHandler.mock.calls.find( + (call) => call[0] === RPC_METHODS.BridgeModelError + )?.[1] as ((payload: unknown) => Promise<{ ok: boolean; reason?: string }>) | undefined; + expect(bridgeHandler).toBeTypeOf('function'); + expect(await bridgeHandler!({ + eventId: recorded?.eventId, + kind: recorded?.kind ?? 'rate_limited', + transient: true, + rawSnippet: recorded?.rawSnippet ?? 'status 429', + lastUserMessage: 'first', + priorAssistantClaimsDone: false + })).toEqual({ ok: true }); + expect(session.queue.queue.some((item) => item.internal?.kind === 'model-error-bridge' && item.internal.eventId === recorded?.eventId)).toBe(true); + expect(session.queue.pendingLocalIds().some((id) => id.startsWith('bridge:'))).toBe(false); + + // Newer user intent arrives after Bridge is already at the head. + session.queue.push('correction instead of retry', { permissionMode: 'default' }); + harness.emitStderrOnPrompt = null; + nextWait.release?.(); + + await vi.waitFor(() => expect(harness.promptCalls).toBe(2)); + const secondPrompt = JSON.stringify(harness.prompts[1] ?? []); + expect(secondPrompt).toContain('correction instead of retry'); + expect(session.queue.queue.some((item) => item.internal?.kind === 'model-error-bridge')).toBe(false); + // Bridge was dropped (not executed); subsequent Bridge RPC must fail closed. + expect(await bridgeHandler!({ + eventId: recorded?.eventId, + kind: recorded?.kind ?? 'rate_limited', + transient: true, + rawSnippet: recorded?.rawSnippet ?? 'status 429', + lastUserMessage: 'first', + priorAssistantClaimsDone: false + })).toEqual({ ok: false, reason: 'superseded_by_newer_turn' }); + + session.queue.close(); + await launchPromise; + }); + + it('records idle stderr as non-bridgeable so it cannot replay a finished turn', async () => { + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + rpcHandlerManager: { registerHandler: ReturnType } + sendSessionEvent: ReturnType + updateMetadata: ReturnType + }; + + let waitCount = 0; + const nextWait = { release: null as (() => void) | null }; + const originalWait = session.queue.waitForMessagesAndGetAsString.bind(session.queue); + session.queue.waitForMessagesAndGetAsString = async (signal) => { + waitCount += 1; + if (waitCount >= 2 && nextWait.release === null) { + await new Promise((resolve) => { + nextWait.release = resolve; + }); + } + return originalWait(signal); + }; + + session.queue.push('first', { permissionMode: 'default' }); + const launchPromise = cursorAcpRemoteLauncher(session); + await vi.waitFor(() => expect(harness.promptCalls).toBe(1)); + await vi.waitFor(() => nextWait.release !== null); + + harness.stderrErrorHandler!({ + type: 'rate_limit', + message: 'Rate limit exceeded.', + raw: 'status 429 ratelimitexceeded' + }); + await vi.waitFor(() => client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'modelError' + )); + + const recorded = client.updateMetadata.mock.calls + .map((c) => { + const u = c[0] as (m: Record) => Record; + if (typeof u !== 'function') return null; + return u({}).lastModelError as { + eventId?: string + bridgeable?: boolean + } | undefined; + }) + .find((err) => typeof err?.eventId === 'string'); + expect(recorded?.bridgeable).toBe(false); + + const bridgeHandler = client.rpcHandlerManager.registerHandler.mock.calls.find( + (call) => call[0] === RPC_METHODS.BridgeModelError + )?.[1] as ((payload: unknown) => Promise<{ ok: boolean; reason?: string }>) | undefined; + + expect(await bridgeHandler!({ + eventId: recorded?.eventId, + kind: 'rate_limited', + transient: true, + rawSnippet: 'status 429', + lastUserMessage: 'first', + priorAssistantClaimsDone: false + })).toEqual({ ok: false, reason: 'not_bridgeable' }); + + session.queue.close(); + nextWait.release?.(); + await launchPromise; + }); + + it('hydrates persisted lastModelError so a post-restart turn supersedes it', async () => { + const eventId = '33333333-3333-4333-8333-333333333333'; + let metadata: Record = { + path: '/tmp/project', + host: 'localhost', + lastModelError: { + eventId, + atTs: 1000, + kind: 'rate_limited', + transient: true, + rawSnippet: 'status 429', + priorAssistantClaimsDone: false, + lastUserMessage: 'old failed turn' + } + }; + + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + getMetadata: ReturnType + updateMetadata: ReturnType + rpcHandlerManager: { registerHandler: ReturnType } + }; + client.getMetadata.mockImplementation(() => metadata); + client.updateMetadata.mockImplementation((updater: unknown) => { + if (typeof updater === 'function') { + metadata = (updater as (m: Record) => Record)(metadata); + } + }); + + session.queue.push('continue after restart', { permissionMode: 'default' }); + const launchPromise = cursorAcpRemoteLauncher(session); + await vi.waitFor(() => expect(harness.promptCalls).toBe(1)); + await vi.waitFor(() => { + const err = metadata.lastModelError as { supersededByUserTurn?: boolean } | undefined; + expect(err?.supersededByUserTurn).toBe(true); + }); + + const bridgeHandler = client.rpcHandlerManager.registerHandler.mock.calls.find( + (call) => call[0] === RPC_METHODS.BridgeModelError + )?.[1] as ((payload: unknown) => Promise<{ ok: boolean; reason?: string }>) | undefined; + expect(bridgeHandler).toBeTypeOf('function'); + + expect(await bridgeHandler!({ + eventId, + kind: 'rate_limited', + transient: true, + rawSnippet: 'status 429', + lastUserMessage: 'old failed turn', + priorAssistantClaimsDone: false + })).toEqual({ ok: false, reason: 'superseded_by_newer_turn' }); + + session.queue.close(); + await launchPromise; + }); + + it('rejects bridge after a newer normal turn succeeds', async () => { + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + rpcHandlerManager: { registerHandler: ReturnType } + updateMetadata: ReturnType + sendSessionEvent: ReturnType + }; + + const readLastModelError = (): { + eventId?: string + supersededByUserTurn?: boolean + } | undefined => { + for (let i = client.updateMetadata.mock.calls.length - 1; i >= 0; i -= 1) { + const updater = client.updateMetadata.mock.calls[i]?.[0] as + | ((m: Record) => Record) + | undefined; + if (typeof updater !== 'function') continue; + const err = updater({}).lastModelError as { + eventId?: string + supersededByUserTurn?: boolean + } | undefined; + if (typeof err?.eventId === 'string') { + return err; + } + } + return undefined; + }; + + let waitCount = 0; + const nextWait = { release: null as (() => void) | null }; + const originalWait = session.queue.waitForMessagesAndGetAsString.bind(session.queue); + session.queue.waitForMessagesAndGetAsString = async (signal) => { + waitCount += 1; + if (waitCount >= 2 && nextWait.release === null) { + await new Promise((resolve) => { + nextWait.release = resolve; + }); + } + return originalWait(signal); + }; + + session.queue.push('first', { permissionMode: 'default' }); + const launchPromise = cursorAcpRemoteLauncher(session); + await vi.waitFor(() => expect(harness.promptCalls).toBe(1)); + await vi.waitFor(() => nextWait.release !== null); + + // Idle structural stderr records a durable modelError (no auto-bridge). + expect(harness.stderrErrorHandler).toBeTypeOf('function'); + harness.stderrErrorHandler!({ + type: 'rate_limit', + message: 'Rate limit exceeded.', + raw: 'status 429 ratelimitexceeded' + }); + await vi.waitFor(() => client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'modelError' + )); + + const eventId = readLastModelError()?.eventId; + expect(eventId).toEqual(expect.any(String)); + + const bridgeHandler = client.rpcHandlerManager.registerHandler.mock.calls.find( + (call) => call[0] === RPC_METHODS.BridgeModelError + )?.[1] as ((payload: unknown) => Promise<{ ok: boolean; reason?: string }>) | undefined; + expect(bridgeHandler).toBeTypeOf('function'); + + // Newer normal turn starts → durable supersededByUserTurn gate. + session.queue.push('continue without bridging', { permissionMode: 'default' }); + nextWait.release?.(); + await vi.waitFor(() => expect(harness.promptCalls).toBe(2)); + await vi.waitFor(() => readLastModelError()?.supersededByUserTurn === true); + + expect(await bridgeHandler!({ + eventId, + kind: 'rate_limited', + transient: true, + rawSnippet: 'status 429', + lastUserMessage: 'first', + priorAssistantClaimsDone: false + })).toEqual({ ok: false, reason: 'superseded_by_newer_turn' }); + + session.queue.close(); + await launchPromise; + }); + + it('cancels a pending bridge when a newer modelError supersedes it', async () => { + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + rpcHandlerManager: { registerHandler: ReturnType } + sendSessionEvent: ReturnType + }; + + let waitCount = 0; + const nextWait = { release: null as (() => void) | null }; + const originalWait = session.queue.waitForMessagesAndGetAsString.bind(session.queue); + session.queue.waitForMessagesAndGetAsString = async (signal) => { + waitCount += 1; + if (waitCount >= 2 && nextWait.release === null) { + await new Promise((resolve) => { + nextWait.release = resolve; + }); + } + return originalWait(signal); + }; + + session.queue.push('first', { permissionMode: 'default' }); + const launchPromise = cursorAcpRemoteLauncher(session); + await vi.waitFor(() => expect(harness.promptCalls).toBe(1)); + await vi.waitFor(() => nextWait.release !== null); + + const bridgeHandler = client.rpcHandlerManager.registerHandler.mock.calls.find( + (call) => call[0] === RPC_METHODS.BridgeModelError + )?.[1] as ((payload: unknown) => Promise<{ ok: boolean; reason?: string }>) | undefined; + + const staleEventId = '22222222-2222-4222-8222-222222222222'; + expect(await bridgeHandler!({ + eventId: staleEventId, + kind: 'rate_limited', + transient: true, + rawSnippet: 'status 429', + lastUserMessage: 'first', + priorAssistantClaimsDone: false + })).toEqual({ ok: true }); + expect(session.queue.queue.some((item) => item.internal?.kind === 'model-error-bridge' && item.internal.eventId === staleEventId)).toBe(true); + expect(session.queue.pendingLocalIds().some((id) => id.startsWith('bridge:'))).toBe(false); + + // Idle structural stderr supersedes the displayed error and drops the pending bridge. + expect(harness.stderrErrorHandler).toBeTypeOf('function'); + harness.stderrErrorHandler!({ + type: 'rate_limit', + message: 'Rate limit exceeded again.', + raw: 'status 429 ratelimitexceeded again' + }); + await vi.waitFor(() => client.sendSessionEvent.mock.calls.some( + (call) => call[0]?.type === 'modelError' + )); + + expect(session.queue.queue.some((item) => item.internal?.kind === 'model-error-bridge' && item.internal.eventId === staleEventId)).toBe(false); + expect(await bridgeHandler!({ + eventId: staleEventId, + kind: 'rate_limited', + transient: true, + rawSnippet: 'status 429', + lastUserMessage: 'first', + priorAssistantClaimsDone: false + })).toEqual({ ok: false, reason: 'model_error_changed' }); + + session.queue.close(); + nextWait.release?.(); + await launchPromise; + }); + + it('does not dispatch Bridge session/prompt after a newer turn arrives during pre-send drain', async () => { + const session = makeSession(null, { keepQueueOpen: true }); + const client = session.client as unknown as { + rpcHandlerManager: { registerHandler: ReturnType } + }; + + let waitCount = 0; + const nextWait = { release: null as (() => void) | null }; + const originalWait = session.queue.waitForMessagesAndGetAsString.bind(session.queue); + session.queue.waitForMessagesAndGetAsString = async (signal) => { + waitCount += 1; + if (waitCount >= 2 && nextWait.release === null) { + await new Promise((resolve) => { + nextWait.release = resolve; + }); + } + return originalWait(signal); + }; + + session.queue.push('first', { permissionMode: 'default' }); + const launchPromise = cursorAcpRemoteLauncher(session); + await vi.waitFor(() => expect(harness.promptSends).toBe(1)); + await vi.waitFor(() => nextWait.release !== null); + + const bridgeHandler = client.rpcHandlerManager.registerHandler.mock.calls.find( + (call) => call[0] === RPC_METHODS.BridgeModelError + )?.[1] as ((payload: unknown) => Promise<{ ok: boolean; reason?: string }>) | undefined; + + expect(await bridgeHandler!({ + eventId: '33333333-3333-4333-8333-333333333333', + kind: 'rate_limited', + transient: true, + rawSnippet: 'status 429', + lastUserMessage: 'first', + priorAssistantClaimsDone: false + })).toEqual({ ok: true }); + + harness.deferBeforeSend = new Promise((resolve) => { + harness.releaseBeforeSend = resolve; + }); + nextWait.release?.(); + await vi.waitFor(() => expect(harness.promptCalls).toBe(2)); + + session.queue.push('correction instead of retry', { permissionMode: 'default' }); + harness.releaseBeforeSend?.(); + + await vi.waitFor(() => expect(harness.promptSends).toBe(2)); + const dispatched = JSON.stringify(harness.prompts); + expect(dispatched).not.toContain('[HAPI bridge'); + expect(dispatched).toContain('correction instead of retry'); + + session.queue.close(); + await launchPromise; + }); }); diff --git a/cli/src/cursor/cursorAcpRemoteLauncher.ts b/cli/src/cursor/cursorAcpRemoteLauncher.ts index 5130a17c6c..d961091582 100644 --- a/cli/src/cursor/cursorAcpRemoteLauncher.ts +++ b/cli/src/cursor/cursorAcpRemoteLauncher.ts @@ -12,7 +12,7 @@ import { } from '@/modules/common/remote/RemoteLauncherBase'; import { OpencodeDisplay } from '@/ui/ink/OpencodeDisplay'; import type { CursorSession } from './session'; -import type { PermissionMode } from './loop'; +import type { EnhancedMode, PermissionMode } from './loop'; import { createCursorAcpBackend, CURSOR_ACP_REQUIRED_MESSAGE, @@ -51,6 +51,21 @@ import { isRetryableCursorError, stripRetryableCursorError } from './cursorAutoRetry'; +import { + classifyAcpRpcRejection, + classifyCursorAgentMessage, + isCompletionClaim, + mapAcpStderrToFailure, + rawSnippetForFailure, + type CursorAgentStreamFailure +} from './cursorAgentMessageClassifier'; +import { + buildModelErrorBridgePrompt, + canBridgeModelError, + mergeBridgeGateFields, + MAX_LAST_USER_MESSAGE_CHARS +} from './cursorModelErrorBridge'; +import { getAutoBridgeTransientModelErrors } from './cursorModelErrorBridgePrefs'; const CURSOR_ABORT_DRAIN_TIMEOUT_MS = 5_000; @@ -68,8 +83,6 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { private unregisterModelApplyHandler: (() => void) | null = null; private modelApplySeq = 0; private activePromptModeHash: string | null = null; - /** True while a backend.prompt turn is in flight. */ - private promptInFlight = false; /** Concurrent soft-steer session/prompt RPCs still running after kickoff. */ private softSteerWaiters: Promise[] = []; /** True when ACP process was spawned with `--auto-review`. */ @@ -81,7 +94,52 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { private pendingRetryableFromStderr = false; private pendingInlineRetryableError = false; private attemptProducedToolActivity = false; + private lastAssistantText: string | null = null; + private turnHasModelError = false; + /** + * Text-classifier hit retained until `backend.prompt` settles. Callbacks + * run before the promise rejects, so recording text immediately would + * beat the structural RPC classification (e.g. unknown_t_prefix vs + * transport_closed for WritableIterable). Flush on success; prefer RPC + * in catch. + */ + private pendingTextFailure: CursorAgentStreamFailure | null = null; + /** + * Typed stderr failure deferred while prompt is in flight so RPC rejection + * keeps precedence (RPC → stderr → text). Flushed on settle like text. + */ + private pendingStderrFailure: CursorAgentStreamFailure | null = null; + /** True while backend.prompt() is in flight — lets stderr model_not_found + * surface as modelError during a turn without breaking setup/load remap. */ + private promptInFlight = false; + /** + * Set in handleAbort before session/cancel. Cursor often rejects the + * in-flight prompt as `Error: T: [canceled] Operation aborted`, which the + * classifier maps to kind=canceled (real model cancel). Intent tracking + * keeps deliberate Abort/Exit/Switch out of the emergency model-error path. + */ private userAbortRequested = false; + private lastUserMessage: string | null = null; + private lastTurnMode: EnhancedMode | null = null; + /** Set only while the bridge prompt itself is the active turn. */ + private bridgingForEventId: string | null = null; + private bridgingSource: 'auto' | 'manual' | null = null; + /** Enqueued but not yet started — must not attribute the current in-flight turn. */ + private pendingBridgeEventId: string | null = null; + private pendingBridgeSource: 'auto' | 'manual' | null = null; + private lastRecordedModelError: { + eventId: string; + atTs: number; + kind: string; + rawSnippet: string; + priorAssistantClaimsDone: boolean; + lastUserMessage: string; + transient: boolean; + bridgedForEventId?: string; + retriedAndFailed?: boolean; + supersededByUserTurn?: boolean; + bridgeable?: boolean; + } | null = null; constructor(session: CursorSession) { super(process.env.DEBUG ? session.logPath : undefined); @@ -423,6 +481,11 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { onSwitch: () => this.handleSwitchRequest() }); + session.client.rpcHandlerManager.registerHandler( + RPC_METHODS.BridgeModelError, + async (payload: unknown) => this.handleBridgeModelErrorRpc(payload) + ); + // Soft steer = Cursor GUI "Send" (next-opportune / soft inject): fire a // concurrent session/prompt without canceling the in-flight turn. Abort // remains the hard stop path (GUI "Stop & send"). @@ -573,7 +636,16 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { } ); + // Restart / resume: hub metadata may still hold an unresolved error from + // the previous process. Hydrate before the queue loop so the first newer + // normal turn can durably set supersededByUserTurn. + this.hydrateLastRecordedModelErrorFromMetadata(); + const sendReady = () => { + if (this.turnHasModelError) { + // Don't clear the error state with a 'ready' — banner stays visible. + return; + } session.sendSessionEvent({ type: 'ready' }); }; @@ -589,6 +661,35 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { break; } + // Activate bridge attribution only from queue-owned provenance — + // never from caller-controlled localId (which can forge `bridge:`). + const bridgeItem = batch.items.find( + (item) => item.internal?.kind === 'model-error-bridge' + ); + if (bridgeItem?.internal?.kind === 'model-error-bridge') { + const eventId = bridgeItem.internal.eventId; + // Enqueue-time queue check can go stale: a normal turn may arrive + // after Bridge is already at the head. Drop Bridge and let the + // newer user intent run next (isolated batch already dequeued). + if (this.session.queue.hasPendingNonBridgeTurn()) { + if (this.pendingBridgeEventId === eventId) { + this.pendingBridgeEventId = null; + this.pendingBridgeSource = null; + } + this.markModelErrorSupersededByUserTurn(); + continue; + } + this.bridgingForEventId = eventId; + this.bridgingSource = this.pendingBridgeSource ?? 'manual'; + if (this.pendingBridgeEventId === eventId) { + this.pendingBridgeEventId = null; + this.pendingBridgeSource = null; + } + } else if (this.lastRecordedModelError && !this.lastRecordedModelError.supersededByUserTurn) { + // A newer normal turn owns the conversation — stale Bridge must not replay the failed prompt. + this.markModelErrorSupersededByUserTurn(); + } + const requestedModel = batch.mode.model === null ? this.defaultBackendModel : batch.mode.model; @@ -610,6 +711,17 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { await applyCursorAcpMode(backend, acpSessionId, batch.mode.permissionMode as PermissionMode); this.applyDisplayMode(batch.mode.permissionMode as PermissionMode); + this.lastUserMessage = batch.message; + this.lastTurnMode = batch.mode; + + // applyLiveModel / applyCursorAcpMode can take time after dequeue. + if (this.bridgingForEventId !== null && this.session.queue.hasPendingNonBridgeTurn()) { + this.bridgingForEventId = null; + this.bridgingSource = null; + this.markModelErrorSupersededByUserTurn(); + continue; + } + const specialCommand = parseCursorSpecialCommand(batch.message); if (specialCommand.type === 'pass-through') { messageBuffer.addMessage(cursorPassThroughStatusMessage(specialCommand.command), 'status'); @@ -624,13 +736,26 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { }]; session.onThinkingChange(true); + this.turnHasModelError = false; + this.userAbortRequested = false; + this.lastAssistantText = null; + this.pendingTextFailure = null; + this.pendingStderrFailure = null; this.promptInFlight = true; session.client.updateAgentState?.((state) => ({ ...state, steeringActive: true })); this.activePromptModeHash = batch.hash; - try { - this.promptInFlight = true; - this.userAbortRequested = false; + const settleFailure = (error: unknown) => { + const rpcFailure = classifyAcpRpcRejection(error); + const genericRpcFailure = rpcFailure !== null && ( + rpcFailure.kind === 'transport_closed' + || rpcFailure.kind === 'agent_crashed' + || rpcFailure.kind === 'prompt_failed' + ); + return genericRpcFailure + ? this.pendingStderrFailure ?? rpcFailure ?? this.pendingTextFailure + : rpcFailure ?? this.pendingStderrFailure ?? this.pendingTextFailure; + }; for (let retryAttempt = 0; retryAttempt <= CURSOR_AUTO_RETRY_LIMIT; retryAttempt += 1) { this.pendingRetryableError = null; this.pendingRetryableFromStderr = false; @@ -638,15 +763,30 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { this.attemptProducedToolActivity = false; let turnCompleted = false; try { - await backend.prompt(acpSessionId, promptContent, (message) => { + const sent = await backend.prompt(acpSessionId, promptContent, (message) => { if (message.type === 'turn_complete') turnCompleted = true; this.handleAgentMessage(message); + }, { + shouldSend: () => !(this.bridgingForEventId !== null + && this.session.queue.hasPendingNonBridgeTurn()) }); + if (sent === false) { + this.bridgingForEventId = null; + this.bridgingSource = null; + this.markModelErrorSupersededByUserTurn(); + break; + } if (this.userAbortRequested) break; if (turnCompleted && this.pendingRetryableFromStderr && !this.pendingInlineRetryableError) { this.pendingRetryableError = null; } if (!this.pendingRetryableError) { + const settled = this.pendingStderrFailure ?? this.pendingTextFailure; + if (settled && !this.turnHasModelError) { + this.recordModelError(settled); + } + this.pendingStderrFailure = null; + this.pendingTextFailure = null; void backend.refreshSessionInfo(acpSessionId, session.path); break; } @@ -655,6 +795,10 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { if (this.userAbortRequested) break; if (!isRetryableCursorError(error)) { this.surfacePromptFailure(error instanceof Error ? error.message : String(error)); + const failure = settleFailure(error); + this.pendingStderrFailure = null; + this.pendingTextFailure = null; + if (failure) this.recordModelError(failure); break; } this.pendingRetryableError = error instanceof Error ? error.message : String(error); @@ -662,6 +806,12 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { if (this.attemptProducedToolActivity) { this.surfacePromptFailure('Cursor connection interrupted after tool activity; the prompt was not retried.'); + const failure = settleFailure( + this.pendingRetryableError ?? 'Cursor connection interrupted after tool activity' + ); + this.pendingStderrFailure = null; + this.pendingTextFailure = null; + if (failure) this.recordModelError(failure); break; } if (retryAttempt < CURSOR_AUTO_RETRY_LIMIT) { @@ -669,6 +819,12 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { continue; } this.surfacePromptFailure(`Cursor Agent failed after ${CURSOR_AUTO_RETRY_LIMIT} retries.`); + const exhaustedFailure = settleFailure( + this.pendingRetryableError ?? 'Cursor Agent failed after retries' + ); + this.pendingStderrFailure = null; + this.pendingTextFailure = null; + if (exhaustedFailure) this.recordModelError(exhaustedFailure); } } finally { this.promptInFlight = false; @@ -701,9 +857,22 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { this.pendingRetryableFromStderr = false; this.pendingInlineRetryableError = false; this.attemptProducedToolActivity = false; + this.pendingStderrFailure = null; + this.pendingTextFailure = null; session.onThinkingChange(false); await this.permissionAdapter?.cancelAll('Prompt finished'); await this.extensionAdapter?.cancelAll('Prompt finished'); + if ( + !this.userAbortRequested + && !this.turnHasModelError + && this.bridgingForEventId !== null + ) { + const eventId = this.bridgingForEventId; + const source = this.bridgingSource ?? 'manual'; + this.markModelErrorBridgeSucceeded(eventId, source); + this.bridgingForEventId = null; + this.bridgingSource = null; + } if (session.queue.size() === 0 && !this.shouldExit) { sendReady(); } @@ -724,6 +893,10 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { try { this.clearAbortHandlers(this.session.client.rpcHandlerManager); + this.session.client.rpcHandlerManager.registerHandler( + RPC_METHODS.BridgeModelError, + async () => ({ ok: false, reason: 'session_ended' }) + ); this.session.client.rpcHandlerManager.registerHandler(RPC_METHODS.SteerQueuedMessage, async () => ({ steered: false, error: 'Session ending' @@ -776,7 +949,18 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { } return; } + // Setup/load remap consumes "Cannot use this model" stderr without + // promoting to modelError. During an active prompt, the same + // signature must become model_not_found (mapper would otherwise + // be unreachable behind this early return). + const failure = mapAcpStderrToFailure(error); if (error.type === 'model_not_found' && extractCannotUseThisModelMessage(hint)) { + // Setup/load remap may reject a stale spawn model then succeed + // after remap — never promote that to a turn alert. Only defer + // during an active prompt so RPC precedence still wins. + if (this.promptInFlight && failure) { + this.pendingStderrFailure ??= failure; + } return; } const converted = convertAgentMessage({ type: 'error', message: error.message }); @@ -784,6 +968,22 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { session.sendAgentMessage(converted); } messageBuffer.addMessage(error.message, 'status'); + // STRUCTURAL signal: route typed stderr into the modelError pipeline + // (rate_limited / quota_exhausted / auth_failed / model_not_found) + // without text matching. Generic `unknown` stderr stays status-only — + // ACP treats stderr as logging, and the transport labels any + // "error"/"failed"/"exception" line as unknown. + // While prompt is in flight, defer so RPC rejection keeps precedence. + // Idle stderr can still surface a banner/notify, but must not be + // Bridgeable — stdout/stderr are independent, so a strong rate-limit + // line can arrive after a turn already succeeded. + if (failure) { + if (this.promptInFlight) { + this.pendingStderrFailure ??= failure; + } else { + this.recordModelError(failure, { bridgeable: false }); + } + } }); } @@ -862,6 +1062,7 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { switch (message.type) { case 'text': this.messageBuffer.addMessage(message.text, 'assistant'); + this.handleTextMessageClassification(message.text); break; case 'reasoning': break; @@ -906,6 +1107,364 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { this.messageBuffer.addMessage(message, 'status'); } + private handleTextMessageClassification(text: string): void { + // FALLBACK PATH ONLY — deferred until prompt settles so structural + // RPC / stderr can win. If a structural signal already classified + // this turn, keep lastAssistantText for priorAssistantClaimsDone. + if (this.turnHasModelError) { + this.lastAssistantText = text; + return; + } + const failure = classifyCursorAgentMessage(text); + if (failure) { + this.pendingTextFailure ??= failure; + } else { + this.lastAssistantText = text; + } + } + + /** + * Single source of truth for emitting modelError. Structural paths + * (RPC catch / stderr) record immediately; text fallback is deferred + * via pendingTextFailure until prompt settles, then flushed here. + * First recorded signal wins for the turn. + */ + private recordModelError( + failure: CursorAgentStreamFailure, + opts?: { bridgeable?: boolean } + ): void { + if (this.turnHasModelError) { + logger.debug( + `[cursor-acp] modelError already recorded for this turn, dropping ${failure.source}/${failure.kind}` + ); + return; + } + // Abort/Exit/Switch can settle as canceled *or* transport/process-close + // shapes (ACP process exited, WritableIterable closed, …). Never promote + // those into lastModelError / notify / auto-bridge after a deliberate stop. + if (this.userAbortRequested) { + logger.debug( + `[cursor-acp] dropping modelError after user abort kind=${failure.kind}` + ); + this.pendingTextFailure = null; + this.pendingStderrFailure = null; + return; + } + this.turnHasModelError = true; + this.pendingTextFailure = null; + this.pendingStderrFailure = null; + + const bridgedFailure = this.bridgingForEventId !== null; + if (bridgedFailure) { + this.bridgingForEventId = null; + this.bridgingSource = null; + } + + // A newer displayed error invalidates any not-yet-started bridge. + this.cancelPendingBridge(); + + // Same-message case: Cursor often appends `Error: T: ...` onto the + // assistant block that already claimed "Done." — lastAssistantText is + // still null because we classify before storing. Check failure.raw too. + const priorAssistantClaimsDone = (this.lastAssistantText !== null + && isCompletionClaim(this.lastAssistantText)) + || (failure.source === 'text' && isCompletionClaim(failure.raw)); + const rawSnippet = rawSnippetForFailure(failure); + const eventId = randomUUID(); + const atTs = Date.now(); + // Fail closed on silently truncated prompts — Bridge must replay exact text. + const fullMessage = this.lastUserMessage ?? ''; + const fitsBridgeLimit = fullMessage.length <= MAX_LAST_USER_MESSAGE_CHARS; + const isPassThroughCommand = parseCursorSpecialCommand(fullMessage).type === 'pass-through'; + const bridgeable = opts?.bridgeable !== false && fitsBridgeLimit && !isPassThroughCommand; + const lastUserMessage = bridgeable ? fullMessage : ''; + + logger.debug( + `[cursor-acp] modelError recorded source=${failure.source} kind=${failure.kind} transient=${failure.transient}${bridgedFailure ? ' (bridge failed)' : ''}${!bridgeable ? ' (not bridgeable)' : ''}` + ); + + this.lastRecordedModelError = { + eventId, + atTs, + kind: failure.kind, + transient: failure.transient, + rawSnippet, + priorAssistantClaimsDone, + lastUserMessage, + ...(bridgeable ? {} : { bridgeable: false }), + ...(bridgedFailure ? { retriedAndFailed: true } : {}) + }; + + this.session.client.updateMetadata((metadata) => ({ + ...metadata, + lastModelError: this.lastRecordedModelError! + })); + + this.session.sendSessionEvent({ + type: 'modelError', + kind: failure.kind, + transient: failure.transient, + rawSnippet, + priorAssistantClaimsDone + }); + + if ( + !bridgedFailure + && bridgeable + && failure.transient + && getAutoBridgeTransientModelErrors() + ) { + this.tryEnqueueModelErrorBridge('auto'); + } + } + + private async handleBridgeModelErrorRpc(payload: unknown): Promise<{ ok: boolean; reason?: string }> { + if (!payload || typeof payload !== 'object') { + return this.tryEnqueueModelErrorBridge('manual'); + } + + const record = payload as Record; + const snapshot = { + eventId: typeof record.eventId === 'string' ? record.eventId : undefined, + atTs: typeof record.atTs === 'number' ? record.atTs : undefined, + kind: typeof record.kind === 'string' ? record.kind : undefined, + rawSnippet: typeof record.rawSnippet === 'string' ? record.rawSnippet : undefined, + lastUserMessage: typeof record.lastUserMessage === 'string' ? record.lastUserMessage : undefined, + priorAssistantClaimsDone: record.priorAssistantClaimsDone === true, + transient: typeof record.transient === 'boolean' + ? record.transient + : (this.lastRecordedModelError?.transient ?? false), + bridgedForEventId: typeof record.bridgedForEventId === 'string' + ? record.bridgedForEventId + : undefined, + retriedAndFailed: record.retriedAndFailed === true, + supersededByUserTurn: record.supersededByUserTurn === true, + bridgeable: record.bridgeable === false ? false : undefined + }; + + if (snapshot.eventId !== undefined) { + // Bind to the displayed error — refuse if a newer local error won. + if ( + this.lastRecordedModelError + && this.lastRecordedModelError.eventId !== snapshot.eventId + ) { + return { ok: false, reason: 'model_error_changed' }; + } + // Merge hub snapshot into local gate state — never clobber + // bridgedForEventId / retriedAndFailed / supersededByUserTurn / + // bridgeable=false with undefined/false from a stale hub payload. + const gates = mergeBridgeGateFields(this.lastRecordedModelError, { + bridgedForEventId: snapshot.bridgedForEventId, + retriedAndFailed: snapshot.retriedAndFailed, + supersededByUserTurn: snapshot.supersededByUserTurn, + bridgeable: snapshot.bridgeable + }); + this.lastRecordedModelError = { + eventId: snapshot.eventId, + atTs: snapshot.atTs ?? this.lastRecordedModelError?.atTs ?? Date.now(), + kind: snapshot.kind ?? this.lastRecordedModelError?.kind ?? 'unknown', + rawSnippet: snapshot.rawSnippet ?? this.lastRecordedModelError?.rawSnippet ?? '', + priorAssistantClaimsDone: snapshot.priorAssistantClaimsDone, + lastUserMessage: snapshot.lastUserMessage + ?? this.lastRecordedModelError?.lastUserMessage + ?? this.lastUserMessage + ?? '', + transient: snapshot.transient, + bridgedForEventId: gates.bridgedForEventId, + retriedAndFailed: gates.retriedAndFailed, + supersededByUserTurn: gates.supersededByUserTurn, + bridgeable: gates.bridgeable + }; + } + + return this.tryEnqueueModelErrorBridge('manual'); + } + + private tryEnqueueModelErrorBridge(source: 'auto' | 'manual'): { ok: boolean; reason?: string } { + const metadataError = this.lastRecordedModelError; + + if (!metadataError) { + return { ok: false, reason: 'no_model_error' }; + } + + // Manual Bridge during a newer in-flight turn would front-queue a stale + // retry that still runs if that turn succeeds (no superseding modelError). + // Auto-bridge may still fire while settling the failed turn itself. + if (source === 'manual' && this.promptInFlight) { + return { ok: false, reason: 'prompt_in_flight' }; + } + + if (metadataError.supersededByUserTurn) { + return { ok: false, reason: 'superseded_by_newer_turn' }; + } + + // Fail closed while a bridge for this eventId is pending or active. + if ( + this.bridgingForEventId === metadataError.eventId + || this.pendingBridgeEventId === metadataError.eventId + ) { + return { ok: false, reason: 'not_bridgeable' }; + } + + const bridgeInput = { + kind: metadataError.kind, + rawSnippet: metadataError.rawSnippet, + priorAssistantClaimsDone: metadataError.priorAssistantClaimsDone, + lastUserMessage: metadataError.lastUserMessage ?? this.lastUserMessage ?? '' + }; + + if (!bridgeInput.lastUserMessage.trim()) { + return { ok: false, reason: 'missing_last_user_message' }; + } + + if (!canBridgeModelError({ + transient: metadataError.transient, + eventId: metadataError.eventId, + bridgedForEventId: metadataError.bridgedForEventId, + retriedAndFailed: metadataError.retriedAndFailed, + supersededByUserTurn: metadataError.supersededByUserTurn, + bridgeable: metadataError.bridgeable + })) { + return { ok: false, reason: 'not_bridgeable' }; + } + + const prompt = buildModelErrorBridgePrompt({ + kind: bridgeInput.kind, + rawSnippet: bridgeInput.rawSnippet, + lastUserMessage: bridgeInput.lastUserMessage, + priorAssistantClaimsDone: bridgeInput.priorAssistantClaimsDone + }); + + const bridgedEventId = metadataError.eventId; + + // Never overtake newer user intent already waiting in the queue. + // supersededByUserTurn is only stamped when a normal batch starts; if we + // unshift Bridge ahead of that batch we replay the old prompt first. + if (this.session.queue.hasPendingNonBridgeTurn()) { + this.markModelErrorSupersededByUserTurn(); + return { ok: false, reason: 'superseded_by_newer_turn' }; + } + + // Drop any stale pending bridge for a different event before enqueue. + if (this.pendingBridgeEventId && this.pendingBridgeEventId !== bridgedEventId) { + this.cancelPendingBridge(); + } + + // Attribution arms when the bridge batch starts, not at enqueue. + this.pendingBridgeEventId = bridgedEventId; + this.pendingBridgeSource = source; + + const mode = this.lastTurnMode ?? { + permissionMode: this.session.getPermissionMode() as PermissionMode, + model: this.currentBackendModel ?? this.session.model ?? undefined + }; + + // Front of queue only when no newer user turn is waiting. + // Provenance is queue-owned (`internal`). Omit synthetic localId so + // dequeue cannot ACK a client prompt that reused `bridge:${eventId}`. + this.session.queue.unshiftIsolated( + prompt, + mode, + undefined, + { kind: 'model-error-bridge', eventId: bridgedEventId } + ); + logger.debug(`[cursor-acp] modelError bridge enqueued for eventId=${bridgedEventId} source=${source}`); + + return { ok: true }; + } + + /** Drop not-yet-started bridge queue entries and clear the retry gate. */ + private cancelPendingBridge(): void { + const eventId = this.pendingBridgeEventId; + if (eventId) { + this.session.queue.cancelModelErrorBridge(eventId); + } else { + // Gate/queue desync: scrub any pending queue-owned bridge rows. + for (const item of this.session.queue.queue) { + if (item.internal?.kind === 'model-error-bridge') { + this.session.queue.cancelModelErrorBridge(item.internal.eventId); + } + } + } + this.pendingBridgeEventId = null; + this.pendingBridgeSource = null; + } + + private markModelErrorBridgeSucceeded(eventId: string, source: 'auto' | 'manual'): void { + const current = this.lastRecordedModelError; + if (!current || current.eventId !== eventId) { + return; + } + this.lastRecordedModelError = { + ...current, + bridgedForEventId: eventId + }; + this.session.client.updateMetadata((metadata) => { + const err = metadata.lastModelError; + if (!err || err.eventId !== eventId) { + return metadata; + } + return { + ...metadata, + lastModelError: { + ...err, + bridgedForEventId: eventId + } + }; + }); + // Chat-visible recovery marker only. Not an AGENT_NOTIFY_SUMMARY. + this.session.sendSessionEvent({ + type: 'modelErrorBridged', + kind: current.kind, + auto: source === 'auto', + eventId + }); + } + + /** Durable invalidation after a newer normal (non-bridge) turn starts. */ + private markModelErrorSupersededByUserTurn(): void { + const current = this.lastRecordedModelError; + if (!current || current.supersededByUserTurn) { + return; + } + this.lastRecordedModelError = { + ...current, + supersededByUserTurn: true + }; + this.session.client.updateMetadata((metadata) => { + const err = metadata.lastModelError; + if (!err || err.eventId !== current.eventId) { + return metadata; + } + return { + ...metadata, + lastModelError: { + ...err, + supersededByUserTurn: true + } + }; + }); + } + + /** Load durable lastModelError from hub metadata after CLI restart/resume. */ + private hydrateLastRecordedModelErrorFromMetadata(): void { + if (this.lastRecordedModelError) { + return; + } + const getMetadata = this.session.client.getMetadata; + if (typeof getMetadata !== 'function') { + return; + } + const persistedError = getMetadata.call(this.session.client)?.lastModelError; + if (!persistedError || typeof persistedError.eventId !== 'string') { + return; + } + this.lastRecordedModelError = { + ...persistedError, + lastUserMessage: persistedError.lastUserMessage ?? '' + }; + } + private installLiveSessionConfigSync( backend: AcpSdkBackend, acpSessionId: string, @@ -1086,7 +1645,13 @@ class CursorAcpRemoteLauncher extends RemoteLauncherBase { } private async handleAbort(): Promise { + // Mark + clear bridge gates BEFORE any await. Otherwise a settling + // bridge prompt can race markModelErrorBridgeSucceeded and falsely + // persist bridgedForEventId / modelErrorBridged after the operator canceled. this.userAbortRequested = true; + this.cancelPendingBridge(); + this.bridgingForEventId = null; + this.bridgingSource = null; const backend = this.backend; const sessionId = this.acpSessionId ?? this.session.sessionId; if (backend && sessionId) { diff --git a/cli/src/cursor/cursorAgentMessageClassifier.test.ts b/cli/src/cursor/cursorAgentMessageClassifier.test.ts new file mode 100644 index 0000000000..e3bcf7ee53 --- /dev/null +++ b/cli/src/cursor/cursorAgentMessageClassifier.test.ts @@ -0,0 +1,557 @@ +import { describe, it, expect } from 'vitest' +import { + classifyAcpRpcRejection, + classifyCursorAgentMessage, + isCompletionClaim, + mapAcpStderrToFailure, + rawSnippetForFailure +} from './cursorAgentMessageClassifier' + +describe('classifyCursorAgentMessage', () => { + it('classifies resource_exhausted', () => { + const result = classifyCursorAgentMessage('Error: T: [resource_exhausted] quota exceeded') + expect(result).not.toBeNull() + expect(result?.kind).toBe('quota_exhausted') + expect(result?.transient).toBe(false) + }) + + it('classifies canceled', () => { + const result = classifyCursorAgentMessage('Error: T: [canceled] the request was cancelled') + expect(result).not.toBeNull() + expect(result?.kind).toBe('canceled') + expect(result?.transient).toBe(true) + }) + + it('classifies deadline_exceeded', () => { + const result = classifyCursorAgentMessage('Error: T: [deadline_exceeded]') + expect(result).not.toBeNull() + expect(result?.kind).toBe('deadline_exceeded') + expect(result?.transient).toBe(true) + }) + + it('classifies unavailable', () => { + const result = classifyCursorAgentMessage('Error: T: [unavailable] service is down') + expect(result).not.toBeNull() + expect(result?.kind).toBe('unavailable') + expect(result?.transient).toBe(true) + }) + + it('classifies connection_stalled', () => { + const result = classifyCursorAgentMessage('Error: T: Connection stalled after 30s') + expect(result).not.toBeNull() + expect(result?.kind).toBe('connection_stalled') + expect(result?.transient).toBe(true) + }) + + it('classifies context_window', () => { + const result = classifyCursorAgentMessage( + 'Gemini prompt failed: token count exceeds the model limit' + ) + expect(result).not.toBeNull() + expect(result?.kind).toBe('context_window') + expect(result?.transient).toBe(false) + }) + + it('classifies capacity_exhausted', () => { + const result = classifyCursorAgentMessage( + 'Gemini prompt failed: you have exhausted your capacity for today' + ) + expect(result).not.toBeNull() + expect(result?.kind).toBe('capacity_exhausted') + expect(result?.transient).toBe(false) + }) + + it('classifies unknown_t_prefix for unrecognised Error: T: variants', () => { + const result = classifyCursorAgentMessage('Error: T: [some_new_error] weird thing happened') + expect(result).not.toBeNull() + expect(result?.kind).toBe('unknown_t_prefix') + expect(result?.transient).toBe(false) + }) + + it('preserves raw text', () => { + const raw = 'Error: T: [canceled] something something' + const result = classifyCursorAgentMessage(raw) + expect(result?.raw).toBe(raw) + }) + + it('returns null for benign messages', () => { + expect(classifyCursorAgentMessage("Here's the diff:")).toBeNull() + expect(classifyCursorAgentMessage('Done.')).toBeNull() + expect(classifyCursorAgentMessage('All done.')).toBeNull() + expect(classifyCursorAgentMessage('I found 3 files.')).toBeNull() + expect(classifyCursorAgentMessage('Successfully updated the config.')).toBeNull() + }) + + it('returns null for empty string', () => { + expect(classifyCursorAgentMessage('')).toBeNull() + }) + + it('is case-insensitive for Error: T: patterns', () => { + const result = classifyCursorAgentMessage('error: t: [resource_exhausted]') + expect(result?.kind).toBe('quota_exhausted') + }) + + it('does not match Error: T: patterns in the middle of text', () => { + // These patterns are anchored at the start + expect(classifyCursorAgentMessage('Partial text before Error: T: [canceled]')).toBeNull() + }) + + it('does not classify prose that describes the pattern', () => { + // Regression: 2026-06-12 self-own. The classifier matched an + // assistant message that *described* the patterns it looks for, + // because the original Gemini patterns used unanchored "contains". + // Real cursor-agent error emits come as the whole message body, + // not embedded in narrative prose. Anchored patterns reject prose. + const proseDescribingPatterns = + "Yes. In the soup since 23:51:24 BST.\n\n" + + "Triggers on:\n" + + " - Error: T: [resource_exhausted]\n" + + " - Error: T: Connection stalled\n" + + " - Gemini prompt failed: .*token count exceeds\n" + + " - Gemini prompt failed: .*exhausted your capacity\n" + expect(classifyCursorAgentMessage(proseDescribingPatterns)).toBeNull() + + // Same idea, single line embedding the literal description. + expect( + classifyCursorAgentMessage( + 'The classifier looks for "Gemini prompt failed: .*token count exceeds" specifically.' + ) + ).toBeNull() + }) + + it('ignores Error: T: / RetriableError examples inside markdown fences', () => { + const fencedQuota = + 'Here is what the wire looks like:\n' + + '```\n' + + 'Error: T: [resource_exhausted] capacity exceeded\n' + + '```\n' + + 'Do not treat that quote as a live failure.' + expect(classifyCursorAgentMessage(fencedQuota)).toBeNull() + + const fencedRetriable = + 'Example:\n' + + '~~~\n' + + 'Error: RetriableError: Connection stalled after 30s\n' + + '~~~\n' + expect(classifyCursorAgentMessage(fencedRetriable)).toBeNull() + + // Four-backtick outer fence quoting a triple-backtick example must + // not close early on the inner ``` and leak Error: T: into classify. + const nestedFence = + 'Docs sample:\n' + + '````markdown\n' + + '```\n' + + 'Error: T: [resource_exhausted] capacity exceeded\n' + + '```\n' + + '````\n' + expect(classifyCursorAgentMessage(nestedFence)).toBeNull() + + // Unclosed fence + failure tail must still classify (truncated generation). + const truncatedFence = + '```ts\n' + + 'partial code\n' + + '\n' + + 'Error: T: [resource_exhausted] Error\n' + expect(classifyCursorAgentMessage(truncatedFence)?.kind).toBe('quota_exhausted') + + // Same-length marker WITH info string is not a closer — keep fenced. + const fakeCloser = + '```\n' + + 'quoted example\n' + + '```ts\n' + + 'Error: T: [resource_exhausted] capacity exceeded\n' + + '```\n' + expect(classifyCursorAgentMessage(fakeCloser)).toBeNull() + + // CRLF closers must still close (otherwise fenced Error: T: leaks). + const crlfFence = + '```\r\n' + + 'Error: T: [resource_exhausted] capacity exceeded\r\n' + + '```\r\n' + expect(classifyCursorAgentMessage(crlfFence)).toBeNull() + + // CommonMark indented code (4+ spaces, or 0–3 spaces + tab) is not live. + expect( + classifyCursorAgentMessage(' Error: T: [resource_exhausted] capacity exceeded') + ).toBeNull() + expect( + classifyCursorAgentMessage('\tError: RetriableError: Connection stalled after 30s') + ).toBeNull() + expect( + classifyCursorAgentMessage(' \tError: T: [resource_exhausted] capacity exceeded') + ).toBeNull() + }) + + it('still classifies real Gemini errors when they ARE the message body', () => { + // Whitespace prefix is OK (trimStart handles it) but prose prefix is not. + expect( + classifyCursorAgentMessage( + ' Gemini prompt failed: token count exceeds limit' + )?.kind + ).toBe('context_window') + expect( + classifyCursorAgentMessage( + 'Gemini prompt failed: you have exhausted your capacity' + )?.kind + ).toBe('capacity_exhausted') + }) + + it('classifies Error: T: patterns with leading whitespace (real wire format)', () => { + // Regression: 2026-06-12 session b52b9117. Cursor ACP transport + // emitted "\n\nError: T: WritableIterable is closed" — leading + // newlines made the unanchored-tolerant `^Error: T:` regex miss + // it because JS `^` matches start of string, not start of line + // (without the `m` flag). trimStart() before the test fixes it. + const realWireFormat = '\n\nError: T: WritableIterable is closed' + const result = classifyCursorAgentMessage(realWireFormat) + expect(result).not.toBeNull() + expect(result?.kind).toBe('unknown_t_prefix') + expect(result?.transient).toBe(false) + // Raw text preserved as-is (leading newlines included) so the + // banner can show the operator exactly what arrived. + expect(result?.raw).toBe(realWireFormat) + }) + + it('classifies leading-whitespace variants of all Error: T: kinds', () => { + expect(classifyCursorAgentMessage('\n\nError: T: [resource_exhausted]')?.kind).toBe('quota_exhausted') + expect(classifyCursorAgentMessage(' Error: T: [canceled]')?.kind).toBe('canceled') + // Tab-indented = CommonMark code (stripped). Light space pad is wire OK. + expect(classifyCursorAgentMessage(' Error: T: [deadline_exceeded]')?.kind).toBe('deadline_exceeded') + expect(classifyCursorAgentMessage('\n\n Error: T: [unavailable]')?.kind).toBe('unavailable') + expect(classifyCursorAgentMessage('\nError: T: Connection stalled after 30s')?.kind).toBe('connection_stalled') + }) + + it('classifies RetriableError prefix from cursor session (real session 0e04ebe7)', () => { + // Regression: 2026-06-20 session 0e04ebe7 ("teams structure"). + // Cursor ACP session (metadata.flavor=cursor) but HAPI persists + // agent text in a codex-shaped envelope via convertAgentMessage — + // that is NOT the Codex runner. The inline error used RetriableError + // instead of Error: T: and was followed immediately by ready. + const realWireFormat = '\n\nError: RetriableError: [canceled] http/2 stream closed with error code CANCEL (0x8)' + const result = classifyCursorAgentMessage(realWireFormat) + expect(result).not.toBeNull() + expect(result?.kind).toBe('canceled') + expect(result?.transient).toBe(true) + expect(result?.source).toBe('text') + }) + + it('marks RetriableError resource_exhausted as transient for auto-bridge', () => { + // Operator ask 2026-07-14: Cursor emits this exact wire string. + // It already classified as quota_exhausted, but with transient:false — + // auto-bridge only retries transient:true. RetriableError means the + // agent labeled the failure retriable; keep Error: T: / stderr quota + // paths non-transient. + const exactWire = 'Error: RetriableError: [resource_exhausted] Error' + const result = classifyCursorAgentMessage(exactWire) + expect(result).not.toBeNull() + expect(result?.kind).toBe('quota_exhausted') + expect(result?.transient).toBe(true) + expect(result?.source).toBe('text') + expect(result?.raw).toBe(exactWire) + }) + + it('classifies error appended to in-flight agent text (real session e7d9b44b)', () => { + // Regression: 2026-06-13 session e7d9b44b. cursor-agent appended + // a gRPC stringification to the END of a normal narrative output + // rather than rejecting the prompt. The structural signals + // (RPC rejection / stderr) didn't fire because the agent never + // crashed - it just dumped the error into its own text stream. + // Start-of-string anchor missed it (text starts with prose); + // multiline `^` (start-of-line after the `\n\n`) catches it. + const realWireFormat = + "Three of the four hit Codex's usage limit (#151, #153, #155) " + + "- no code review delivered. Only #157 actually got reviewed. " + + "Let me pull the inline comments to see Codex's specific suggestions:\n\n" + + "Error: T: [resource_exhausted] Error" + const result = classifyCursorAgentMessage(realWireFormat) + expect(result).not.toBeNull() + expect(result?.kind).toBe('quota_exhausted') + expect(result?.transient).toBe(false) + }) + + it('still rejects bullet-listed pattern descriptions (no false positive)', () => { + // Lines indented with whitespace+dash do NOT start with "Error:" - + // multiline `^` requires the line to literally begin with the + // pattern. Self-own from 2026-06-12 stays prevented. + const proseDescribingPatterns = + "Yes. In the soup since 23:51:24 BST.\n\n" + + "Triggers on:\n" + + " - Error: T: [resource_exhausted]\n" + + " - Error: T: Connection stalled\n" + + " - Gemini prompt failed: .*token count exceeds\n" + + " - Gemini prompt failed: .*exhausted your capacity\n" + expect(classifyCursorAgentMessage(proseDescribingPatterns)).toBeNull() + }) + + it('catches all gRPC kinds when appended after prose+newlines', () => { + const prefix = "Working on the task. Got partial results before failure:\n\n" + expect(classifyCursorAgentMessage(prefix + 'Error: T: [resource_exhausted] Error')?.kind).toBe('quota_exhausted') + expect(classifyCursorAgentMessage(prefix + 'Error: T: [canceled] Operation aborted')?.kind).toBe('canceled') + expect(classifyCursorAgentMessage(prefix + 'Error: T: [deadline_exceeded]')?.kind).toBe('deadline_exceeded') + expect(classifyCursorAgentMessage(prefix + 'Error: T: [unavailable] Service down')?.kind).toBe('unavailable') + expect(classifyCursorAgentMessage(prefix + 'Error: T: Connection stalled')?.kind).toBe('connection_stalled') + expect(classifyCursorAgentMessage(prefix + 'Error: T: WritableIterable is closed')?.kind).toBe('unknown_t_prefix') + expect(classifyCursorAgentMessage(prefix + 'Gemini prompt failed: token count exceeds 1M')?.kind).toBe('context_window') + expect(classifyCursorAgentMessage(prefix + 'Gemini prompt failed: exhausted your capacity')?.kind).toBe('capacity_exhausted') + }) + + it("tags text-classifier results with source='text'", () => { + const result = classifyCursorAgentMessage('Error: T: [canceled]') + expect(result?.source).toBe('text') + }) +}) + +describe('mapAcpStderrToFailure (structural stderr signal)', () => { + it('maps rate_limit -> rate_limited (transient)', () => { + const out = mapAcpStderrToFailure({ type: 'rate_limit', raw: 'status 429 ratelimitexceeded' }) + expect(out).not.toBeNull() + expect(out!.kind).toBe('rate_limited') + expect(out!.transient).toBe(true) + expect(out!.source).toBe('stderr') + expect(out!.raw).toBe('status 429 ratelimitexceeded') + }) + + it('maps quota_exceeded -> quota_exhausted (non-transient)', () => { + const out = mapAcpStderrToFailure({ type: 'quota_exceeded', raw: 'resource exhausted' }) + expect(out).not.toBeNull() + expect(out!.kind).toBe('quota_exhausted') + expect(out!.transient).toBe(false) + expect(out!.source).toBe('stderr') + }) + + it('maps authentication -> auth_failed (non-transient)', () => { + const out = mapAcpStderrToFailure({ type: 'authentication', raw: 'status 401 unauthenticated' }) + expect(out).not.toBeNull() + expect(out!.kind).toBe('auth_failed') + expect(out!.transient).toBe(false) + expect(out!.source).toBe('stderr') + }) + + it('maps model_not_found -> model_not_found (non-transient)', () => { + const out = mapAcpStderrToFailure({ type: 'model_not_found', raw: 'status 404 model not found' }) + expect(out).not.toBeNull() + expect(out!.kind).toBe('model_not_found') + expect(out!.transient).toBe(false) + }) + + it('maps Cursor Cannot use this model rejection', () => { + const out = mapAcpStderrToFailure({ + type: 'model_not_found', + raw: 'Cannot use this model: grok-4.5[fast=true]. Available models: auto' + }) + expect(out?.kind).toBe('model_not_found') + }) + + it('ignores unknown stderr (status-only; do not promote to modelError)', () => { + // Transport emits type:unknown for any stderr containing error/failed/ + // exception — too broad for urgent model-error alerts. + const out = mapAcpStderrToFailure({ type: 'unknown', raw: 'unexpected exception in agent' }) + expect(out).toBeNull() + }) + + it('ignores weak typed stderr that only matches transport substrings', () => { + expect(mapAcpStderrToFailure({ + type: 'authentication', + raw: 'authentication provider initialized' + })).toBeNull() + expect(mapAcpStderrToFailure({ + type: 'rate_limit', + raw: 'checking rate limit configuration' + })).toBeNull() + expect(mapAcpStderrToFailure({ + type: 'quota_exceeded', + raw: 'quota tracker warmed up' + })).toBeNull() + expect(mapAcpStderrToFailure({ + type: 'model_not_found', + raw: 'catalog refresh skipped for unknown models' + })).toBeNull() + expect(mapAcpStderrToFailure({ + type: 'authentication', + raw: 'failed to read config: permission denied' + })).toBeNull() + expect(mapAcpStderrToFailure({ + type: 'model_not_found', + raw: 'plugin cache entry not_found' + })).toBeNull() + }) +}) + +describe('classifyAcpRpcRejection (structural RPC signal)', () => { + it('classifies WritableIterable is closed -> transport_closed (non-bridgeable)', () => { + // Real session b52b9117: the agent rejected sendRequest with this + // exact message after the writable side of the ACP transport died. + const err = new Error('WritableIterable is closed') + const out = classifyAcpRpcRejection(err) + expect(out).not.toBeNull() + expect(out?.kind).toBe('transport_closed') + expect(out?.transient).toBe(false) + expect(out?.source).toBe('rpc') + }) + + it('classifies ACP transport closed -> transport_closed (non-bridgeable)', () => { + const err = new Error('ACP transport is closed') + const out = classifyAcpRpcRejection(err) + expect(out?.kind).toBe('transport_closed') + expect(out?.transient).toBe(false) + }) + + it('classifies process exit -> transport_closed (rejectAllPending pathway)', () => { + // markClosed wraps the process-exit message in a new Error; pending + // sendRequests reject with that error. + const err = new Error('ACP process exited (code=137, signal=SIGKILL)') + const out = classifyAcpRpcRejection(err) + expect(out?.kind).toBe('transport_closed') + expect(out?.transient).toBe(false) + }) + + it('classifies spawn failure -> agent_crashed (non-bridgeable)', () => { + const err = new Error('Failed to spawn cursor-agent: ENOENT. Is it installed and on PATH?') + const out = classifyAcpRpcRejection(err) + expect(out?.kind).toBe('agent_crashed') + expect(out?.transient).toBe(false) + }) + + it('classifies request timeout -> rpc_timeout', () => { + const err = new Error("ACP request 'session/prompt' timed out after 120000ms") + const out = classifyAcpRpcRejection(err) + expect(out?.kind).toBe('rpc_timeout') + expect(out?.transient).toBe(true) + }) + + it('returns null for user cancellations (NOT model errors)', () => { + expect(classifyAcpRpcRejection(new Error('Aborted by user'))).toBeNull() + expect(classifyAcpRpcRejection(new Error('user cancelled the request'))).toBeNull() + expect(classifyAcpRpcRejection(new Error('user canceled the request'))).toBeNull() + }) + + it('passes through gRPC-status RPC errors with source=rpc', () => { + // Cursor-agent sometimes returns the gRPC status as a JSON-RPC + // error.message rather than emitting it as a text message. + const err = new Error('Error: T: [resource_exhausted] quota exceeded') + const out = classifyAcpRpcRejection(err) + expect(out?.kind).toBe('quota_exhausted') + expect(out?.transient).toBe(false) + // The structural source overrides text classification's source tag. + expect(out?.source).toBe('rpc') + }) + + it('classifies [canceled] Operation aborted via text path before user-abort filter', () => { + // Bare "aborted" must not null out real gRPC canceled shapes that + // happen to include that word in the status message. + const err = new Error('Error: T: [canceled] Operation aborted') + const out = classifyAcpRpcRejection(err) + expect(out).not.toBeNull() + expect(out?.kind).toBe('canceled') + expect(out?.source).toBe('rpc') + }) + + it('classifies plain Rate limit exceeded RPC messages as transient', () => { + const out = classifyAcpRpcRejection(new Error('Rate limit exceeded')) + expect(out?.kind).toBe('rate_limited') + expect(out?.transient).toBe(true) + expect(out?.source).toBe('rpc') + }) + + it('classifies status 429 RPC messages as transient', () => { + const out = classifyAcpRpcRejection(new Error('status 429 ratelimitexceeded')) + expect(out?.kind).toBe('rate_limited') + expect(out?.transient).toBe(true) + expect(out?.source).toBe('rpc') + }) + + it('falls through to prompt_failed for unrecognised RPC errors', () => { + const err = new Error('Some weird internal SDK assertion failed') + const out = classifyAcpRpcRejection(err) + expect(out?.kind).toBe('prompt_failed') + expect(out?.transient).toBe(false) + expect(out?.source).toBe('rpc') + }) + + it('handles non-Error rejection values', () => { + const out = classifyAcpRpcRejection('plain string rejection') + expect(out?.kind).toBe('prompt_failed') + expect(out?.raw).toBe('plain string rejection') + }) +}) + +describe('isCompletionClaim', () => { + it('matches Done', () => expect(isCompletionClaim('Done.')).toBe(true)) + it('matches All done', () => expect(isCompletionClaim('All done. The PR is filed.')).toBe(true)) + it('matches Committed', () => expect(isCompletionClaim('Committed all changes.')).toBe(true)) + it('matches Successfully completed', () => expect(isCompletionClaim('Successfully completed the migration.')).toBe(true)) + it('matches Successfully fixed', () => expect(isCompletionClaim('Successfully fixed the bug.')).toBe(true)) + it('matches Fixed', () => expect(isCompletionClaim('Fixed the bug.')).toBe(true)) + it('matches Complete', () => expect(isCompletionClaim('Complete.')).toBe(true)) + it('matches Completed', () => expect(isCompletionClaim('Completed.')).toBe(true)) + it('is case-insensitive', () => { + expect(isCompletionClaim('DONE everything')).toBe(true) + expect(isCompletionClaim('all done')).toBe(true) + }) + it('matches Done when Error: T is appended to the same block', () => { + // Cursor free-text path: completion claim + gRPC error in one message. + expect(isCompletionClaim( + 'Done.\n\nError: T: [canceled] Operation aborted' + )).toBe(true) + }) + it('does not match non-completion phrases', () => { + expect(isCompletionClaim("Here's the plan")).toBe(false) + expect(isCompletionClaim("I'm working on it")).toBe(false) + }) + it('does not match Completely… as a completion claim', () => { + expect(isCompletionClaim('Completely unable to finish the task.')).toBe(false) + }) + it('does not match Successfully reproduced… as a completion claim', () => { + expect(isCompletionClaim('Successfully reproduced the issue, but the fix is pending.')).toBe(false) + }) + it('handles empty string', () => { + expect(isCompletionClaim('')).toBe(false) + }) +}) + +describe('rawSnippetForFailure', () => { + it('slices from Error: T marker when prose exceeds 400 chars', () => { + const prose = 'A'.repeat(450) + const failure = { + kind: 'canceled' as const, + transient: true, + raw: `${prose}\n\nError: T: [canceled] Operation aborted`, + source: 'text' as const + } + const snippet = rawSnippetForFailure(failure) + expect(snippet.startsWith('Error: T:')).toBe(true) + expect(snippet).toContain('[canceled]') + expect(snippet.length).toBeLessThanOrEqual(400) + }) + + it('keeps start-of-string for non-text sources', () => { + const failure = { + kind: 'rate_limited' as const, + transient: true, + raw: 'status 429 ratelimitexceeded', + source: 'stderr' as const + } + expect(rawSnippetForFailure(failure)).toBe('status 429 ratelimitexceeded') + }) + + it('prefers the last Error: T marker over an earlier fenced example', () => { + const filler = 'x'.repeat(450) + const raw = [ + 'Here is an example of a failure:', + '```', + 'Error: T: [canceled] example only', + '```', + filler, + 'Error: T: [resource_exhausted] real failure' + ].join('\n') + const failure = { + kind: 'quota_exhausted' as const, + transient: true, + raw, + source: 'text' as const + } + const snippet = rawSnippetForFailure(failure) + expect(snippet.startsWith('Error: T: [resource_exhausted]')).toBe(true) + expect(snippet).not.toContain('example only') + }) +}) diff --git a/cli/src/cursor/cursorAgentMessageClassifier.ts b/cli/src/cursor/cursorAgentMessageClassifier.ts new file mode 100644 index 0000000000..561dcadf63 --- /dev/null +++ b/cli/src/cursor/cursorAgentMessageClassifier.ts @@ -0,0 +1,390 @@ +/** + * Structural-first classifier for cursor-agent failure signals. + * + * Sources, in priority order: + * 1. JSON-RPC rejection on session/prompt (structural: thrown Error from + * `@zed-industries/agent-client-protocol` SDK or transport close). + * 2. Stderr lines parsed by AcpStdioTransport into typed AcpStderrError. + * 3. Text in agent/message events (last-resort fallback, brittle by nature + * because cursor-agent stringifies internal errors as plain prose). + * + * The text fallback (1)-(3) only fires when no structural signal already + * classified the turn. Each source carries a `source` tag so the operator + * (and tests) can tell where the classification came from. + */ +export type CursorAgentStreamFailureKind = + // --- text/agent-message classifier kinds (legacy, fallback) --- + | 'quota_exhausted' + | 'canceled' + | 'deadline_exceeded' + | 'unavailable' + | 'connection_stalled' + | 'context_window' + | 'capacity_exhausted' + | 'unknown_t_prefix' + // --- structural kinds --- + | 'transport_closed' // ACP transport closed mid-turn (WritableIterable, etc.) + | 'rpc_timeout' // sendRequest timeout + | 'rpc_error' // JSON-RPC error response from the agent + | 'agent_crashed' // process exit during in-flight prompt + | 'rate_limited' // stderr-derived rate limit + | 'auth_failed' // stderr-derived authentication failure + | 'model_not_found' // stderr-derived model-not-found + | 'unknown_stderr' // stderr line classified as error but not typed + | 'prompt_failed' // catch-all for prompt rejections + +export type CursorAgentStreamFailureSource = 'rpc' | 'stderr' | 'text' + +export type CursorAgentStreamFailure = { + kind: CursorAgentStreamFailureKind + transient: boolean + raw: string + source: CursorAgentStreamFailureSource +} + +type Pattern = { + test: (text: string) => boolean + kind: CursorAgentStreamFailureKind + transient: boolean +} + +// Patterns are anchored to the start of A LINE (multiline `m` flag), not +// just start-of-string. Three real cursor-agent failure shapes drove this: +// +// 1. Whole-body emit (session b52b9117, 2026-06-12): +// "\n\nError: T: WritableIterable is closed" +// -- start of message, after leading whitespace. +// +// 2. Mid-stream append (session e7d9b44b, 2026-06-13): +// "Three of the four hit Codex's usage limit (#151, #153, #155) - +// no code review delivered. Only #157 actually got reviewed. +// Let me pull the inline comments to see Codex's specific +// suggestions:\n\nError: T: [resource_exhausted] Error" +// -- cursor-agent appended the gRPC status to the END of an in-flight +// text stream rather than rejecting the prompt. Start-of-line `m` +// anchor catches this: the `\n\n` separator means `Error: T:` is at +// the start of a new line. Pure start-of-string anchoring missed it. +// +// 3. Prose that DESCRIBES the patterns (2026-06-12 self-own): +// "Triggers on:\n - Error: T: [resource_exhausted]\n - ..." +// -- each bullet line starts with whitespace+dash, NOT with "Error:". +// Multiline `^Error:` rejects it. +// +// 4. RetriableError prefix (session 0e04ebe7, 2026-06-20): +// "\n\nError: RetriableError: [canceled] http/2 stream closed..." +// -- same gRPC bracket notation as `Error: T:` but cursor-agent +// sometimes stringifies via RetriableError instead. Still a Cursor +// ACP session; HAPI persists agent messages in a codex-shaped +// envelope (`convertAgentMessage`) which is NOT the Codex runner. +// +// The diagnostic strength is in the `[snake_case]` bracket form (gRPC's +// stable status notation) and `Error: T:` prefix - both characteristic of +// cursor-agent's runtime stringification, rare in genuine prose. The +// catch-all `Error: T:` is intentionally narrow enough that benign mention +// would have to literally start a line with "Error: T:" - documented +// trade-off, the false-positive surface is small and recoverable +// (operator dismisses banner). The miss surface (this exact failure +// class going unflagged) is much worse. +// `^[ \t]*` allows horizontal whitespace before the marker (covers +// session b52b9117's " Error: T: [canceled]" wire format with leading +// spaces). It does NOT allow `\n` consumption, so multi-line strings +// only match where the marker actually sits at the start of a line. +// Bullet-list prose like " - Error: T: ..." still rejects: after +// `[ \t]*` consumes the spaces, the next char is `-`, not `Error`. +const PATTERNS: Pattern[] = [ + // RetriableError resource_exhausted is auto-bridgeable: cursor-agent + // labeled the failure RetriableError, so mark transient:true. Keep the + // `Error: T:` form (and stderr quota_exceeded mapping) non-transient — + // those are harder quota / capacity stops, not hiccups. + { + test: (t) => /^[ \t]*Error: RetriableError: \[resource_exhausted\]/im.test(t), + kind: 'quota_exhausted', + transient: true + }, + { + test: (t) => /^[ \t]*Error: T: \[resource_exhausted\]/im.test(t), + kind: 'quota_exhausted', + transient: false + }, + { + test: (t) => /^[ \t]*Error: T: \[canceled\]/im.test(t) + || /^[ \t]*Error: RetriableError: \[canceled\]/im.test(t), + kind: 'canceled', + transient: true + }, + { + test: (t) => /^[ \t]*Error: T: \[deadline_exceeded\]/im.test(t) + || /^[ \t]*Error: RetriableError: \[deadline_exceeded\]/im.test(t), + kind: 'deadline_exceeded', + transient: true + }, + { + test: (t) => /^[ \t]*Error: T: \[unavailable\]/im.test(t) + || /^[ \t]*Error: RetriableError: \[unavailable\]/im.test(t), + kind: 'unavailable', + transient: true + }, + { + test: (t) => /^[ \t]*Error: T: Connection stalled/im.test(t) + || /^[ \t]*Error: RetriableError: Connection stalled/im.test(t), + kind: 'connection_stalled', + transient: true + }, + { + test: (t) => /^[ \t]*Gemini prompt failed:.*token count exceeds/im.test(t), + kind: 'context_window', + transient: false + }, + { + test: (t) => /^[ \t]*Gemini prompt failed:.*exhausted your capacity/im.test(t), + kind: 'capacity_exhausted', + transient: false + }, + // catch-all for unknown `Error: T:` / `Error: RetriableError:` prefixes — placed last + { + test: (t) => /^[ \t]*Error: T:/im.test(t) + || /^[ \t]*Error: RetriableError:/im.test(t), + kind: 'unknown_t_prefix', + transient: false + } +] + +/** + * Returns a failure descriptor when the message text matches a known + * cursor-agent inline model error pattern, or null for benign messages. + * + * NOTE: this is the LAST-RESORT fallback. Prefer structural signals + * (classifyAcpRpcRejection, mapAcpStderrToFailure) — they fire before + * this path runs and are not subject to false positives from prose that + * happens to match the pattern shape. + */ +/** Drop closed markdown fenced regions so quoted Error: T: examples are not + * classified. Tracks opener char + length (four-backtick outer ≠ closed by + * inner ```). Unclosed fences are restored at EOF so a truncated generation + * that ends with Error: T: is still classifiable. */ +function stripMarkdownFences(text: string): string { + const visible: string[] = [] + let fence: { char: '`' | '~'; length: number; buffered: string[] } | null = null + for (const line of text.split(/\r?\n/)) { + const match = line.match(/^[ \t]{0,3}(`{3,}|~{3,})/) + if (!match) { + if (fence) { + fence.buffered.push(line) + } else { + visible.push(line) + } + continue + } + const marker = match[1]! + const char = marker[0]! as '`' | '~' + if (!fence) { + fence = { char, length: marker.length, buffered: [line] } + continue + } + fence.buffered.push(line) + // Closers are bare markers only (no info string like ```ts). + const close = line.match(/^[ \t]{0,3}(`{3,}|~{3,})[ \t]*$/) + if ( + close + && close[1]![0] === fence.char + && close[1]!.length >= fence.length + ) { + fence = null + } + } + if (fence) { + visible.push(...fence.buffered) + } + return visible.join('\n') +} + +/** Drop CommonMark indented-code lines so quoted Error: T: examples are not + * treated as live wire failures. Tab stops of 4: 4+ spaces, or 0–3 spaces + * then a tab. Keep bare 1–3 space prefixes — real ACP emits sometimes pad. */ +function stripIndentedCode(text: string): string { + return text + .split(/\r?\n/) + .filter((line) => !/^(?: {4,}| {0,3}\t)/.test(line)) + .join('\n') +} + +export function classifyCursorAgentMessage(text: string): CursorAgentStreamFailure | null { + const candidate = stripIndentedCode(stripMarkdownFences(text)) + for (const pattern of PATTERNS) { + if (pattern.test(candidate)) { + return { kind: pattern.kind, transient: pattern.transient, raw: text, source: 'text' } + } + } + return null +} + +/** + * Strong failure signatures required before promoting a typed ACP stderr + * event to modelError. The transport assigns rate_limit / authentication / + * quota_exceeded from bare substrings ("rate limit", "authentication", + * "quota"), so a line like "authentication provider initialized" becomes + * type authentication. We re-check raw text here. + */ +const STRONG_STDERR_PATTERNS = { + rate_limit: /status 429|ratelimitexceeded|rate limit (?:exceeded|reached|hit)/i, + model_not_found: /Cannot use this model:\s*\S|status 404[^\n]*\bmodel\b|model (?:is )?not found/i, + authentication: /status (?:401|403)|\bunauthenticated\b|authentication (?:failed|required|expired)|(?:token|credential)[^\n]{0,80}permission denied/i, + quota_exceeded: /quota (?:exceeded|exhausted|limit reached)|resource ?exhausted/i +} as const + +/** + * Maps an AcpStderrError (typed by AcpStdioTransport.parseStderrError) to + * a CursorAgentStreamFailure when the raw text also carries a strong + * failure signature. Accepts a minimal AcpStderrError shape so this module + * stays decoupled from the transport package. + * + * Returns null for: + * - `unknown` (transport labels any "error"/"failed"/"exception" line) + * - typed events whose raw text is only a weak substring match + * + * ACP treats stderr as logging (`Clients MAY capture, forward, or ignore`); + * urgent model-error alerts need a definitive failure signature. + */ +export function mapAcpStderrToFailure(error: { + type: 'rate_limit' | 'model_not_found' | 'authentication' | 'quota_exceeded' | 'unknown' + raw: string +}): CursorAgentStreamFailure | null { + if (error.type === 'unknown') { + return null + } + + if (!STRONG_STDERR_PATTERNS[error.type].test(error.raw)) { + return null + } + + switch (error.type) { + case 'rate_limit': + return { kind: 'rate_limited', transient: true, raw: error.raw, source: 'stderr' } + case 'model_not_found': + return { kind: 'model_not_found', transient: false, raw: error.raw, source: 'stderr' } + case 'authentication': + return { kind: 'auth_failed', transient: false, raw: error.raw, source: 'stderr' } + case 'quota_exceeded': + return { kind: 'quota_exhausted', transient: false, raw: error.raw, source: 'stderr' } + default: + return null + } +} + +/** + * Inspects an Error thrown by `backend.prompt(...)` (i.e. the JSON-RPC + * `session/prompt` call) and classifies it. + * + * The error sources are STRUCTURAL: thrown by either the + * `@zed-industries/agent-client-protocol` SDK on transport close, or by + * `AcpStdioTransport.markClosed` -> `rejectAllPending` on process exit / + * stream close, or by the JSON-RPC layer on `error.message` from a typed + * agent-side error response. We do match against `error.message` strings + * here — but those strings are emitted by code we control or vendor, not + * by free-text agent prose. The blast radius for false positives is the + * library's stable error vocabulary, which is much smaller than "any + * message a cursor-agent might emit." + * + * Returns null if the error doesn't look like a model-side failure + * (e.g. user cancellation, programmer error in our own code) — the + * caller should still log/surface the error but not fire modelError. + */ +export function classifyAcpRpcRejection(error: unknown): CursorAgentStreamFailure | null { + const raw = error instanceof Error ? error.message : String(error) + const lower = raw.toLowerCase() + + // gRPC / agent text shapes can arrive as JSON-RPC error.message (including + // "... [canceled] Operation aborted"). Classify those BEFORE the user-abort + // filter — a bare `aborted` substring must not swallow real model cancels. + const textMatch = classifyCursorAgentMessage(raw) + if (textMatch) { + return { ...textMatch, source: 'rpc' } + } + + // Programmer/user signals that should NOT fire modelError. + if ( + lower.includes('aborted by user') + || lower.includes('user cancelled') + || lower.includes('user canceled') + ) { + return null + } + + // Transport-level closure: WritableIterable closed, ACP transport closed, + // or process exited. All come through markClosed() -> rejectAllPending(). + // Not bridgeable: Bridge reuses the same backend.prompt path, and a closed + // AcpStdioTransport rejects every subsequent sendRequest until reconnect. + if ( + lower.includes('writableiterable is closed') || + lower.includes('acp transport is closed') || + lower.includes('acp transport closed') || + lower.includes('acp process exited') + ) { + return { kind: 'transport_closed', transient: false, raw, source: 'rpc' } + } + + // Process error during spawn / IO failure during write. + // Same as transport_closed: no reconnect/reload yet, so Bridge cannot recover. + if (lower.includes('failed to spawn') || lower.includes('epipe') || lower.includes('ecanceled')) { + return { kind: 'agent_crashed', transient: false, raw, source: 'rpc' } + } + + // Request-level timeout (DEFAULT_TIMEOUT_MS in AcpStdioTransport). + if (lower.includes('timed out after') || lower.includes('timeout')) { + return { kind: 'rpc_timeout', transient: true, raw, source: 'rpc' } + } + + // Plain JSON-RPC error.message (no [rate_limit] brackets). Same signatures + // as stderr promotion so Bridge can retry structural 429s. + if (STRONG_STDERR_PATTERNS.rate_limit.test(raw)) { + return { kind: 'rate_limited', transient: true, raw, source: 'rpc' } + } + + // Catch-all: the prompt rejected for SOME reason. Conservative: + // fire modelError as 'prompt_failed' (non-transient) so the operator + // sees the turn was degraded. + return { kind: 'prompt_failed', transient: false, raw, source: 'rpc' } +} + +const PRIOR_DONE_PATTERNS = [ + /^done(?:[.!:]|\s|$)/, + /^all done(?:[.!:]|\s|$)/, + /^committed(?:[.!:]|\s|$)/, + /^fixed(?:[.!:]|\s|$)/, + /^complete(?:d)?(?:[.!:]|\s|$)/, + /^successfully\s+(?:completed|fixed|committed)\b/ +] + +/** + * Returns true when the text looks like the agent claimed task completion + * (e.g. "Done.", "All done.", "Successfully completed."). + * Word-boundary patterns — not startsWith — so "Completely unable…" and + * "Successfully reproduced…" do not falsely set priorAssistantClaimsDone. + */ +export function isCompletionClaim(text: string): boolean { + const lower = text.trim().toLowerCase() + return PRIOR_DONE_PATTERNS.some((pattern) => pattern.test(lower)) +} + +const TEXT_ERROR_MARKER = /^[ \t]*(?:Error: (?:T|RetriableError):|Gemini prompt failed:)/gim + +/** + * Prefer the matched error marker when slicing rawSnippet so "View raw error" + * shows the failure, not the preceding assistant prose when Error: T: is + * appended after a long response. + * + * Use the LAST marker: classifyCursorAgentMessage strips closed fences / + * indented code before matching, so an earlier quoted example in raw text + * must not steal the snippet from the real appended failure. + */ +export function rawSnippetForFailure(failure: CursorAgentStreamFailure, maxLen = 400): string { + let start = 0 + if (failure.source === 'text') { + const matches = Array.from(failure.raw.matchAll(TEXT_ERROR_MARKER)) + const last = matches.at(-1) + if (typeof last?.index === 'number') { + start = last.index + } + } + return failure.raw.slice(start, start + maxLen) +} diff --git a/cli/src/cursor/cursorLegacyRemoteLauncher.ts b/cli/src/cursor/cursorLegacyRemoteLauncher.ts index a273423f3d..0abe95d31b 100644 --- a/cli/src/cursor/cursorLegacyRemoteLauncher.ts +++ b/cli/src/cursor/cursorLegacyRemoteLauncher.ts @@ -20,6 +20,16 @@ import { RPC_METHODS } from '@hapi/protocol/rpcMethods'; import type { CursorStreamEvent } from './utils/cursorLegacyEventConverter'; import { parseCursorEvent, convertCursorEventToAgentMessage } from './utils/cursorLegacyEventConverter'; import { cursorPassThroughStatusMessage, parseCursorSpecialCommand } from './cursorSpecialCommands'; +// NOTE: model-error detection (cursorAgentMessageClassifier) is intentionally +// NOT wired into the legacy launcher. The hub auto-migrates legacy stream-json +// sessions to ACP at resume time via maybeAutoMigrateLegacyCursorSession (PR +// #844). The legacy launcher is reached only when migration soft-fails — a +// degraded fallback path, not a supported flow. Carrying duplicate +// model-error logic here would (a) double the surface for bugs in the +// model-error contract, (b) imply legacy is a real path that needs feature +// parity, and (c) be dead code in practice. New cursor sessions are ACP from +// inception. If you find yourself wanting model-error detection on this +// path, fix the migration failure case instead. // Transient `agent` failures (auth expiry, rate limits, transient network) come back // as exit code 1 with a recognisable stderr signature. We requeue and retry instead diff --git a/cli/src/cursor/cursorModelErrorBridge.test.ts b/cli/src/cursor/cursorModelErrorBridge.test.ts new file mode 100644 index 0000000000..264a6593e4 --- /dev/null +++ b/cli/src/cursor/cursorModelErrorBridge.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it } from 'vitest'; +import { + buildModelErrorBridgePrompt, + canBridgeModelError, + mergeBridgeGateFields, + MODEL_ERROR_BRIDGE_HEADER, + MAX_LAST_USER_MESSAGE_CHARS, + truncateLastUserMessage +} from './cursorModelErrorBridge'; + +describe('buildModelErrorBridgePrompt', () => { + it('wraps the last user message with bridge context', () => { + const prompt = buildModelErrorBridgePrompt({ + kind: 'transport_closed', + rawSnippet: 'WritableIterable is closed', + lastUserMessage: 'Fix the login bug in auth.ts', + priorAssistantClaimsDone: false + }); + + expect(prompt).toContain(MODEL_ERROR_BRIDGE_HEADER); + expect(prompt).toContain('transport_closed: WritableIterable is closed'); + expect(prompt).toContain('Re-sending your last message below.'); + expect(prompt).toContain('---'); + expect(prompt.endsWith('Fix the login bug in auth.ts')).toBe(true); + expect(prompt).not.toContain('verify what is actually done'); + }); + + it('adds completion verification when priorAssistantClaimsDone is true', () => { + const prompt = buildModelErrorBridgePrompt({ + kind: 'rpc_timeout', + rawSnippet: 'request timed out after 30000ms', + lastUserMessage: 'ship the feature', + priorAssistantClaimsDone: true + }); + + expect(prompt).toContain('verify what is actually done before proceeding.'); + }); +}); + +describe('canBridgeModelError', () => { + const base = { + transient: true, + eventId: 'evt-1000' + }; + + it('allows transient errors that have not been bridged', () => { + expect(canBridgeModelError(base)).toBe(true); + }); + + it('blocks non-transient errors', () => { + expect(canBridgeModelError({ ...base, transient: false })).toBe(false); + }); + + it('blocks quota and auth style failures via transient=false', () => { + expect(canBridgeModelError({ + ...base, + transient: false + })).toBe(false); + }); + + it('dedupes when this eventId was already bridged', () => { + expect(canBridgeModelError({ + ...base, + bridgedForEventId: 'evt-1000' + })).toBe(false); + }); + + it('allows a new error after a prior bridge on a different eventId', () => { + expect(canBridgeModelError({ + ...base, + eventId: 'evt-2000', + bridgedForEventId: 'evt-1000' + })).toBe(true); + }); + + it('blocks when bridge already failed', () => { + expect(canBridgeModelError({ + ...base, + retriedAndFailed: true + })).toBe(false); + }); + + it('blocks when a newer normal turn superseded the error', () => { + expect(canBridgeModelError({ + ...base, + supersededByUserTurn: true + })).toBe(false); + }); + + it('blocks when bridgeable is explicitly false', () => { + expect(canBridgeModelError({ + ...base, + bridgeable: false + })).toBe(false); + }); +}); + +describe('mergeBridgeGateFields', () => { + it('keeps prior bridgedForEventId when hub snapshot omits it', () => { + const merged = mergeBridgeGateFields( + { bridgedForEventId: 'evt-1000', retriedAndFailed: false }, + { bridgedForEventId: undefined, retriedAndFailed: false } + ); + expect(merged.bridgedForEventId).toBe('evt-1000'); + expect(canBridgeModelError({ + transient: true, + eventId: 'evt-1000', + ...merged + })).toBe(false); + }); + + it('preserves retriedAndFailed when hub sends false', () => { + const merged = mergeBridgeGateFields( + { bridgedForEventId: 'evt-1000', retriedAndFailed: true }, + { bridgedForEventId: undefined, retriedAndFailed: false } + ); + expect(merged.retriedAndFailed).toBe(true); + }); + + it('preserves supersededByUserTurn when hub omits it', () => { + const merged = mergeBridgeGateFields( + { supersededByUserTurn: true }, + { bridgedForEventId: undefined, retriedAndFailed: false, supersededByUserTurn: undefined } + ); + expect(merged.supersededByUserTurn).toBe(true); + expect(canBridgeModelError({ + transient: true, + eventId: 'evt-1000', + ...merged + })).toBe(false); + }); +}); + +describe('truncateLastUserMessage', () => { + it('passes through short messages unchanged', () => { + expect(truncateLastUserMessage('hello')).toBe('hello'); + }); + + it('caps very long messages', () => { + const long = 'x'.repeat(40_000); + expect(truncateLastUserMessage(long).length).toBe(MAX_LAST_USER_MESSAGE_CHARS); + }); +}); + +describe('MAX_LAST_USER_MESSAGE_CHARS', () => { + it('is the bridge fail-closed limit', () => { + expect(MAX_LAST_USER_MESSAGE_CHARS).toBe(32_000); + }); +}); diff --git a/cli/src/cursor/cursorModelErrorBridge.ts b/cli/src/cursor/cursorModelErrorBridge.ts new file mode 100644 index 0000000000..1c75af08be --- /dev/null +++ b/cli/src/cursor/cursorModelErrorBridge.ts @@ -0,0 +1,91 @@ +export const MODEL_ERROR_BRIDGE_HEADER = '[HAPI bridge — transient model error]'; + +export const MAX_LAST_USER_MESSAGE_CHARS = 32_000; +const MAX_EXCERPT_CHARS = 120; + +export type ModelErrorBridgeInput = { + kind: string; + rawSnippet: string; + lastUserMessage: string; + priorAssistantClaimsDone: boolean; +}; + +export type ModelErrorBridgeGate = { + transient: boolean; + eventId: string; + bridgedForEventId?: string; + retriedAndFailed?: boolean; + supersededByUserTurn?: boolean; + /** Explicit false blocks Bridge (e.g. idle stderr after a successful turn). */ + bridgeable?: boolean; +}; + +export function truncateLastUserMessage(message: string): string { + if (message.length <= MAX_LAST_USER_MESSAGE_CHARS) { + return message; + } + return message.slice(0, MAX_LAST_USER_MESSAGE_CHARS); +} + +export function buildModelErrorBridgePrompt(input: ModelErrorBridgeInput): string { + const excerpt = input.rawSnippet + .replace(/\s+/g, ' ') + .trim() + .slice(0, MAX_EXCERPT_CHARS); + + const lines = [ + MODEL_ERROR_BRIDGE_HEADER, + '', + `The previous turn failed before completing (${input.kind}: ${excerpt}).`, + '', + 'Re-sending your last message below. Continue the task from where you left off.', + 'Do not repeat work you already finished unless the error invalidated it.' + ]; + + if (input.priorAssistantClaimsDone) { + lines.push(''); + lines.push('You may have reported completion before this error — verify what is actually done before proceeding.'); + } + + lines.push(''); + lines.push('---'); + lines.push(input.lastUserMessage); + + return lines.join('\n'); +} + +export function canBridgeModelError(gate: ModelErrorBridgeGate): boolean { + if (!gate.transient) { + return false; + } + if (gate.bridgeable === false) { + return false; + } + if (gate.retriedAndFailed) { + return false; + } + if (gate.supersededByUserTurn) { + return false; + } + if (gate.bridgedForEventId === gate.eventId) { + return false; + } + return true; +} + +/** Merge hub RPC snapshot gates into local state without clobbering. */ +export function mergeBridgeGateFields( + prior: Pick | null | undefined, + incoming: Pick +): Pick { + return { + bridgedForEventId: incoming.bridgedForEventId ?? prior?.bridgedForEventId, + retriedAndFailed: incoming.retriedAndFailed === true || prior?.retriedAndFailed === true, + supersededByUserTurn: incoming.supersededByUserTurn === true + || prior?.supersededByUserTurn === true, + // false wins — never re-open Bridge from a stale hub omit. + bridgeable: incoming.bridgeable === false || prior?.bridgeable === false + ? false + : (incoming.bridgeable ?? prior?.bridgeable) + }; +} diff --git a/cli/src/cursor/cursorModelErrorBridgePrefs.ts b/cli/src/cursor/cursorModelErrorBridgePrefs.ts new file mode 100644 index 0000000000..0c5697c9fa --- /dev/null +++ b/cli/src/cursor/cursorModelErrorBridgePrefs.ts @@ -0,0 +1,9 @@ +let autoBridgeTransientModelErrors = false; + +export function setAutoBridgeTransientModelErrors(enabled: boolean): void { + autoBridgeTransientModelErrors = enabled; +} + +export function getAutoBridgeTransientModelErrors(): boolean { + return autoBridgeTransientModelErrors; +} diff --git a/cli/src/cursor/cursorUserMessageQueue.test.ts b/cli/src/cursor/cursorUserMessageQueue.test.ts index a80c7cf1b8..830544eeca 100644 --- a/cli/src/cursor/cursorUserMessageQueue.test.ts +++ b/cli/src/cursor/cursorUserMessageQueue.test.ts @@ -7,6 +7,23 @@ import type { EnhancedMode } from './loop'; const mode: EnhancedMode = { permissionMode: 'default' }; describe('enqueueCursorUserMessage', () => { + it('preserves caller bridge: localId but keeps the turn non-bridge provenance', async () => { + const consumed: string[] = []; + const queue = new MessageQueue2((m) => m.permissionMode); + queue.onBatchConsumed = (localIds) => { + consumed.push(...localIds); + }; + enqueueCursorUserMessage(queue, 'please continue', mode, 'bridge:evt-1'); + expect(queue.queue).toHaveLength(1); + expect(queue.queue[0]?.localId).toBe('bridge:evt-1'); + expect(queue.queue[0]?.internal).toBeUndefined(); + expect(queue.hasPendingNonBridgeTurn()).toBe(true); + + const batch = await queue.waitForMessagesAndGetAsString(); + expect(batch?.items[0]?.internal).toBeUndefined(); + expect(consumed).toEqual(['bridge:evt-1']); + }); + it('isolates /compress from a following same-mode prompt', async () => { const queue = new MessageQueue2((m) => m.permissionMode); enqueueCursorUserMessage(queue, '/compress keep recap', mode, 'a'); diff --git a/cli/src/cursor/cursorUserMessageQueue.ts b/cli/src/cursor/cursorUserMessageQueue.ts index 28ae72425f..c39f8577f2 100644 --- a/cli/src/cursor/cursorUserMessageQueue.ts +++ b/cli/src/cursor/cursorUserMessageQueue.ts @@ -12,6 +12,9 @@ export function enqueueCursorUserMessage( enhancedMode: EnhancedMode, localId?: string ): void { + // Preserve caller localId for messages-consumed / cancel. Bridge provenance + // is queue-owned (`internal.kind === 'model-error-bridge'`), so a forged + // `bridge:*` localId cannot impersonate a Bridge turn. const specialCommand = parseCursorSpecialCommand(formattedText); if (specialCommand.type !== null) { messageQueue.pushIsolated(formattedText.trim(), enhancedMode, localId); diff --git a/cli/src/cursor/runCursor.ts b/cli/src/cursor/runCursor.ts index 0cf4636781..1871af7837 100644 --- a/cli/src/cursor/runCursor.ts +++ b/cli/src/cursor/runCursor.ts @@ -16,6 +16,7 @@ import { formatMessageWithAttachments } from '@/utils/attachmentFormatter'; import { getInvokedCwd } from '@/utils/invokedCwd'; import { enqueueCursorUserMessage } from './cursorUserMessageQueue'; import { RPC_METHODS } from '@hapi/protocol/rpcMethods'; +import { setAutoBridgeTransientModelErrors } from './cursorModelErrorBridgePrefs'; const formatFailureReason = (message: string): string => { const maxLength = 200; @@ -123,16 +124,24 @@ export async function runCursor(opts: { permissionMode?: unknown; model?: unknown; modelReasoningEffort?: unknown; + autoBridgeTransientModelErrors?: unknown; }; const applied: { permissionMode?: PermissionMode; model?: string | null; + autoBridgeTransientModelErrors?: boolean; } = {}; if (config.modelReasoningEffort !== undefined) { throw new Error('Invalid model reasoning effort'); } + if (config.autoBridgeTransientModelErrors !== undefined) { + const enabled = config.autoBridgeTransientModelErrors === true; + setAutoBridgeTransientModelErrors(enabled); + applied.autoBridgeTransientModelErrors = enabled; + } + const nextPermissionMode = config.permissionMode !== undefined ? resolveSessionConfigPermissionMode(config.permissionMode, 'cursor') : undefined; diff --git a/cli/src/utils/MessageQueue2.test.ts b/cli/src/utils/MessageQueue2.test.ts index d1dee25e20..ae4e6ed110 100644 --- a/cli/src/utils/MessageQueue2.test.ts +++ b/cli/src/utils/MessageQueue2.test.ts @@ -486,6 +486,29 @@ describe('MessageQueue2', () => { expect(queue.size()).toBe(0); }); + it('does not ACK a later forged bridge: localId when dequeuing a synthetic Bridge', async () => { + const queue = new MessageQueue2(mode => mode); + const received: string[][] = []; + queue.onBatchConsumed = (localIds) => { received.push(localIds); }; + + const eventId = 'evt-ack'; + queue.unshiftIsolated( + 'bridge retry', + 'local', + undefined, + { kind: 'model-error-bridge', eventId } + ); + queue.push('real prompt', 'local', `bridge:${eventId}`); + + const batch1 = await queue.waitForMessagesAndGetAsString(); + expect(batch1?.message).toBe('bridge retry'); + expect(received).toEqual([]); + + const batch2 = await queue.waitForMessagesAndGetAsString(); + expect(batch2?.message).toBe('real prompt'); + expect(received).toEqual([[`bridge:${eventId}`]]); + }); + it('should skip onBatchConsumed when batch has no localIds', async () => { const queue = new MessageQueue2(mode => mode); let called = false; diff --git a/cli/src/utils/MessageQueue2.ts b/cli/src/utils/MessageQueue2.ts index 6c724ba1ae..8465eaa698 100644 --- a/cli/src/utils/MessageQueue2.ts +++ b/cli/src/utils/MessageQueue2.ts @@ -1,5 +1,8 @@ import { logger } from "@/ui/logger"; +export type QueueItemInternal = + | { kind: 'model-error-bridge'; eventId: string }; + export interface QueueItem { message: string; mode: T; @@ -8,6 +11,8 @@ export interface QueueItem { isolate?: boolean; // If true, this message must be processed alone /** Stable FIFO key used when an async reservation is restored later. */ enqueueOrder?: number; + /** Queue-owned provenance — never inferred from caller localId. */ + internal?: QueueItemInternal; } export type QueueReservation = { @@ -21,6 +26,12 @@ export type QueueReservation = { state: 'reserved' | 'dispatching' | 'indeterminate' | 'cancelled'; }; +export type CollectedQueueItem = { + message: string + localId?: string + internal?: QueueItemInternal +}; + /** * A mode-aware message queue that stores messages with their modes. * Returns consistent batches of messages with the same mode. @@ -261,7 +272,12 @@ export class MessageQueue2 { * that failed transiently and must retry without batching against sibling * prompts). */ - unshiftIsolated(message: string, mode: T, localId?: string): void { + unshiftIsolated( + message: string, + mode: T, + localId?: string, + internal?: QueueItemInternal + ): void { if (this.closed) { throw new Error('Cannot unshift to closed queue'); } @@ -275,6 +291,7 @@ export class MessageQueue2 { modeHash, localId, isolate: true, + internal, enqueueOrder: this.previousEnqueueOrder-- }; Object.defineProperty(item, 'enqueueOrder', { value: item.enqueueOrder, enumerable: false, writable: true }); @@ -293,6 +310,23 @@ export class MessageQueue2 { logger.debug(`[MessageQueue2] unshiftIsolated() completed. Queue size: ${this.queue.length}`); } + /** True when a non-bridge user/API turn is already waiting. */ + hasPendingNonBridgeTurn(): boolean { + return this.queue.some((item) => item.internal?.kind !== 'model-error-bridge'); + } + + /** Drop a pending model-error bridge by its eventId (queue-owned provenance). */ + cancelModelErrorBridge(eventId: string): boolean { + if (!eventId) return false; + const idx = this.queue.findIndex( + (item) => item.internal?.kind === 'model-error-bridge' + && item.internal.eventId === eventId + ); + if (idx === -1) return false; + this.queue.splice(idx, 1); + return true; + } + /** * Remove the first queued message that matches the given localId. * Returns true if a message was removed, false if not found. @@ -551,7 +585,7 @@ export class MessageQueue2 { * Wait for messages and return all messages with the same mode as a single string * Returns { message: string, mode: T } or null if aborted/closed */ - async waitForMessagesAndGetAsString(abortSignal?: AbortSignal): Promise<{ message: string, mode: T, isolate: boolean, hash: string, items: Array<{ message: string, localId?: string }> } | null> { + async waitForMessagesAndGetAsString(abortSignal?: AbortSignal): Promise<{ message: string, mode: T, isolate: boolean, hash: string, items: CollectedQueueItem[] } | null> { // If we have messages, return them immediately if (this.queue.length > 0) { return this.collectBatch(); @@ -575,7 +609,7 @@ export class MessageQueue2 { /** * Collect a batch of messages with the same mode, respecting isolation requirements */ - private collectBatch(): { message: string, mode: T, hash: string, isolate: boolean, items: Array<{ message: string, localId?: string }> } | null { + private collectBatch(): { message: string, mode: T, hash: string, isolate: boolean, items: CollectedQueueItem[] } | null { if (this.queue.length === 0) { return null; } @@ -587,7 +621,7 @@ export class MessageQueue2 { // `message` string below so callers that need to requeue individual // messages (e.g. restoring a failed batch with each item's own // localId intact) don't have to re-split an already-joined string. - const items: Array<{ message: string, localId?: string }> = []; + const items: CollectedQueueItem[] = []; let mode = firstItem.mode; let isolate = firstItem.isolate ?? false; const targetModeHash = firstItem.modeHash; @@ -596,7 +630,7 @@ export class MessageQueue2 { if (firstItem.isolate) { const item = this.queue.shift()!; sameModeMessages.push(item.message); - items.push({ message: item.message, localId: item.localId }); + items.push({ message: item.message, localId: item.localId, internal: item.internal }); if (item.localId) consumedLocalIds.push(item.localId); logger.debug(`[MessageQueue2] Collected isolated message with mode hash: ${targetModeHash}`); } else { @@ -606,7 +640,7 @@ export class MessageQueue2 { !this.queue[0].isolate) { const item = this.queue.shift()!; sameModeMessages.push(item.message); - items.push({ message: item.message, localId: item.localId }); + items.push({ message: item.message, localId: item.localId, internal: item.internal }); if (item.localId) consumedLocalIds.push(item.localId); } logger.debug(`[MessageQueue2] Collected batch of ${sameModeMessages.length} messages with mode hash: ${targetModeHash}`); diff --git a/docs/api/native-companion-contract.md b/docs/api/native-companion-contract.md index c7621c4b81..5c7a6f518c 100644 --- a/docs/api/native-companion-contract.md +++ b/docs/api/native-companion-contract.md @@ -61,16 +61,17 @@ namespace to avoid duplicate OS notifications. | Key | Example | Purpose | |-----|---------|---------| -| `type` | `ready` | `ready`, `permission-request`, `task-notification` | +| `type` | `ready` | `ready`, `permission-request`, `task-notification`, `model-error` | | `sessionId` | uuid | Target session | | `sessionName` | string | Display name (`agent - project`) | | `url` | `/sessions/{id}` | Deep link path | | `requestId` | uuid | Permission only - approve/deny | | `title` | string | Notification title | | `body` | string | Notification body | -| `severity` | `info` | `info` (ready), `warning` (permission), `success` / `error` (task) | +| `severity` | `info` | `info` (ready), `warning` (permission), `success` / `error` (task), `error` (`model-error`) | | `contractVersion` | `1` | Present on every message; see [Versioning](#versioning) | | `notifySummary` | JSON string | Only on `ready`: parsed `AGENT_NOTIFY_SUMMARY` line from agent text, when present | +| `tag` | `model-error--` | Coalescing identity. Required for `model-error` so distinct errors do not overwrite. Clients must prefer `tag` over reconstructing `type-`. | Native apps **must** handle `data` for Wear; notification block is for display. @@ -156,9 +157,10 @@ decrypts `hapi.e` with the Keychain `pushKey` and replaces title/body with the real content; `hapi.v` is the envelope version (currently `1`). Delivery headers: `apns-push-type: alert`, `apns-priority: 10`, -`apns-expiration: 0`, `apns-collapse-id: "-"` (truncated to -64 bytes) - so newer notifications for the same session/type replace older -ones. +`apns-expiration: 0`, `apns-collapse-id: "-"` (or the +payload `tag` when present, e.g. `model-error--`), +truncated to 64 bytes - so newer notifications for the same coalescing +identity replace older ones. ### Transports: self-host (direct APNs) vs official relay diff --git a/hub/src/config/autoBridgeTransientModelErrors.ts b/hub/src/config/autoBridgeTransientModelErrors.ts new file mode 100644 index 0000000000..956bd491c3 --- /dev/null +++ b/hub/src/config/autoBridgeTransientModelErrors.ts @@ -0,0 +1,36 @@ +import { + getSettingsFile, + readSettingsOrThrow, + updateSettings, + type Settings +} from './settings' + +/** + * Hub-persisted opt-in for Cursor auto-bridge after transient model errors. + * Default is off (undefined / false). Applied to CLI processes on session + * create/get so unattended / restarted sessions honor Settings. + */ +export function isAutoBridgeTransientModelErrorsSettingEnabled(settings: Settings): boolean { + return settings.autoBridgeTransientModelErrors === true +} + +export async function readAutoBridgeTransientModelErrorsEnabled(dataDir: string): Promise { + const settings = await readSettingsOrThrow(getSettingsFile(dataDir)) + return isAutoBridgeTransientModelErrorsSettingEnabled(settings) +} + +export async function writeAutoBridgeTransientModelErrorsEnabled( + dataDir: string, + enabled: boolean +): Promise { + return updateSettings(getSettingsFile(dataDir), (current) => { + const settings = { + ...current, + autoBridgeTransientModelErrors: enabled + } + return { + settings, + result: settings.autoBridgeTransientModelErrors === true + } + }) +} diff --git a/hub/src/config/settings.ts b/hub/src/config/settings.ts index 0db04271a2..5b0113c354 100644 --- a/hub/src/config/settings.ts +++ b/hub/src/config/settings.ts @@ -45,6 +45,11 @@ export interface Settings { * Default off: render/copy strip the footer; store stays raw. */ sessionSummaryInChat?: boolean + /** + * When true, Cursor CLI sessions auto-enqueue a one-shot bridge after a + * transient model error. Default off. Survives hub/CLI restarts. + */ + autoBridgeTransientModelErrors?: boolean /** * Hub-side provider API keys / endpoints managed from Settings. * Env vars still win when set at process start (ops override). diff --git a/hub/src/fcm/fcmNotificationChannel.test.ts b/hub/src/fcm/fcmNotificationChannel.test.ts index 50c00e54c7..32e5206e12 100644 --- a/hub/src/fcm/fcmNotificationChannel.test.ts +++ b/hub/src/fcm/fcmNotificationChannel.test.ts @@ -592,4 +592,112 @@ describe('FcmNotificationChannel', () => { expect(sent[0].body).toContain('...') expect(sent[0].body.length).toBeLessThan(350) }) + + it('sendModelError fires FCM with severity=error even when PWA is foreground', async () => { + const sent: FcmSendPayload[] = [] + const toasts: unknown[] = [] + const channel = new FcmNotificationChannel( + { + sendToNamespace: async (_namespace: string, payload: FcmSendPayload) => { + sent.push(payload) + } + } as never, + { + sendToast: async (_namespace: string, event: unknown) => { + toasts.push(event) + return 1 + } + } as never, + { + hasVisibleConnection: () => true + } as never + ) + + await channel.sendModelError(createSession(), { + eventId: 'evt-1710000000000', + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'You have hit your usage limit', + priorAssistantClaimsDone: true, + atTs: 1710000000000 + }) + + expect(sent).toHaveLength(1) + expect(toasts).toHaveLength(0) + expect(sent[0].data.type).toBe('model-error') + expect(sent[0].data.severity).toBe('error') + expect(sent[0].tag).toBe('model-error-session-ready-evt-1710000000000') + expect(sent[0].data.tag).toBe('model-error-session-ready-evt-1710000000000') + expect(sent[0].title).toBe('Quota exhausted') + expect(sent[0].body).toContain('Codex') + expect(sent[0].body).toContain('Demo') + }) + + it('sendModelError uses distinct tags per eventId so errors do not collapse', async () => { + const sent: FcmSendPayload[] = [] + const channel = new FcmNotificationChannel( + { sendToNamespace: async (_n: string, p: FcmSendPayload) => { sent.push(p) } } as never, + { sendToast: async () => 0 } as never, + { hasVisibleConnection: () => false } as never + ) + + const base = { + kind: 'rate_limited', + transient: true, + rawSnippet: 'slow down', + priorAssistantClaimsDone: false + } + + await channel.sendModelError(createSession(), { ...base, eventId: 'evt-1', atTs: 1 }) + await channel.sendModelError(createSession(), { ...base, eventId: 'evt-2', atTs: 2 }) + + expect(sent[0].tag).toBe('model-error-session-ready-evt-1') + expect(sent[1].tag).toBe('model-error-session-ready-evt-2') + expect(sent[0].data.type).toBe('model-error') + expect(sent[1].data.severity).toBe('error') + }) + + it('sendModelError sets nativeGate.sent when FCM delivers', async () => { + const gate = { sent: false } + const channel = new FcmNotificationChannel( + { + sendToNamespace: async () => ({ sent: 1, failed: 0, invalidTokens: [] }) + } as never, + { sendToast: async () => 0 } as never, + { hasVisibleConnection: () => false } as never + ) + + await channel.sendModelError(createSession(), { + eventId: 'evt-9', + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'limit', + priorAssistantClaimsDone: false, + atTs: 9 + }, { nativeGate: gate }) + + expect(gate.sent).toBe(true) + }) + + it('sendModelError leaves nativeGate.sent false when FCM sends zero', async () => { + const gate = { sent: false } + const channel = new FcmNotificationChannel( + { + sendToNamespace: async () => ({ sent: 0, failed: 1, invalidTokens: [] }) + } as never, + { sendToast: async () => 0 } as never, + { hasVisibleConnection: () => false } as never + ) + + await channel.sendModelError(createSession(), { + eventId: 'evt-9', + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'limit', + priorAssistantClaimsDone: false, + atTs: 9 + }, { nativeGate: gate }) + + expect(gate.sent).toBe(false) + }) }) diff --git a/hub/src/fcm/fcmNotificationChannel.ts b/hub/src/fcm/fcmNotificationChannel.ts index 06c1331372..77b93a8a43 100644 --- a/hub/src/fcm/fcmNotificationChannel.ts +++ b/hub/src/fcm/fcmNotificationChannel.ts @@ -1,7 +1,9 @@ import type { Session } from '../sync/syncEngine' -import type { NotificationChannel, TaskNotification } from '../notifications/notificationTypes' +import type { ModelErrorNotification, ModelErrorSendOutcome, NotificationChannel, TaskNotification } from '../notifications/notificationTypes' import type { NotificationSendContext } from '../notifications/notificationSendContext' import { NATIVE_CONTRACT_VERSION, NativeNotificationComposer, type ComposedNativeNotification } from '../notifications/nativeNotificationComposer' +import { formatModelErrorBody, formatModelErrorTitle } from '../notifications/modelErrorCopy' +import { getAgentName, getSessionName } from '../notifications/sessionInfo' import type { Store } from '../store' import type { SSEManager } from '../sse/sseManager' import type { VisibilityTracker } from '../visibility/visibilityTracker' @@ -43,6 +45,47 @@ export class FcmNotificationChannel implements NotificationChannel { await this.deliver(session, this.toFcmPayload(this.composer.composeTask(session, notification)), ctx) } + async sendModelError( + session: Session, + notification: ModelErrorNotification, + ctx?: NotificationSendContext + ): Promise { + // No active-session guard: NotificationHub only starts dispatch for + // active sessions, but a bounded backoff retry must still deliver if + // the session went inactive before the timer fired. + + const agentName = getAgentName(session) + const sessionName = getSessionName(session) + const title = formatModelErrorTitle(notification.kind) + const body = formatModelErrorBody(notification, { agentName, sessionName }) + const tag = `model-error-${session.id}-${notification.eventId}` + + const result = await this.deliver(session, { + title, + body, + tag, + data: { + type: 'model-error', + sessionId: session.id, + sessionName, + url: `/sessions/${session.id}`, + title, + body, + contractVersion: NATIVE_CONTRACT_VERSION, + severity: 'error', + tag + } + }, ctx) + if ((result?.sent ?? 0) > 0) { + return 'delivered' + } + // No devices registered for this namespace - not a hard failure. + if ((result?.failed ?? 0) === 0) { + return 'unavailable' + } + return 'failed' + } + private toFcmPayload(composed: ComposedNativeNotification): FcmSendPayload { return { title: composed.title, @@ -63,7 +106,7 @@ export class FcmNotificationChannel implements NotificationChannel { } } - private async deliver(session: Session, payload: FcmSendPayload, ctx?: NotificationSendContext): Promise { + private async deliver(session: Session, payload: FcmSendPayload, ctx?: NotificationSendContext) { // Native companion is the canonical surface: always fire FCM when the // hub asks us to. The previous SSE-toast shortcut here meant that // when the operator had the PWA open in foreground, the watch got @@ -77,5 +120,6 @@ export class FcmNotificationChannel implements NotificationChannel { if ((result?.sent ?? 0) > 0 && ctx?.nativeGate) { ctx.nativeGate.sent = true } + return result } } diff --git a/hub/src/fcm/fcmService.test.ts b/hub/src/fcm/fcmService.test.ts index d2526a47d0..fd302123b1 100644 --- a/hub/src/fcm/fcmService.test.ts +++ b/hub/src/fcm/fcmService.test.ts @@ -60,6 +60,26 @@ describe('FcmService.sendToNamespace', () => { globalThis.fetch = originalFetch }) + it('serializes payload.tag onto message.data.tag in the HTTP body', async () => { + const store = makeStore([ + { namespace: 'default', token: 'tok-1', platform: 'phone', deviceId: 'p1' } + ]) + let requestBody: unknown = null + globalThis.fetch = mock(async (_url: string, init?: RequestInit) => { + requestBody = JSON.parse(String(init?.body ?? '{}')) + return new Response('{}', { status: 200 }) + }) as unknown as typeof fetch + + const svc = new FcmService('proj-id', { client_email: 'x', private_key: 'y' }, store as never) + await svc.sendToNamespace('default', { + ...makePayload({ type: 'model-error', severity: 'error' }), + tag: 'model-error-sess-1-1710000000000' + }) + + const message = (requestBody as { message?: { data?: Record } }).message + expect(message?.data?.tag).toBe('model-error-sess-1-1710000000000') + }) + it('removes the device row when FCM returns 404 UNREGISTERED (token rotated)', async () => { const store = makeStore([ { namespace: 'default', token: 'rotated-token', platform: 'phone', deviceId: 'p1' } diff --git a/hub/src/fcm/fcmService.ts b/hub/src/fcm/fcmService.ts index 8fcec571ec..d78baab1ca 100644 --- a/hub/src/fcm/fcmService.ts +++ b/hub/src/fcm/fcmService.ts @@ -29,6 +29,11 @@ export type FcmDataPayload = { * when the agent did not emit a summary. */ notifySummary?: string + /** + * Coalescing identity. Present on `model-error` as + * `model-error--` so distinct errors do not overwrite. + */ + tag?: string } export type FcmSendPayload = { @@ -178,6 +183,11 @@ export class FcmService { if (payload.data.notifySummary) { dataRecord.notifySummary = payload.data.notifySummary } + if (payload.tag) { + dataRecord.tag = payload.tag + } else if (payload.data.tag) { + dataRecord.tag = payload.data.tag + } // Data-only: if we also send `notification`, Android does not call // onMessageReceived while backgrounded — Wear relay never runs. diff --git a/hub/src/notifications/modelErrorCopy.test.ts b/hub/src/notifications/modelErrorCopy.test.ts new file mode 100644 index 0000000000..752a15aad7 --- /dev/null +++ b/hub/src/notifications/modelErrorCopy.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'bun:test' +import type { ModelErrorNotification } from './notificationTypes' +import { formatModelErrorBody, formatModelErrorTitle } from './modelErrorCopy' + +const baseNotification = (overrides: Partial = {}): ModelErrorNotification => ({ + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'Error: T: [resource_exhausted] capacity exceeded for the day', + priorAssistantClaimsDone: false, + eventId: 'evt-copy-1', + atTs: 1700000000000, + ...overrides +}) + +describe('formatModelErrorTitle', () => { + it('returns kind-specific titles for known kinds', () => { + expect(formatModelErrorTitle('quota_exhausted')).toBe('Quota exhausted') + expect(formatModelErrorTitle('rate_limited')).toBe('Rate limited') + expect(formatModelErrorTitle('transport_closed')).toBe('Agent transport closed') + expect(formatModelErrorTitle('agent_crashed')).toBe('Agent crashed') + expect(formatModelErrorTitle('rpc_timeout')).toBe('Agent request timed out') + expect(formatModelErrorTitle('context_window')).toBe('Context window exceeded') + }) + + it('falls back to generic Model error for unknown kinds', () => { + expect(formatModelErrorTitle('unknown_t_prefix')).toBe('Model error') + expect(formatModelErrorTitle('unknown_stderr')).toBe('Model error') + expect(formatModelErrorTitle('something_we_have_not_seen')).toBe('Model error') + }) +}) + +describe('formatModelErrorBody', () => { + const ctx = { agentName: 'Cursor', sessionName: 'feature-x' } + + it('leads with the lying-completion warning when priorAssistantClaimsDone', () => { + const body = formatModelErrorBody( + baseNotification({ priorAssistantClaimsDone: true }), + ctx + ) + const firstLine = body.split('\n')[0] + expect(firstLine).toContain('claimed completion') + expect(firstLine).toContain('INCOMPLETE') + }) + + it('omits the warning line when prior claim is false', () => { + const body = formatModelErrorBody( + baseNotification({ priorAssistantClaimsDone: false }), + ctx + ) + expect(body).not.toContain('claimed completion') + expect(body).toContain('Cursor - feature-x') + }) + + it('appends the transient hint when transient', () => { + const body = formatModelErrorBody( + baseNotification({ transient: true }), + ctx + ) + expect(body).toContain('(transient - safe to retry)') + }) + + it('omits the transient hint when not transient', () => { + const body = formatModelErrorBody( + baseNotification({ transient: false }), + ctx + ) + expect(body).not.toContain('transient') + }) + + it('omits rawSnippet from external notification bodies', () => { + const secretish = 'Error: T: [auth_failed] Bearer sk-live-DO-NOT-LEAK path=/home/op/.secrets' + const body = formatModelErrorBody( + baseNotification({ rawSnippet: secretish }), + ctx + ) + expect(body).not.toContain(secretish) + expect(body).not.toContain('sk-live') + expect(body).not.toContain('resource_exhausted') + expect(body).toContain('Cursor - feature-x') + }) + + it('still omits raw even when it is long or multiline', () => { + const body = formatModelErrorBody( + baseNotification({ rawSnippet: `line 1\n\n ${'A'.repeat(500)}` }), + ctx + ) + expect(body).not.toContain('AAAA') + expect(body).not.toContain('line 1') + expect(body.split('\n')).toEqual(['Cursor - feature-x']) + }) +}) diff --git a/hub/src/notifications/modelErrorCopy.ts b/hub/src/notifications/modelErrorCopy.ts new file mode 100644 index 0000000000..3180a75846 --- /dev/null +++ b/hub/src/notifications/modelErrorCopy.ts @@ -0,0 +1,63 @@ +import type { ModelErrorNotification } from './notificationTypes' + +/** + * Map model-error kinds to human-readable titles. Shared by all + * notification channels (FCM / Web Push / Telegram) so the wrist + * glance, browser toast, and chat message all read the same. + * + * Title is the GLANCE line: short, scannable, kind-specific. The body + * (separate) carries the priorAssistantClaimsDone alert and agent + + * session names. Raw provider/RPC text stays in the authenticated web + * banner only — never in push/Telegram (prompt text, paths, auth crumbs). + * + * Unknown kinds fall through to "Model error" so we never ship a + * notification with `[object Object]` or an internal kind string. + */ +export function formatModelErrorTitle(kind: string): string { + switch (kind) { + case 'quota_exhausted': return 'Quota exhausted' + case 'rate_limited': return 'Rate limited' + case 'capacity_exhausted': return 'Capacity exhausted' + case 'context_window': return 'Context window exceeded' + case 'auth_failed': return 'Authentication failed' + case 'model_not_found': return 'Model not found' + case 'transport_closed': return 'Agent transport closed' + case 'agent_crashed': return 'Agent crashed' + case 'rpc_timeout': return 'Agent request timed out' + case 'connection_stalled': return 'Connection stalled' + case 'deadline_exceeded': return 'Deadline exceeded' + case 'unavailable': return 'Service unavailable' + case 'canceled': return 'Agent canceled' + case 'prompt_failed': return 'Prompt failed' + case 'unknown_stderr': + case 'unknown_t_prefix': + default: return 'Model error' + } +} + +/** + * Body line strategy (external channels only — Web Push / Telegram / FCM): + * - If priorAssistantClaimsDone, lead with the lying-completion warning + * ("agent claimed completion before this error -- work likely + * INCOMPLETE"). This is the high-value disambiguator from the + * operator's POV: an "all done" green dot followed by an error means + * the agent walked away from a half-finished task. + * - Append agent/session context. + * - Do NOT append rawSnippet: that text is provider/RPC stderr and can + * contain prompt fragments, paths, or auth material. Operators see the + * full excerpt in the authenticated ModelErrorBanner. + */ +export function formatModelErrorBody( + notification: ModelErrorNotification, + context: { agentName: string; sessionName: string } +): string { + const lines: string[] = [] + if (notification.priorAssistantClaimsDone) { + lines.push('Agent claimed completion before this error - work likely INCOMPLETE.') + } + lines.push(`${context.agentName} - ${context.sessionName}`) + if (notification.transient) { + lines.push('(transient - safe to retry)') + } + return lines.join('\n') +} diff --git a/hub/src/notifications/notificationHub.test.ts b/hub/src/notifications/notificationHub.test.ts index c28443882f..69a3ca9afe 100644 --- a/hub/src/notifications/notificationHub.test.ts +++ b/hub/src/notifications/notificationHub.test.ts @@ -1,7 +1,12 @@ import { describe, expect, it } from 'bun:test' import type { Session, SyncEvent, SyncEventListener, SyncEngine } from '../sync/syncEngine' import type { SessionEndReason } from '@hapi/protocol' -import type { NotificationChannel, TaskNotification } from './notificationTypes' +import type { + ModelErrorNotification, + ModelErrorSendOutcome, + NotificationChannel, + TaskNotification +} from './notificationTypes' import { NotificationHub } from './notificationHub' const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) @@ -9,6 +14,9 @@ const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) class FakeSyncEngine { private readonly listeners: Set = new Set() private readonly sessions: Map = new Map() + readonly modelErrorNotifiedMarks: Array<{ sessionId: string; eventId: string }> = [] + /** When > 0, next N markModelErrorNotified calls throw before succeeding. */ + markModelErrorNotifiedFailuresRemaining = 0 subscribe(listener: SyncEventListener): () => void { this.listeners.add(listener) @@ -19,10 +27,37 @@ class FakeSyncEngine { return this.sessions.get(sessionId) } + getSessions(): Session[] { + return Array.from(this.sessions.values()) + } + setSession(session: Session): void { this.sessions.set(session.id, session) } + async markModelErrorNotified(sessionId: string, eventId: string): Promise { + if (this.markModelErrorNotifiedFailuresRemaining > 0) { + this.markModelErrorNotifiedFailuresRemaining -= 1 + throw new Error('version conflict') + } + this.modelErrorNotifiedMarks.push({ sessionId, eventId }) + const session = this.sessions.get(sessionId) + const err = session?.metadata?.lastModelError + if (!session || !err || err.eventId !== eventId) { + return + } + this.sessions.set(sessionId, { + ...session, + metadata: { + ...session.metadata!, + lastModelError: { + ...err, + notifiedAt: Date.now() + } + } + }) + } + emit(event: SyncEvent): void { for (const listener of this.listeners) { listener(event) @@ -35,6 +70,7 @@ class StubChannel implements NotificationChannel { readonly permissionSessions: Session[] = [] readonly taskNotifications: Array<{ session: Session; notification: TaskNotification }> = [] readonly sessionCompletions: Session[] = [] + readonly modelErrors: Array<{ session: Session; notification: ModelErrorNotification }> = [] async sendReady(session: Session): Promise { this.readySessions.push(session) @@ -51,6 +87,11 @@ class StubChannel implements NotificationChannel { async sendSessionCompletion(session: Session): Promise { this.sessionCompletions.push(session) } + + async sendModelError(session: Session, notification: ModelErrorNotification): Promise { + this.modelErrors.push({ session, notification }) + return 'delivered' + } } function createSession(overrides: Partial = {}): Session { @@ -245,4 +286,660 @@ describe('NotificationHub', () => { hub.stop() }) + + it('fires model-error notification when lastModelError.atTs advances', async () => { + const engine = new FakeSyncEngine() + const channel = new StubChannel() + const hub = new NotificationHub(engine as unknown as SyncEngine, [channel]) + + const session = createSession({ + metadata: { + lastModelError: { + eventId: 'evt-1000', + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'Error: T: [resource_exhausted] capacity exceeded', + atTs: 1000, + priorAssistantClaimsDone: true + } + } as Session['metadata'] + }) + + engine.setSession(session) + engine.emit({ type: 'session-updated', sessionId: session.id }) + await sleep(5) + + expect(channel.modelErrors).toHaveLength(1) + const fired = channel.modelErrors[0]?.notification + expect(fired?.kind).toBe('quota_exhausted') + expect(fired?.transient).toBe(false) + expect(fired?.priorAssistantClaimsDone).toBe(true) + expect(fired?.atTs).toBe(1000) + + hub.stop() + }) + + it('dedupes model-error notifications across repeat session-updated events', async () => { + const engine = new FakeSyncEngine() + const channel = new StubChannel() + const hub = new NotificationHub(engine as unknown as SyncEngine, [channel]) + + const session = createSession({ + metadata: { + lastModelError: { + eventId: 'evt-2000', + kind: 'transport_closed', + transient: true, + rawSnippet: 'WritableIterable is closed', + atTs: 2000, + priorAssistantClaimsDone: false + } + } as Session['metadata'] + }) + + engine.setSession(session) + engine.emit({ type: 'session-updated', sessionId: session.id }) + engine.emit({ type: 'session-updated', sessionId: session.id }) + engine.emit({ type: 'session-updated', sessionId: session.id }) + await sleep(5) + + expect(channel.modelErrors).toHaveLength(1) + + hub.stop() + }) + + it('fires again when a NEW lastModelError replaces an older one', async () => { + const engine = new FakeSyncEngine() + const channel = new StubChannel() + const hub = new NotificationHub(engine as unknown as SyncEngine, [channel]) + + const firstSession = createSession({ + metadata: { + lastModelError: { + eventId: 'evt-1000', + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'first', + atTs: 1000, + priorAssistantClaimsDone: false + } + } as Session['metadata'] + }) + engine.setSession(firstSession) + engine.emit({ type: 'session-updated', sessionId: firstSession.id }) + await sleep(5) + expect(channel.modelErrors).toHaveLength(1) + + const secondSession = createSession({ + metadata: { + lastModelError: { + eventId: 'evt-2000', + kind: 'transport_closed', + transient: true, + rawSnippet: 'second', + atTs: 2000, + priorAssistantClaimsDone: false + } + } as Session['metadata'] + }) + engine.setSession(secondSession) + engine.emit({ type: 'session-updated', sessionId: secondSession.id }) + await sleep(5) + + expect(channel.modelErrors).toHaveLength(2) + expect(channel.modelErrors[1]?.notification.atTs).toBe(2000) + + hub.stop() + }) + + it('rehydrates undelivered model-error for inactive sessions on hub construct', async () => { + const engine = new FakeSyncEngine() + const channel = new StubChannel() + const session = createSession({ + active: false, + metadata: { + lastModelError: { + eventId: 'evt-4500', + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'Error: T: [resource_exhausted]', + atTs: 4500, + priorAssistantClaimsDone: true + } + } as Session['metadata'] + }) + // Seed before hub exists — mirrors SyncEngine.reloadAll() before + // NotificationHub is constructed in startHub. + engine.setSession(session) + + const hub = new NotificationHub(engine as unknown as SyncEngine, [channel]) + await sleep(10) + + expect(channel.modelErrors).toHaveLength(1) + expect(channel.modelErrors[0]?.notification.atTs).toBe(4500) + expect(engine.modelErrorNotifiedMarks).toEqual([{ sessionId: session.id, eventId: 'evt-4500' }]) + hub.stop() + }) + + it('persists notifiedAt after successful delivery and skips after hub restart', async () => { + const engine = new FakeSyncEngine() + const channel = new StubChannel() + const hub = new NotificationHub(engine as unknown as SyncEngine, [channel]) + + const session = createSession({ + metadata: { + lastModelError: { + eventId: 'evt-4000', + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'Error: T: [resource_exhausted]', + atTs: 4000, + priorAssistantClaimsDone: false + } + } as Session['metadata'] + }) + engine.setSession(session) + engine.emit({ type: 'session-updated', sessionId: session.id }) + await sleep(10) + + expect(channel.modelErrors).toHaveLength(1) + expect(engine.modelErrorNotifiedMarks).toEqual([{ sessionId: session.id, eventId: 'evt-4000' }]) + expect(engine.getSession(session.id)?.metadata?.lastModelError?.notifiedAt).toEqual( + expect.any(Number) + ) + hub.stop() + + // Fresh hub = lost in-memory watermark; durable notifiedAt must gate. + const channel2 = new StubChannel() + const hub2 = new NotificationHub(engine as unknown as SyncEngine, [channel2]) + engine.emit({ type: 'session-updated', sessionId: session.id }) + await sleep(10) + expect(channel2.modelErrors).toHaveLength(0) + hub2.stop() + }) + + it('does not fire model-error for already-acknowledged errors', async () => { + const engine = new FakeSyncEngine() + const channel = new StubChannel() + const hub = new NotificationHub(engine as unknown as SyncEngine, [channel]) + + const session = createSession({ + metadata: { + lastModelError: { + eventId: 'evt-1000', + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'first', + atTs: 1000, + priorAssistantClaimsDone: false, + acknowledgedAt: 1500 + } + } as Session['metadata'] + }) + engine.setSession(session) + engine.emit({ type: 'session-updated', sessionId: session.id }) + await sleep(5) + + expect(channel.modelErrors).toHaveLength(0) + + hub.stop() + }) + + it('skips model-error dispatch when no channels implement it', async () => { + const engine = new FakeSyncEngine() + // Channel WITHOUT sendModelError -- should silently skip, no throw. + const minimalChannel: NotificationChannel = { + async sendReady() {}, + async sendPermissionRequest() {}, + async sendTaskNotification() {} + } + const hub = new NotificationHub(engine as unknown as SyncEngine, [minimalChannel]) + + const session = createSession({ + metadata: { + lastModelError: { + eventId: 'evt-3000', + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'no-channel', + atTs: 3000, + priorAssistantClaimsDone: false + } + } as Session['metadata'] + }) + engine.setSession(session) + engine.emit({ type: 'session-updated', sessionId: session.id }) + await sleep(5) + + // No assertions other than "didn't throw"; the channel has no + // recording surface. Test passes if hub.stop() returns cleanly. + hub.stop() + }) + + it('shares nativeGate across model-error channels so later channels can defer', async () => { + const engine = new FakeSyncEngine() + const fcmCalls: ModelErrorNotification[] = [] + const pushCalls: ModelErrorNotification[] = [] + + const fcmChannel: NotificationChannel = { + async sendReady() {}, + async sendPermissionRequest() {}, + async sendTaskNotification() {}, + async sendModelError(_session, notification, ctx) { + fcmCalls.push(notification) + if (ctx?.nativeGate) { + ctx.nativeGate.sent = true + } + return 'delivered' + } + } + const pushChannel: NotificationChannel = { + async sendReady() {}, + async sendPermissionRequest() {}, + async sendTaskNotification() {}, + async sendModelError(_session, notification, ctx) { + if (ctx?.nativeGate?.sent) { + return 'unavailable' + } + pushCalls.push(notification) + return 'delivered' + } + } + + const hub = new NotificationHub(engine as unknown as SyncEngine, [fcmChannel, pushChannel]) + const session = createSession({ + metadata: { + lastModelError: { + eventId: 'evt-4000', + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'gate-test', + atTs: 4000, + priorAssistantClaimsDone: false + } + } as Session['metadata'] + }) + engine.setSession(session) + engine.emit({ type: 'session-updated', sessionId: session.id }) + await sleep(5) + + expect(fcmCalls).toHaveLength(1) + expect(pushCalls).toHaveLength(0) + + hub.stop() + }) + + it('falls back to later model-error channels when native gate stays unset', async () => { + const engine = new FakeSyncEngine() + const pushCalls: ModelErrorNotification[] = [] + + const fcmChannel: NotificationChannel = { + async sendReady() {}, + async sendPermissionRequest() {}, + async sendTaskNotification() {}, + async sendModelError() { + // Delivered zero - leave nativeGate.sent false + return 'failed' + } + } + const pushChannel: NotificationChannel = { + async sendReady() {}, + async sendPermissionRequest() {}, + async sendTaskNotification() {}, + async sendModelError(_session, notification, ctx) { + if (ctx?.nativeGate?.sent) { + return 'unavailable' + } + pushCalls.push(notification) + return 'delivered' + } + } + + const hub = new NotificationHub(engine as unknown as SyncEngine, [fcmChannel, pushChannel]) + const session = createSession({ + metadata: { + lastModelError: { + eventId: 'evt-5000', + kind: 'rate_limited', + transient: true, + rawSnippet: 'fallback-test', + atTs: 5000, + priorAssistantClaimsDone: false + } + } as Session['metadata'] + }) + engine.setSession(session) + engine.emit({ type: 'session-updated', sessionId: session.id }) + await sleep(5) + + expect(pushCalls).toHaveLength(1) + expect(pushCalls[0]?.atTs).toBe(5000) + + hub.stop() + }) + + it('schedules bounded backoff retry when every channel throws (keeps watermark)', async () => { + const engine = new FakeSyncEngine() + let attempts = 0 + const channel: NotificationChannel = { + async sendReady() {}, + async sendPermissionRequest() {}, + async sendTaskNotification() {}, + async sendModelError() { + attempts++ + throw new Error('transient outage') + } + } + const hub = new NotificationHub(engine as unknown as SyncEngine, [channel], { + modelErrorRetryDelaysMs: [25] + }) + const session = createSession({ + metadata: { + lastModelError: { + eventId: 'evt-6000', + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'retry-me', + atTs: 6000, + priorAssistantClaimsDone: false + } + } as Session['metadata'] + }) + engine.setSession(session) + engine.emit({ type: 'session-updated', sessionId: session.id }) + await sleep(10) + expect(attempts).toBe(1) + + // Keepalive session-updated must NOT storm - watermark stays. + engine.emit({ type: 'session-updated', sessionId: session.id }) + await sleep(5) + expect(attempts).toBe(1) + + // Backoff timer fires the retry. + await sleep(35) + expect(attempts).toBe(2) + + hub.stop() + }) + + it('schedules backoff retry when channels resolve with failed (zero deliveries)', async () => { + const engine = new FakeSyncEngine() + let attempts = 0 + const channel: NotificationChannel = { + async sendReady() {}, + async sendPermissionRequest() {}, + async sendTaskNotification() {}, + async sendModelError() { + attempts++ + return 'failed' + } + } + const hub = new NotificationHub(engine as unknown as SyncEngine, [channel], { + modelErrorRetryDelaysMs: [15] + }) + const session = createSession({ + metadata: { + lastModelError: { + eventId: 'evt-6500', + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'zero-sent', + atTs: 6500, + priorAssistantClaimsDone: false + } + } as Session['metadata'] + }) + engine.setSession(session) + engine.emit({ type: 'session-updated', sessionId: session.id }) + await sleep(5) + expect(attempts).toBe(1) + + engine.emit({ type: 'session-updated', sessionId: session.id }) + await sleep(5) + expect(attempts).toBe(1) + + await sleep(25) + expect(attempts).toBe(2) + + hub.stop() + }) + + it('retries model-error for inactive sessions via backoff timer', async () => { + const engine = new FakeSyncEngine() + let attempts = 0 + const channel: NotificationChannel = { + async sendReady() {}, + async sendPermissionRequest() {}, + async sendTaskNotification() {}, + async sendModelError(session) { + attempts++ + // Production channels used to return unavailable when !active — + // that marked retries "completed" and dropped the ping. + if (attempts === 1) { + expect(session.active).toBe(true) + return 'failed' + } + expect(session.active).toBe(false) + return 'delivered' + } + } + const hub = new NotificationHub(engine as unknown as SyncEngine, [channel], { + modelErrorRetryDelaysMs: [20] + }) + const session = createSession({ + metadata: { + lastModelError: { + eventId: 'evt-6600', + kind: 'rate_limited', + transient: true, + rawSnippet: 'status 429', + atTs: 6600, + priorAssistantClaimsDone: false + } + } as Session['metadata'] + }) + engine.setSession(session) + engine.emit({ type: 'session-updated', sessionId: session.id }) + await sleep(5) + expect(attempts).toBe(1) + + // Go inactive before the retry - timer must still fire AND deliver. + engine.setSession({ ...session, active: false }) + engine.emit({ type: 'session-updated', sessionId: session.id }) + await sleep(30) + expect(attempts).toBe(2) + + hub.stop() + }) + + it('does not let an obsolete retry schedule over a newer atTs', async () => { + const engine = new FakeSyncEngine() + const outcomes: number[] = [] + const channel: NotificationChannel = { + async sendReady() {}, + async sendPermissionRequest() {}, + async sendTaskNotification() {}, + async sendModelError(_session, notification) { + outcomes.push(notification.atTs) + // First error always fails; second succeeds. + return notification.atTs === 7000 ? 'failed' : 'delivered' + } + } + const hub = new NotificationHub(engine as unknown as SyncEngine, [channel], { + modelErrorRetryDelaysMs: [40] + }) + + const first = createSession({ + id: 'session-1', + metadata: { + lastModelError: { + eventId: 'evt-7000', + kind: 'canceled', + transient: true, + rawSnippet: 'first', + atTs: 7000, + priorAssistantClaimsDone: false + } + } as Session['metadata'] + }) + engine.setSession(first) + engine.emit({ type: 'session-updated', sessionId: first.id }) + await sleep(5) + expect(outcomes).toEqual([7000]) + + // Newer error arrives while first retry is pending. + const second = { + ...first, + metadata: { + lastModelError: { + eventId: 'evt-8000', + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'second', + atTs: 8000, + priorAssistantClaimsDone: false + } + } as Session['metadata'] + } + engine.setSession(second) + engine.emit({ type: 'session-updated', sessionId: second.id }) + await sleep(5) + expect(outcomes).toEqual([7000, 8000]) + + // Obsolete timer for 7000 must not steal / block retries for 8000. + // Force 8000 to fail once so it needs its own retry, then wait. + // (8000 already delivered above — verify stale timer is a no-op.) + await sleep(50) + expect(outcomes.filter((ts) => ts === 7000)).toHaveLength(1) + expect(outcomes.filter((ts) => ts === 8000)).toHaveLength(1) + + hub.stop() + }) + + it('does not re-fire model-error after inactive/resume for the same atTs', async () => { + const engine = new FakeSyncEngine() + const channel = new StubChannel() + const hub = new NotificationHub(engine as unknown as SyncEngine, [channel]) + const session = createSession({ + metadata: { + lastModelError: { + eventId: 'evt-7000', + kind: 'rate_limited', + transient: true, + rawSnippet: 'status 429', + atTs: 7000, + priorAssistantClaimsDone: false + } + } as Session['metadata'] + }) + engine.setSession(session) + engine.emit({ type: 'session-updated', sessionId: session.id }) + await sleep(10) + expect(channel.modelErrors).toHaveLength(1) + + // Become inactive (clears timers but keeps watermark). + engine.setSession({ ...session, active: false }) + engine.emit({ type: 'session-updated', sessionId: session.id }) + await sleep(5) + + // Resume with same unacknowledged eventId - must not re-ping. + engine.setSession({ ...session, active: true }) + engine.emit({ type: 'session-updated', sessionId: session.id }) + await sleep(10) + expect(channel.modelErrors).toHaveLength(1) + + hub.stop() + }) + + it('fires a later eventId even when wall-clock atTs went backwards', async () => { + const engine = new FakeSyncEngine() + const channel = new StubChannel() + const hub = new NotificationHub(engine as unknown as SyncEngine, [channel]) + + const first = createSession({ + metadata: { + lastModelError: { + eventId: 'evt-clock-high', + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'first', + atTs: 5_000, + priorAssistantClaimsDone: false + } + } as Session['metadata'] + }) + engine.setSession(first) + engine.emit({ type: 'session-updated', sessionId: first.id }) + await sleep(5) + expect(channel.modelErrors).toHaveLength(1) + + // NTP/sleep rewind: newer logical error has a lower atTs. + const second = createSession({ + metadata: { + lastModelError: { + eventId: 'evt-clock-low', + kind: 'transport_closed', + transient: true, + rawSnippet: 'second', + atTs: 1_000, + priorAssistantClaimsDone: false + } + } as Session['metadata'] + }) + engine.setSession(second) + engine.emit({ type: 'session-updated', sessionId: second.id }) + await sleep(5) + + expect(channel.modelErrors).toHaveLength(2) + expect(channel.modelErrors[1]?.notification.eventId).toBe('evt-clock-low') + expect(channel.modelErrors[1]?.notification.atTs).toBe(1_000) + + hub.stop() + }) + + it('retries notifiedAt persistence after contention without resending channels', async () => { + const engine = new FakeSyncEngine() + engine.markModelErrorNotifiedFailuresRemaining = 1 + let sends = 0 + const channel: NotificationChannel = { + async sendReady() {}, + async sendPermissionRequest() {}, + async sendTaskNotification() {}, + async sendModelError() { + sends++ + return 'delivered' + } + } + const hub = new NotificationHub(engine as unknown as SyncEngine, [channel], { + modelErrorRetryDelaysMs: [20] + }) + const session = createSession({ + metadata: { + lastModelError: { + eventId: 'evt-wm-1', + kind: 'rate_limited', + transient: true, + rawSnippet: 'wm-retry', + atTs: 9000, + priorAssistantClaimsDone: false + } + } as Session['metadata'] + }) + engine.setSession(session) + engine.emit({ type: 'session-updated', sessionId: session.id }) + await sleep(10) + expect(sends).toBe(1) + expect(engine.modelErrorNotifiedMarks).toHaveLength(0) + + await sleep(40) + expect(sends).toBe(1) + expect(engine.modelErrorNotifiedMarks).toEqual([ + { sessionId: session.id, eventId: 'evt-wm-1' } + ]) + expect(engine.getSession(session.id)?.metadata?.lastModelError?.notifiedAt).toEqual( + expect.any(Number) + ) + + hub.stop() + }) }) diff --git a/hub/src/notifications/notificationHub.ts b/hub/src/notifications/notificationHub.ts index f9d627e1e7..174b422d02 100644 --- a/hub/src/notifications/notificationHub.ts +++ b/hub/src/notifications/notificationHub.ts @@ -1,6 +1,11 @@ import type { Session, SyncEngine, SyncEvent } from '../sync/syncEngine' import type { SessionEndReason } from '@hapi/protocol' -import type { NotificationChannel, NotificationHubOptions, TaskNotification } from './notificationTypes' +import type { + ModelErrorNotification, + NotificationChannel, + NotificationHubOptions, + TaskNotification +} from './notificationTypes' import type { NotificationSendContext } from './notificationSendContext' import { extractMessageEventType, extractTaskNotification } from './eventParsing' @@ -8,10 +13,27 @@ export class NotificationHub { private readonly channels: NotificationChannel[] private readonly readyCooldownMs: number private readonly permissionDebounceMs: number + private readonly modelErrorRetryDelaysMs: number[] private readonly lastKnownRequests: Map> = new Map() private readonly notificationDebounce: Map = new Map() private readonly lastReadyNotificationAt: Map = new Map() + /** + * sessionId -> the `eventId` of the last `lastModelError` we already + * notified for. Fire ONCE per distinct error event; subsequent + * `session-updated` storms must not re-trigger. Identity is eventId + * (CLI UUID), not wall-clock atTs — clocks can move backwards. + * + * Kept across failed deliveries while a bounded backoff retry timer + * is pending — do not delete it to "retry on next session-updated" + * (that storms on keepalive and misses inactive sessions). + */ + private readonly lastModelErrorNotifiedId: Map = new Map() + private readonly modelErrorRetryTimers: Map = new Map() + private readonly modelErrorRetryAttempts: Map = new Map() + /** In-flight notifiedAt persistence tasks (delivery already succeeded). */ + private readonly modelErrorWatermarkTasks: Map> = new Map() private unsubscribeSyncEvents: (() => void) | null = null + private stopped = false constructor( private readonly syncEngine: SyncEngine, @@ -21,12 +43,33 @@ export class NotificationHub { this.channels = channels this.readyCooldownMs = options?.readyCooldownMs ?? 5000 this.permissionDebounceMs = options?.permissionDebounceMs ?? 500 + this.modelErrorRetryDelaysMs = options?.modelErrorRetryDelaysMs + ?? [5_000, 15_000, 45_000, 120_000] this.unsubscribeSyncEvents = this.syncEngine.subscribe((event) => { this.handleSyncEvent(event) }) + // SyncEngine.reloadAll() emits session-added before NotificationHub + // exists. Rehydrate any undelivered model-error alerts (including + // inactive sessions) so a hub restart cannot drop a pending page. + this.rehydrateUndeliveredModelErrors() + } + + private rehydrateUndeliveredModelErrors(): void { + for (const session of this.syncEngine.getSessions()) { + const error = session.metadata?.lastModelError + if ( + !error + || typeof error.acknowledgedAt === 'number' + || typeof error.notifiedAt === 'number' + ) { + continue + } + this.checkForModelErrorNotification(session) + } } stop(): void { + this.stopped = true if (this.unsubscribeSyncEvents) { this.unsubscribeSyncEvents() this.unsubscribeSyncEvents = null @@ -38,21 +81,41 @@ export class NotificationHub { this.notificationDebounce.clear() this.lastKnownRequests.clear() this.lastReadyNotificationAt.clear() + this.lastModelErrorNotifiedId.clear() + for (const timer of this.modelErrorRetryTimers.values()) { + clearTimeout(timer) + } + this.modelErrorRetryTimers.clear() + this.modelErrorRetryAttempts.clear() + this.modelErrorWatermarkTasks.clear() } private handleSyncEvent(event: SyncEvent): void { if ((event.type === 'session-updated' || event.type === 'session-added') && event.sessionId) { const session = this.syncEngine.getSession(event.sessionId) - if (!session || !session.active) { - this.clearSessionState(event.sessionId) + if (!session) { + this.clearSessionState(event.sessionId, true) + return + } + if (!session.active) { + // Keep lastModelErrorNotifiedId across inactive/resume so the + // same eventId does not re-ping when the session comes back. + this.clearSessionState(event.sessionId, false) + // Still dispatch undelivered model-errors (no notifiedAt) — + // emergency pages must not wait for the session to become + // active again after a failed delivery + hub restart. + this.checkForModelErrorNotification(session) return } this.checkForPermissionNotification(session) + // Model-error gating: fire when metadata.lastModelError.eventId + // differs from what we last notified for this session. + this.checkForModelErrorNotification(session) return } if (event.type === 'session-removed' && event.sessionId) { - this.clearSessionState(event.sessionId) + this.clearSessionState(event.sessionId, true) return } @@ -82,7 +145,7 @@ export class NotificationHub { } } - private clearSessionState(sessionId: string): void { + private clearSessionState(sessionId: string, removeModelErrorWatermark = false): void { const existingTimer = this.notificationDebounce.get(sessionId) if (existingTimer) { clearTimeout(existingTimer) @@ -90,6 +153,184 @@ export class NotificationHub { } this.lastKnownRequests.delete(sessionId) this.lastReadyNotificationAt.delete(sessionId) + if (removeModelErrorWatermark) { + this.lastModelErrorNotifiedId.delete(sessionId) + this.clearModelErrorRetry(sessionId) + } + } + + private clearModelErrorRetry(sessionId: string): void { + const timer = this.modelErrorRetryTimers.get(sessionId) + if (timer) { + clearTimeout(timer) + } + this.modelErrorRetryTimers.delete(sessionId) + this.modelErrorRetryAttempts.delete(sessionId) + } + + private checkForModelErrorNotification(session: Session): void { + const lastModelError = session.metadata?.lastModelError + if ( + !lastModelError + || typeof lastModelError.eventId !== 'string' + || typeof lastModelError.atTs !== 'number' + ) { + return + } + // Don't ping for already-acknowledged errors. The web UI sets + // acknowledgedAt when the operator dismisses the banner; if they + // dismissed and the row gets re-emitted (e.g. a different metadata + // field changed), we don't want to re-ring the wrist. + if (typeof lastModelError.acknowledgedAt === 'number') { + this.clearModelErrorRetry(session.id) + return + } + // Durable watermark (survives hub restart). Seed the in-memory map + // so concurrent session-updated storms stay quiet in this process. + if (typeof lastModelError.notifiedAt === 'number') { + this.lastModelErrorNotifiedId.set(session.id, lastModelError.eventId) + this.clearModelErrorRetry(session.id) + return + } + const lastNotifiedId = this.lastModelErrorNotifiedId.get(session.id) + if (lastNotifiedId === lastModelError.eventId) { + return + } + const eventId = lastModelError.eventId + const atTs = lastModelError.atTs + // New error supersedes any in-flight retry for an older eventId. + this.clearModelErrorRetry(session.id) + // Optimistic watermark: prevents concurrent session-updated storms + // from double-firing. On delivery failure we KEEP it and schedule a + // bounded backoff retry (do not rely on keepalive session-updated). + this.lastModelErrorNotifiedId.set(session.id, eventId) + + const notification: ModelErrorNotification = { + kind: lastModelError.kind, + transient: lastModelError.transient, + rawSnippet: lastModelError.rawSnippet, + priorAssistantClaimsDone: Boolean(lastModelError.priorAssistantClaimsDone), + eventId, + atTs + } + + void this.notifyModelError(session, notification).then(async (completed) => { + if (this.lastModelErrorNotifiedId.get(session.id) !== eventId) { + return + } + if (completed) { + this.persistModelErrorWatermark(session.id, eventId) + return + } + this.scheduleModelErrorRetry(session.id, eventId) + }).catch((error) => { + console.error('[NotificationHub] Failed to send model-error notification:', error) + if (this.lastModelErrorNotifiedId.get(session.id) === eventId) { + this.scheduleModelErrorRetry(session.id, eventId) + } + }) + } + + /** + * Delivery already succeeded — retry only the durable notifiedAt write so + * a hub restart cannot re-page. Never re-invokes channels. + */ + private persistModelErrorWatermark(sessionId: string, eventId: string): void { + const task = this.runPersistModelErrorWatermark(sessionId, eventId) + this.modelErrorWatermarkTasks.set(sessionId, task) + void task.finally(() => { + if (this.modelErrorWatermarkTasks.get(sessionId) === task) { + this.modelErrorWatermarkTasks.delete(sessionId) + } + }) + } + + private async runPersistModelErrorWatermark(sessionId: string, eventId: string): Promise { + for (const delayMs of [0, ...this.modelErrorRetryDelaysMs]) { + if (delayMs > 0) { + await new Promise((resolve) => setTimeout(resolve, delayMs)) + } + if (this.stopped) { + return + } + if (this.lastModelErrorNotifiedId.get(sessionId) !== eventId) { + return + } + try { + await this.syncEngine.markModelErrorNotified(sessionId, eventId) + return + } catch (error) { + console.error('[NotificationHub] Failed to persist model-error notifiedAt:', error) + } + } + } + + private scheduleModelErrorRetry(sessionId: string, eventId: string): void { + if (this.modelErrorRetryTimers.has(sessionId)) { + return + } + const attempt = this.modelErrorRetryAttempts.get(sessionId) ?? 0 + if (attempt >= this.modelErrorRetryDelaysMs.length) { + console.error( + `[NotificationHub] Exhausted model-error delivery retries for session=${sessionId} eventId=${eventId}` + ) + return + } + const delayMs = this.modelErrorRetryDelaysMs[attempt] ?? 5_000 + this.modelErrorRetryAttempts.set(sessionId, attempt + 1) + const timer = setTimeout(() => { + this.modelErrorRetryTimers.delete(sessionId) + void this.retryModelErrorNotification(sessionId, eventId) + }, delayMs) + this.modelErrorRetryTimers.set(sessionId, timer) + } + + private async retryModelErrorNotification(sessionId: string, eventId: string): Promise { + const session = this.syncEngine.getSession(sessionId) + if (!session) { + this.clearModelErrorRetry(sessionId) + this.lastModelErrorNotifiedId.delete(sessionId) + return + } + const lastModelError = session.metadata?.lastModelError + if ( + !lastModelError + || lastModelError.eventId !== eventId + || typeof lastModelError.acknowledgedAt === 'number' + ) { + this.clearModelErrorRetry(sessionId) + return + } + + const notification: ModelErrorNotification = { + kind: lastModelError.kind, + transient: lastModelError.transient, + rawSnippet: lastModelError.rawSnippet, + priorAssistantClaimsDone: Boolean(lastModelError.priorAssistantClaimsDone), + eventId, + atTs: lastModelError.atTs + } + + try { + const completed = await this.notifyModelError(session, notification) + // A newer eventId may have taken the watermark while we were in flight. + // Do not install an obsolete retry that blocks the latest error. + if (this.lastModelErrorNotifiedId.get(sessionId) !== eventId) { + return + } + if (completed) { + this.modelErrorRetryAttempts.delete(sessionId) + this.persistModelErrorWatermark(sessionId, eventId) + return + } + this.scheduleModelErrorRetry(sessionId, eventId) + } catch (error) { + console.error('[NotificationHub] Failed to retry model-error notification:', error) + if (this.lastModelErrorNotifiedId.get(sessionId) !== eventId) { + return + } + this.scheduleModelErrorRetry(sessionId, eventId) + } } private getNotifiableSession(sessionId: string): Session | null { @@ -227,4 +468,30 @@ export class NotificationHub { } } } + + private async notifyModelError(session: Session, notification: ModelErrorNotification): Promise { + const ctx: NotificationSendContext = { nativeGate: { sent: false } } + let attempted = false + let delivered = false + for (const channel of this.channels) { + if (typeof channel.sendModelError !== 'function') { + continue + } + try { + const outcome = await channel.sendModelError(session, notification, ctx) + if (outcome !== 'unavailable') { + attempted = true + } + if (outcome === 'delivered') { + delivered = true + } + } catch (error) { + attempted = true + console.error('[NotificationHub] Failed to send model-error notification:', error) + } + } + // No implementers / all unavailable: keep watermark (avoid retry storm). + // At least one channel tried and none delivered: roll back for retry. + return !attempted || delivered + } } diff --git a/hub/src/notifications/notificationTypes.ts b/hub/src/notifications/notificationTypes.ts index 6130cf30bd..f4565dae1f 100644 --- a/hub/src/notifications/notificationTypes.ts +++ b/hub/src/notifications/notificationTypes.ts @@ -7,14 +7,64 @@ export type TaskNotification = { status?: string } +/** + * Model error notification: fires when cursor-agent (or another flavor's + * runtime) hits an internal model-side failure that HAPI detects either + * structurally (typed AcpStderrError, RPC rejection, transport close) or + * via the text-classifier fallback for stringified-into-prose errors. + * + * Higher urgency than ready/task: an operator who walks away from the + * web UI MUST get a phone-side / wrist-side ping for this, otherwise the + * "all done" green dot lies to them. Banner-only is opt-in (requires + * looking); notification is push (regardless of attention). + */ +export type ModelErrorNotification = { + kind: string // e.g. 'quota_exhausted', 'transport_closed' + transient: boolean // retryable hint (rate_limit / canceled / timeout) + rawSnippet: string // first 400 chars of the raw error text + priorAssistantClaimsDone: boolean // agent said "Done"/"Committed" right before the error + eventId: string // metadata.lastModelError.eventId — notify/ack identity + atTs: number // display / telemetry only (not monotonic identity) +} + +/** + * Outcome of a model-error channel send. Used by NotificationHub to decide + * whether to keep or roll back the per-session watermark: + * - delivered: at least one destination accepted the ping + * - unavailable: channel had nothing to do (no subs, deferred to native, inactive) + * - failed: channel tried and every destination failed + */ +export type ModelErrorSendOutcome = 'delivered' | 'unavailable' | 'failed' + export type NotificationChannel = { sendReady: (session: Session, ctx?: NotificationSendContext) => Promise sendPermissionRequest: (session: Session, ctx?: NotificationSendContext) => Promise sendTaskNotification: (session: Session, notification: TaskNotification, ctx?: NotificationSendContext) => Promise sendSessionCompletion?: (session: Session, reason: SessionEndReason) => Promise + /** + * Optional. Channels that don't implement it just skip model-error + * pings (matches sendSessionCompletion's pattern). Wire this when + * the channel can render a higher-urgency error variant. + * + * Pass the same NotificationSendContext as ready/permission/task so + * FCM can set nativeGate.sent and Web Push can defer (one OS ping). + * Return a real delivery outcome so the hub watermark is not consumed + * when every destination failed. + */ + sendModelError?: ( + session: Session, + notification: ModelErrorNotification, + ctx?: NotificationSendContext + ) => Promise } export type NotificationHubOptions = { readyCooldownMs?: number permissionDebounceMs?: number + /** + * Backoff delays (ms) for model-error delivery retries after a failed + * dispatch. Empty / omitted uses the default ladder. Exhausted delays + * keep the watermark so session-updated storms do not re-fire forever. + */ + modelErrorRetryDelaysMs?: number[] } diff --git a/hub/src/push-ios/iosPushChannel.test.ts b/hub/src/push-ios/iosPushChannel.test.ts index 388d9e33f3..fcc4b27e50 100644 --- a/hub/src/push-ios/iosPushChannel.test.ts +++ b/hub/src/push-ios/iosPushChannel.test.ts @@ -116,4 +116,26 @@ describe('IosPushNotificationChannel', () => { expect(payload.title).toBe('Task failed') expect(payload.body).toContain('exploded') }) + + it('sends model-error with event-specific tag and error severity', async () => { + const { channel, service } = makeChannel({ sent: 1, failed: 0, invalidTokens: [] }) + const ctx: NotificationSendContext = { nativeGate: { sent: false } } + + const outcome = await channel.sendModelError(createSession(), { + eventId: 'evt-1', + kind: 'rate_limited', + transient: true, + rawSnippet: 'status 429', + priorAssistantClaimsDone: false, + atTs: 1710000000000 + }, ctx) + + expect(outcome).toBe('delivered') + expect(ctx.nativeGate?.sent).toBe(true) + const payload = service.calls[0].payload + expect(payload.type).toBe('model-error') + expect(payload.severity).toBe('error') + expect(payload.tag).toBe('model-error-session-1-evt-1') + expect(payload.contractVersion).toBe('1') + }) }) diff --git a/hub/src/push-ios/iosPushChannel.ts b/hub/src/push-ios/iosPushChannel.ts index 639089613d..0452d0d1c1 100644 --- a/hub/src/push-ios/iosPushChannel.ts +++ b/hub/src/push-ios/iosPushChannel.ts @@ -1,7 +1,9 @@ import type { Session } from '../sync/syncEngine' -import type { NotificationChannel, TaskNotification } from '../notifications/notificationTypes' +import type { ModelErrorNotification, ModelErrorSendOutcome, NotificationChannel, TaskNotification } from '../notifications/notificationTypes' import type { NotificationSendContext } from '../notifications/notificationSendContext' import { NATIVE_CONTRACT_VERSION, NativeNotificationComposer, type ComposedNativeNotification } from '../notifications/nativeNotificationComposer' +import { formatModelErrorBody, formatModelErrorTitle } from '../notifications/modelErrorCopy' +import { getAgentName, getSessionName } from '../notifications/sessionInfo' import type { Store } from '../store' import type { IosPushNotificationPayload, IosPushService } from './iosPushService' @@ -47,6 +49,40 @@ export class IosPushNotificationChannel implements NotificationChannel { await this.deliver(session, this.composer.composeTask(session, notification), ctx) } + async sendModelError( + session: Session, + notification: ModelErrorNotification, + ctx?: NotificationSendContext + ): Promise { + const agentName = getAgentName(session) + const sessionName = getSessionName(session) + const title = formatModelErrorTitle(notification.kind) + const body = formatModelErrorBody(notification, { agentName, sessionName }) + const tag = `model-error-${session.id}-${notification.eventId}` + + const result = await this.iosPushService.sendToNamespace(session.namespace, { + type: 'model-error', + sessionId: session.id, + sessionName, + url: `/sessions/${session.id}`, + title, + body, + contractVersion: NATIVE_CONTRACT_VERSION, + severity: 'error', + tag + }) + if ((result?.sent ?? 0) > 0) { + if (ctx?.nativeGate) { + ctx.nativeGate.sent = true + } + return 'delivered' + } + if ((result?.failed ?? 0) === 0) { + return 'unavailable' + } + return 'failed' + } + private toPlaintextPayload(composed: ComposedNativeNotification): IosPushNotificationPayload { return { type: composed.type, diff --git a/hub/src/push-ios/iosPushService.test.ts b/hub/src/push-ios/iosPushService.test.ts index 63580f4283..9ef0be340e 100644 --- a/hub/src/push-ios/iosPushService.test.ts +++ b/hub/src/push-ios/iosPushService.test.ts @@ -186,6 +186,10 @@ describe('buildCollapseId', () => { expect(buildCollapseId('ready', 's1')).toBe('ready-s1') }) + it('prefers an explicit tag so model-error events do not collapse', () => { + expect(buildCollapseId('model-error', 's1', 'model-error-s1-evt-2')).toBe('model-error-s1-evt-2') + }) + it('truncates to 64 bytes', () => { const longSession = 'x'.repeat(100) const collapseId = buildCollapseId('permission-request', longSession) diff --git a/hub/src/push-ios/iosPushService.ts b/hub/src/push-ios/iosPushService.ts index ce0ede558e..b94442ddb9 100644 --- a/hub/src/push-ios/iosPushService.ts +++ b/hub/src/push-ios/iosPushService.ts @@ -19,6 +19,8 @@ export type IosPushNotificationPayload = { contractVersion: string requestId?: string notifySummary?: string + /** Coalescing identity; when set, used as APNs collapse-id. */ + tag?: string } export type IosPushSendResult = { @@ -33,8 +35,8 @@ const APNS_COLLAPSE_ID_MAX_BYTES = 64 * APNs collapse id: `-`, truncated to 64 bytes on a UTF-8 * character boundary (APNs rejects oversized collapse ids outright). */ -export function buildCollapseId(type: string, sessionId: string): string { - const raw = `${type}-${sessionId}` +export function buildCollapseId(type: string, sessionId: string, tag?: string): string { + const raw = tag && tag.length > 0 ? tag : `${type}-${sessionId}` if (Buffer.byteLength(raw, 'utf8') <= APNS_COLLAPSE_ID_MAX_BYTES) { return raw } @@ -91,7 +93,7 @@ export class IosPushService { } const plaintext = canonicalJson(payload) - const collapseId = buildCollapseId(payload.type, payload.sessionId) + const collapseId = buildCollapseId(payload.type, payload.sessionId, payload.tag) const invalidTokens: string[] = [] let sent = 0 diff --git a/hub/src/push/pushNotificationChannel.test.ts b/hub/src/push/pushNotificationChannel.test.ts index bd0d0c330b..4317ed94db 100644 --- a/hub/src/push/pushNotificationChannel.test.ts +++ b/hub/src/push/pushNotificationChannel.test.ts @@ -22,6 +22,7 @@ describe('PushNotificationChannel', () => { { sendToNamespace: async (namespace: string, payload: PushPayload) => { pushed.push({ namespace, payload }) + return { sent: 1, failed: 0, subscriptions: 1 } } } as never, { @@ -51,6 +52,7 @@ describe('PushNotificationChannel', () => { { sendToNamespace: async (namespace: string, payload: PushPayload) => { pushed.push({ namespace, payload }) + return { sent: 1, failed: 0, subscriptions: 1 } } } as never, { @@ -82,6 +84,7 @@ describe('PushNotificationChannel', () => { { sendToNamespace: async (namespace: string, payload: PushPayload) => { pushed.push({ namespace, payload }) + return { sent: 1, failed: 0, subscriptions: 1 } } } as never, { @@ -115,6 +118,7 @@ describe('PushNotificationChannel', () => { { sendToNamespace: async (namespace: string, payload: PushPayload) => { pushed.push({ namespace, payload }) + return { sent: 1, failed: 0, subscriptions: 1 } } } as never, { @@ -137,6 +141,7 @@ describe('PushNotificationChannel', () => { { sendToNamespace: async (namespace: string, payload: PushPayload) => { pushed.push({ namespace, payload }) + return { sent: 1, failed: 0, subscriptions: 1 } } } as never, { @@ -160,6 +165,7 @@ describe('PushNotificationChannel', () => { { sendToNamespace: async (namespace: string, payload: PushPayload) => { pushed.push({ namespace, payload }) + return { sent: 1, failed: 0, subscriptions: 1 } } } as never, { @@ -191,4 +197,57 @@ describe('PushNotificationChannel', () => { expect(toasts).toHaveLength(0) expect(pushed).toHaveLength(0) }) + + it('sendModelError defers when nativeGate.sent is true', async () => { + const pushed: Array<{ namespace: string; payload: PushPayload }> = [] + const channel = new PushNotificationChannel( + { + sendToNamespace: async (namespace: string, payload: PushPayload) => { + pushed.push({ namespace, payload }) + return { sent: 1, failed: 0, subscriptions: 1 } + } + } as never, + { sendToast: async () => 0 } as never, + { hasVisibleConnection: () => false } as never, + '' + ) + + await channel.sendModelError(createSession(), { + eventId: 'evt-1', + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'x', + priorAssistantClaimsDone: false, + atTs: 1 + }, { nativeGate: { sent: true } }) + + expect(pushed).toHaveLength(0) + }) + + it('sendModelError fires web-push when nativeGate.sent is false', async () => { + const pushed: Array<{ namespace: string; payload: PushPayload }> = [] + const channel = new PushNotificationChannel( + { + sendToNamespace: async (namespace: string, payload: PushPayload) => { + pushed.push({ namespace, payload }) + return { sent: 1, failed: 0, subscriptions: 1 } + } + } as never, + { sendToast: async () => 0 } as never, + { hasVisibleConnection: () => false } as never, + '' + ) + + await channel.sendModelError(createSession(), { + eventId: 'evt-2', + kind: 'rate_limited', + transient: true, + rawSnippet: 'y', + priorAssistantClaimsDone: false, + atTs: 2 + }, { nativeGate: { sent: false } }) + + expect(pushed).toHaveLength(1) + expect(pushed[0]?.payload.data?.type).toBe('model-error') + }) }) diff --git a/hub/src/push/pushNotificationChannel.ts b/hub/src/push/pushNotificationChannel.ts index bfe751f557..34d45ccb8d 100644 --- a/hub/src/push/pushNotificationChannel.ts +++ b/hub/src/push/pushNotificationChannel.ts @@ -1,7 +1,13 @@ import type { Session } from '../sync/syncEngine' -import type { NotificationChannel, TaskNotification } from '../notifications/notificationTypes' +import type { + ModelErrorNotification, + ModelErrorSendOutcome, + NotificationChannel, + TaskNotification +} from '../notifications/notificationTypes' import type { NotificationSendContext } from '../notifications/notificationSendContext' import { getAgentName, getSessionName } from '../notifications/sessionInfo' +import { formatModelErrorBody, formatModelErrorTitle } from '../notifications/modelErrorCopy' import type { SSEManager } from '../sse/sseManager' import type { VisibilityTracker } from '../visibility/visibilityTracker' import type { PushPayload, PushService } from './pushService' @@ -135,6 +141,57 @@ export class PushNotificationChannel implements NotificationChannel { await this.pushService.sendToNamespace(session.namespace, payload) } + async sendModelError( + session: Session, + notification: ModelErrorNotification, + ctx?: NotificationSendContext + ): Promise { + // No active-session guard: scheduled retries must still deliver after + // the session goes inactive (watermark + timer survive that transition). + + if (ctx?.nativeGate?.sent) { + this.logBranch('model-error', session.namespace, 'defer-to-native', 'fcm-delivered-this-dispatch') + return 'unavailable' + } + + const agentName = getAgentName(session) + const sessionName = getSessionName(session) + const title = formatModelErrorTitle(notification.kind) + const body = formatModelErrorBody(notification, { agentName, sessionName }) + const url = this.buildSessionPath(session.id) + + const payload: PushPayload = { + title, + body, + // Distinct tag from `ready-${id}` so the model-error ping never + // collapses into the prior "all done" notification on the same + // session. Tag keyed by eventId so distinct errors in the same + // session DON'T overwrite each other on the lock screen. + tag: `model-error-${session.id}-${notification.eventId}`, + data: { + type: 'model-error', + sessionId: session.id, + url + } + } + + // Skip the in-page toast shortcut for model errors. Toasts are + // ephemeral and easy to miss; an error of this severity should + // ALWAYS surface as a real push so a backgrounded operator gets + // a system-tray ping. The web banner + pulsing-dot already + // cover the foreground case. Still defer when FCM already + // delivered this dispatch (nativeGate). + this.logBranch('model-error', session.namespace, 'web-push-fired') + const result = await this.pushService.sendToNamespace(session.namespace, payload) + if (result.sent > 0) { + return 'delivered' + } + if (result.subscriptions === 0) { + return 'unavailable' + } + return 'failed' + } + private buildSessionPath(sessionId: string): string { return `/sessions/${sessionId}` } diff --git a/hub/src/push/pushService.ts b/hub/src/push/pushService.ts index e44a8fd9ef..9975166a3c 100644 --- a/hub/src/push/pushService.ts +++ b/hub/src/push/pushService.ts @@ -38,23 +38,33 @@ export class PushService { webPush.setVapidDetails(this.subject, this.vapidKeys.publicKey, this.vapidKeys.privateKey) } - async sendToNamespace(namespace: string, payload: PushPayload): Promise { + async sendToNamespace(namespace: string, payload: PushPayload): Promise<{ sent: number; failed: number; subscriptions: number }> { const subscriptions = this.store.push.getPushSubscriptionsByNamespace(namespace) if (subscriptions.length === 0) { - return + return { sent: 0, failed: 0, subscriptions: 0 } } const body = JSON.stringify(payload) - await Promise.all(subscriptions.map((subscription) => { + const results = await Promise.all(subscriptions.map((subscription) => { return this.sendToSubscription(namespace, subscription, body) })) + let sent = 0 + let failed = 0 + for (const result of results) { + if (result === 'sent') { + sent++ + } else { + failed++ + } + } + return { sent, failed, subscriptions: subscriptions.length } } private async sendToSubscription( namespace: string, subscription: StoredSubscription, body: string - ): Promise { + ): Promise<'sent' | 'failed'> { const pushSubscription: PushSubscription = { endpoint: subscription.endpoint, keys: { @@ -65,6 +75,7 @@ export class PushService { try { await webPush.sendNotification(pushSubscription, body) + return 'sent' } catch (error) { const statusCode = typeof (error as { statusCode?: unknown }).statusCode === 'number' ? (error as { statusCode: number }).statusCode @@ -72,10 +83,11 @@ export class PushService { if (statusCode === 410) { this.store.push.removePushSubscription(namespace, subscription.endpoint) - return + return 'failed' } console.error('[PushService] Failed to send notification:', error) + return 'failed' } } } diff --git a/hub/src/store/sessions.test.ts b/hub/src/store/sessions.test.ts index 94762c86e5..7996e95f33 100644 --- a/hub/src/store/sessions.test.ts +++ b/hub/src/store/sessions.test.ts @@ -488,6 +488,150 @@ describe('updateSessionMetadata: protocol resume token preservation', () => { expect(metadata?.lifecycleState).toBe('archived') }) + it('preserves lastModelError across sparse archive (durable alert state)', () => { + const store = makeStore() + const lastModelError = { + eventId: 'evt-1700000000000', + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'Error: T: [resource_exhausted]', + atTs: 1_700_000_000_000, + priorAssistantClaimsDone: false + } + const session = store.sessions.getOrCreateSession( + 'cursor-model-error-survives', + { + path: '/tmp/project', + host: 'example', + flavor: 'cursor', + cursorSessionId: 'err-uuid', + lastModelError + }, + null, + 'default' + ) + + store.sessions.updateSessionMetadata( + session.id, + { + lifecycleState: 'archived', + archivedBy: 'cli', + archiveReason: 'Session crashed' + }, + session.metadataVersion, + 'default' + ) + + const metadata = getMetadata(store, session.id) as Record | null + expect(metadata?.lastModelError).toEqual(lastModelError) + expect(metadata?.lifecycleState).toBe('archived') + }) + + it('preserves lastModelError.acknowledgedAt against stale CLI rewrite of same eventId', () => { + const store = makeStore() + const eventId = 'evt-ack-1700000000111' + const atTs = 1_700_000_000_111 + const session = store.sessions.getOrCreateSession( + 'cursor-model-error-ack-survives', + { + path: '/tmp/project', + host: 'example', + flavor: 'cursor', + cursorSessionId: 'ack-uuid', + lastModelError: { + eventId, + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'Error: T: [resource_exhausted]', + atTs, + priorAssistantClaimsDone: false, + acknowledgedAt: 1_700_000_000_222 + } + }, + null, + 'default' + ) + + // CLI local snapshot still has the same error without acknowledgedAt. + store.sessions.updateSessionMetadata( + session.id, + { + path: '/tmp/project', + host: 'example', + flavor: 'cursor', + cursorSessionId: 'ack-uuid', + lastModelError: { + eventId, + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'Error: T: [resource_exhausted]', + atTs, + priorAssistantClaimsDone: false + } + }, + session.metadataVersion, + 'default' + ) + + const metadata = getMetadata(store, session.id) as { + lastModelError?: { atTs?: number; acknowledgedAt?: number } + } | null + expect(metadata?.lastModelError?.atTs).toBe(atTs) + expect(metadata?.lastModelError?.acknowledgedAt).toBe(1_700_000_000_222) + }) + + it('preserves lastModelError.notifiedAt against stale CLI rewrite of same eventId', () => { + const store = makeStore() + const eventId = 'evt-notified-1700000000333' + const atTs = 1_700_000_000_333 + const session = store.sessions.getOrCreateSession( + 'cursor-model-error-notified-survives', + { + path: '/tmp/project', + host: 'example', + flavor: 'cursor', + cursorSessionId: 'notify-uuid', + lastModelError: { + eventId, + kind: 'rate_limited', + transient: true, + rawSnippet: 'Error: T: [resource_exhausted]', + atTs, + priorAssistantClaimsDone: false, + notifiedAt: 1_700_000_000_444 + } + }, + null, + 'default' + ) + + store.sessions.updateSessionMetadata( + session.id, + { + path: '/tmp/project', + host: 'example', + flavor: 'cursor', + cursorSessionId: 'notify-uuid', + lastModelError: { + eventId, + kind: 'rate_limited', + transient: true, + rawSnippet: 'Error: T: [resource_exhausted]', + atTs, + priorAssistantClaimsDone: false + } + }, + session.metadataVersion, + 'default' + ) + + const metadata = getMetadata(store, session.id) as { + lastModelError?: { atTs?: number; notifiedAt?: number } + } | null + expect(metadata?.lastModelError?.atTs).toBe(atTs) + expect(metadata?.lastModelError?.notifiedAt).toBe(1_700_000_000_444) + }) + it('does not invent path or host when prior had none', () => { const store = makeStore() // create with minimal raw metadata (path is technically required by diff --git a/hub/src/store/sessions.ts b/hub/src/store/sessions.ts index c0d23236de..b4aaa1d247 100644 --- a/hub/src/store/sessions.ts +++ b/hub/src/store/sessions.ts @@ -36,6 +36,11 @@ import { updateVersionedField } from './versionedUpdates' // write-once-keep semantics. Mirror of pickExistingSessionMetadata // in cli/src/agent/sessionFactory.ts. // +// - ALERT_STATE_FIELDS: durable operator-facing alert state that must +// survive sparse metadata writes (e.g. archive). Without this, +// lastModelError (banner / amber dot / ack) vanishes when a write +// omits it — the user never dismissed the error. +// // `cursorSessionProtocol` is paired with `cursorSessionId`: protocol is // tied to a specific chat id, so a write that explicitly sets a new // `cursorSessionId` must drop a stale prior protocol. Handled in @@ -65,6 +70,8 @@ const SIMPLE_RESUME_TOKENS = [ 'piSessionId' ] as const +const ALERT_STATE_FIELDS = ['lastModelError'] as const + function isPlainObject(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value) } @@ -120,6 +127,67 @@ function preserveCursorProtocolPair( return merged } +/** + * Hub-owned fields on lastModelError (ack + delivery watermark) must survive + * stale CLI metadata rewrites of the same eventId. Web ack / NotificationHub + * write these on the hub copy; the CLI's local snapshot often lacks them. + * Identity is eventId — wall-clock atTs is display-only. + */ +function preserveModelErrorHubFields( + prior: Record, + next: Record, + merged: Record | null +): Record | null { + const oldError = isPlainObject(prior.lastModelError) ? prior.lastModelError : null + const newError = isPlainObject(next.lastModelError) ? next.lastModelError : null + if ( + !oldError + || !newError + || typeof oldError.eventId !== 'string' + || oldError.eventId !== newError.eventId + ) { + return merged + } + + const preserved: Record = { ...newError } + let changed = false + if (typeof oldError.acknowledgedAt === 'number' && newError.acknowledgedAt === undefined) { + preserved.acknowledgedAt = oldError.acknowledgedAt + changed = true + } + if (typeof oldError.notifiedAt === 'number' && newError.notifiedAt === undefined) { + preserved.notifiedAt = oldError.notifiedAt + changed = true + } + if (typeof oldError.bridgedForEventId === 'string' && newError.bridgedForEventId === undefined) { + preserved.bridgedForEventId = oldError.bridgedForEventId + changed = true + } + if (oldError.retriedAndFailed === true && newError.retriedAndFailed !== true) { + preserved.retriedAndFailed = true + changed = true + } + if (oldError.supersededByUserTurn === true && newError.supersededByUserTurn !== true) { + preserved.supersededByUserTurn = true + changed = true + } + if (oldError.bridgeable === false && newError.bridgeable !== false) { + preserved.bridgeable = false + changed = true + } + if (typeof oldError.lastUserMessage === 'string' && newError.lastUserMessage === undefined) { + preserved.lastUserMessage = oldError.lastUserMessage + changed = true + } + if (!changed) { + return merged + } + + const result = merged ?? { ...next } + result.lastModelError = preserved + return result +} + export function mergeSessionMetadata(prior: unknown, next: unknown): unknown { if (!isPlainObject(prior) || !isPlainObject(next)) { return next @@ -128,7 +196,9 @@ export function mergeSessionMetadata(prior: unknown, next: unknown): unknown { merged = carryForwardIfMissing(prior, next, merged, PARSE_IDENTITY_FIELDS) merged = carryForwardIfMissing(prior, next, merged, ROUTING_FIELDS) merged = carryForwardIfMissing(prior, next, merged, SIMPLE_RESUME_TOKENS) + merged = carryForwardIfMissing(prior, next, merged, ALERT_STATE_FIELDS) merged = preserveCursorProtocolPair(prior, next, merged) + merged = preserveModelErrorHubFields(prior, next, merged) return merged ?? next } diff --git a/hub/src/sync/autoBridgeReconcile.test.ts b/hub/src/sync/autoBridgeReconcile.test.ts new file mode 100644 index 0000000000..ba1f5e9965 --- /dev/null +++ b/hub/src/sync/autoBridgeReconcile.test.ts @@ -0,0 +1,364 @@ +import { afterEach, describe, expect, it } from 'bun:test' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Store } from '../store' +import { RpcRegistry } from '../socket/rpcRegistry' +import { + readAutoBridgeTransientModelErrorsEnabled, + writeAutoBridgeTransientModelErrorsEnabled +} from '../config/autoBridgeTransientModelErrors' +import { SyncEngine } from './syncEngine' + +const directories: string[] = [] + +afterEach(async () => { + await Promise.all(directories.splice(0).map((path) => rm(path, { recursive: true, force: true }))) +}) + +async function waitUntil( + predicate: () => boolean, + label: string, + timeoutMs = 2_000 +): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (predicate()) { + return + } + await new Promise((resolve) => setTimeout(resolve, 5)) + } + throw new Error(`timed out waiting for ${label}`) +} + +describe('cursor auto-bridge reconcile on session-ready', () => { + async function setup(opts?: { namespace?: string; flavor?: string }) { + const dataDir = await mkdtemp(join(tmpdir(), 'hapi-auto-bridge-reconcile-')) + directories.push(dataDir) + + const store = new Store(':memory:') + const engine = new SyncEngine( + store, + {} as never, + new RpcRegistry(), + { broadcast() {} } as never + ) + engine.setSettingsDataDirForTests(dataDir) + + const session = engine.getOrCreateSession( + 'session-cursor-bootstrapping', + { + path: '/tmp/project', + host: 'localhost', + machineId: 'machine-1', + flavor: opts?.flavor ?? 'cursor' + }, + null, + opts?.namespace ?? 'default' + ) + + const configCalls: Array<{ sessionId: string; config: Record }> = [] + ;(engine as unknown as { + rpcGateway: { + requestSessionConfig: ( + sessionId: string, + config: Record + ) => Promise + } + }).rpcGateway.requestSessionConfig = async (sessionId, config) => { + configCalls.push({ sessionId, config }) + return { applied: config } + } + + return { engine, session, dataDir, configCalls } + } + + it('pushes enable after session-ready when CLI fetched while inactive', async () => { + const { engine, session, dataDir, configCalls } = await setup() + // Simulate Settings toggle while the row is still inactive (create/get + // already handed the CLI the previous false default). + expect(session.active).toBe(false) + await writeAutoBridgeTransientModelErrorsEnabled(dataDir, true) + + engine.handleSessionReady({ sid: session.id, time: Date.now() }) + await waitUntil(() => configCalls.length >= 1, 'session-ready enable fanout') + + expect(configCalls).toEqual([ + { + sessionId: session.id, + config: { autoBridgeTransientModelErrors: true } + } + ]) + }) + + it('pushes disable after session-ready so a stale CLI cannot keep auto-bridging', async () => { + const { engine, session, dataDir, configCalls } = await setup() + await writeAutoBridgeTransientModelErrorsEnabled(dataDir, true) + await writeAutoBridgeTransientModelErrorsEnabled(dataDir, false) + + engine.handleSessionReady({ sid: session.id, time: Date.now() }) + await waitUntil(() => configCalls.length >= 1, 'session-ready disable fanout') + + expect(configCalls).toEqual([ + { + sessionId: session.id, + config: { autoBridgeTransientModelErrors: false } + } + ]) + }) + + it('skips tenant namespaces and non-cursor flavors', async () => { + const tenant = await setup({ namespace: 'tenant-a' }) + await writeAutoBridgeTransientModelErrorsEnabled(tenant.dataDir, true) + tenant.engine.handleSessionReady({ sid: tenant.session.id, time: Date.now() }) + // Fire-and-forget skip path: wait long enough for a mistaken RPC, then assert none. + await new Promise((resolve) => setTimeout(resolve, 50)) + expect(tenant.configCalls).toEqual([]) + + const claude = await setup({ flavor: 'claude' }) + await writeAutoBridgeTransientModelErrorsEnabled(claude.dataDir, true) + claude.engine.handleSessionReady({ sid: claude.session.id, time: Date.now() }) + await new Promise((resolve) => setTimeout(resolve, 50)) + expect(claude.configCalls).toEqual([]) + }) + + it('reconciles on first inactive → active transition after a toggle while inactive', async () => { + const { engine, session, dataDir, configCalls } = await setup() + expect(session.active).toBe(false) + await writeAutoBridgeTransientModelErrorsEnabled(dataDir, true) + + engine.handleSessionAlive({ sid: session.id, time: Date.now() }) + await waitUntil(() => configCalls.length >= 1, 'first-active enable fanout') + + expect(configCalls).toEqual([ + { + sessionId: session.id, + config: { autoBridgeTransientModelErrors: true } + } + ]) + }) + + it('serializes settings fanout with an in-flight session-ready reconcile', async () => { + const { engine, session, dataDir, configCalls } = await setup() + await writeAutoBridgeTransientModelErrorsEnabled(dataDir, false) + + const firstRpc = { release: null as (() => void) | null } + ;(engine as unknown as { + rpcGateway: { + requestSessionConfig: ( + sessionId: string, + config: Record + ) => Promise + } + }).rpcGateway.requestSessionConfig = async (sessionId, config) => { + configCalls.push({ sessionId, config }) + if (configCalls.length === 1) { + await new Promise((resolve) => { + firstRpc.release = resolve + }) + } + return { applied: config } + } + + const reconcilePromise = engine.reconcileCursorAutoBridgeSetting(session.id) + await waitUntil(() => typeof firstRpc.release === 'function', 'first RPC hold') + + // Become active while the first reconcile still holds the lock, then + // toggle + fanout so the serialized tail sees the new value. + engine.handleSessionAlive({ sid: session.id, time: Date.now() }) + await writeAutoBridgeTransientModelErrorsEnabled(dataDir, true) + const fanoutPromise = engine.fanoutAutoBridgeTransientModelErrors(true) + + firstRpc.release?.() + await Promise.all([reconcilePromise, fanoutPromise]) + await waitUntil( + () => configCalls.at(-1)?.config?.autoBridgeTransientModelErrors === true, + 'serialized fanout true' + ) + + expect(configCalls.some((call) => call.config.autoBridgeTransientModelErrors === true)).toBe(true) + expect(configCalls.at(-1)?.config).toEqual({ autoBridgeTransientModelErrors: true }) + }) + + it('heartbeats repair a CLI left enabled after partial fanout + failed rollback', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'hapi-auto-bridge-reconcile-')) + directories.push(dataDir) + await writeAutoBridgeTransientModelErrorsEnabled(dataDir, false) + + const store = new Store(':memory:') + const engine = new SyncEngine( + store, + {} as never, + new RpcRegistry(), + { broadcast() {} } as never + ) + engine.setSettingsDataDirForTests(dataDir) + + const sessionA = engine.getOrCreateSession( + 'session-cursor-a', + { path: '/tmp/a', host: 'localhost', machineId: 'm1', flavor: 'cursor' }, + null, + 'default' + ) + const sessionB = engine.getOrCreateSession( + 'session-cursor-b', + { path: '/tmp/b', host: 'localhost', machineId: 'm1', flavor: 'cursor' }, + null, + 'default' + ) + + const configCalls: Array<{ sessionId: string; config: Record }> = [] + // Fail B on enable (partial forward) and A on the first post-forward disable + // (failed rollback). Later heartbeats must repair A to persisted false. + let sawForwardEnableForA = false + let applyFinished = false + ;(engine as unknown as { + rpcGateway: { + requestSessionConfig: ( + sessionId: string, + config: Record + ) => Promise + } + }).rpcGateway.requestSessionConfig = async (sessionId, config) => { + configCalls.push({ sessionId, config }) + const enabled = config.autoBridgeTransientModelErrors === true + if (enabled && sessionId === sessionA.id) { + sawForwardEnableForA = true + } + if (!applyFinished && enabled && sessionId === sessionB.id) { + throw new Error('forward fanout B failed') + } + if ( + !applyFinished + && sawForwardEnableForA + && !enabled + && sessionId === sessionA.id + ) { + throw new Error('rollback fanout A failed') + } + return { applied: config } + } + + engine.handleSessionAlive({ sid: sessionA.id, time: Date.now() }) + engine.handleSessionAlive({ sid: sessionB.id, time: Date.now() }) + await waitUntil( + () => configCalls.filter((call) => call.sessionId === sessionA.id).length >= 1 + && configCalls.filter((call) => call.sessionId === sessionB.id).length >= 1, + 'first-active reconciles' + ) + + await expect( + engine.applyAutoBridgeTransientModelErrorsSetting(dataDir, true) + ).rejects.toThrow('Failed to update every active Cursor session') + applyFinished = true + expect(await readAutoBridgeTransientModelErrorsEnabled(dataDir)).toBe(false) + expect(configCalls.some((call) => ( + call.sessionId === sessionA.id + && call.config.autoBridgeTransientModelErrors === true + ))).toBe(true) + + const beforeRepair = configCalls.length + // Already active — heartbeat must still retry pending reconcile. + engine.handleSessionAlive({ sid: sessionA.id, time: Date.now() + 1 }) + await waitUntil( + () => configCalls.slice(beforeRepair).some((call) => ( + call.sessionId === sessionA.id + && call.config.autoBridgeTransientModelErrors === false + )), + 'heartbeat repair to false' + ) + }) + + it('serializes concurrent apply so a failed PUT cannot clobber a later success', async () => { + const { engine, session, dataDir, configCalls } = await setup() + engine.handleSessionAlive({ sid: session.id, time: Date.now() }) + await writeAutoBridgeTransientModelErrorsEnabled(dataDir, false) + + const failHold = { release: null as (() => void) | null } + let failArmed = true + ;(engine as unknown as { + rpcGateway: { + requestSessionConfig: ( + sessionId: string, + config: Record + ) => Promise + } + }).rpcGateway.requestSessionConfig = async (sessionId, config) => { + configCalls.push({ sessionId, config }) + if (failArmed && config.autoBridgeTransientModelErrors === true) { + failArmed = false + await new Promise((resolve) => { + failHold.release = resolve + }) + throw new Error('simulated fanout failure') + } + return { applied: config } + } + + const failing = engine.applyAutoBridgeTransientModelErrorsSetting(dataDir, true) + await waitUntil(() => typeof failHold.release === 'function', 'failing apply hold') + + const succeeding = engine.applyAutoBridgeTransientModelErrorsSetting(dataDir, false) + failHold.release?.() + + await expect(failing).rejects.toThrow('Failed to update every active Cursor session') + await succeeding + + expect(await readAutoBridgeTransientModelErrorsEnabled(dataDir)).toBe(false) + expect(configCalls.at(-1)?.config).toEqual({ autoBridgeTransientModelErrors: false }) + }) + + it('reconciles an already-active stored Cursor row on first heartbeat after hub restart', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'hapi-auto-bridge-reconcile-')) + directories.push(dataDir) + await writeAutoBridgeTransientModelErrorsEnabled(dataDir, false) + + const store = new Store(':memory:') + const boot = new SyncEngine( + store, + {} as never, + new RpcRegistry(), + { broadcast() {} } as never + ) + const session = boot.getOrCreateSession( + 'session-cursor-restart', + { path: '/tmp/restart', host: 'localhost', machineId: 'm1', flavor: 'cursor' }, + null, + 'default' + ) + boot.handleSessionAlive({ sid: session.id, time: Date.now() }) + store.sessions.setSessionActive(session.id, true, Date.now(), 'default') + boot.stop() + + const restarted = new SyncEngine( + store, + {} as never, + new RpcRegistry(), + { broadcast() {} } as never + ) + restarted.setSettingsDataDirForTests(dataDir) + const configCalls: Array<{ sessionId: string; config: Record }> = [] + ;(restarted as unknown as { + rpcGateway: { + requestSessionConfig: ( + sessionId: string, + config: Record + ) => Promise + } + }).rpcGateway.requestSessionConfig = async (sessionId, config) => { + configCalls.push({ sessionId, config }) + return { applied: config } + } + + expect(restarted.getSession(session.id)?.active).toBe(true) + restarted.handleSessionAlive({ sid: session.id, time: Date.now() + 1 }) + await waitUntil( + () => configCalls.some((call) => ( + call.sessionId === session.id + && call.config.autoBridgeTransientModelErrors === false + )), + 'post-restart heartbeat reconcile' + ) + restarted.stop() + }) +}) diff --git a/hub/src/sync/rpcGateway.ts b/hub/src/sync/rpcGateway.ts index 656da1dda9..2d9f70fc98 100644 --- a/hub/src/sync/rpcGateway.ts +++ b/hub/src/sync/rpcGateway.ts @@ -145,6 +145,7 @@ export class RpcGateway { effort?: string | null collaborationMode?: CodexCollaborationMode copilotAgentMode?: CopilotAgentMode + autoBridgeTransientModelErrors?: boolean } ): Promise { return await this.sessionRpc(sessionId, RPC_METHODS.SetSessionConfig, config) @@ -165,6 +166,28 @@ export class RpcGateway { await this.sessionRpc(sessionId, RPC_METHODS.HandoffLocal, {}) } + async bridgeModelError( + sessionId: string, + payload: { + eventId: string + atTs: number + kind: string + rawSnippet: string + lastUserMessage?: string + priorAssistantClaimsDone: boolean + transient: boolean + bridgedForEventId?: string + retriedAndFailed?: boolean + supersededByUserTurn?: boolean + bridgeable?: boolean + } + ): Promise<{ ok: boolean; reason?: string }> { + return await this.sessionRpc(sessionId, RPC_METHODS.BridgeModelError, payload) as { + ok: boolean + reason?: string + } + } + async spawnSession( machineId: string, directory: string, diff --git a/hub/src/sync/sessionCache-merge-scratchlist.test.ts b/hub/src/sync/sessionCache-merge-scratchlist.test.ts index 7d1bae8d31..ab323a50ef 100644 --- a/hub/src/sync/sessionCache-merge-scratchlist.test.ts +++ b/hub/src/sync/sessionCache-merge-scratchlist.test.ts @@ -268,3 +268,92 @@ describe('cascade-delete safety (regression)', () => { expect(store.scratchlist.list(newSession.id)).toEqual([]) }) }) + +describe('mergeSessions preserves lastModelError alert state', () => { + it('carries unresolved lastModelError from old row onto new (incl hub watermarks)', async () => { + const { store, cache } = setup() + const { oldSession, newSession } = makeSessions(cache) + const eventId = 'evt-merge-1800000000001' + const atTs = 1_800_000_000_001 + + const result = store.sessions.updateSessionMetadata( + oldSession.id, + { + ...(oldSession.metadata ?? { path: '/tmp/project', host: 'localhost' }), + lastModelError: { + eventId, + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'Error: T: [resource_exhausted]', + atTs, + priorAssistantClaimsDone: false, + acknowledgedAt: 1_800_000_000_002, + notifiedAt: 1_800_000_000_003 + } + }, + oldSession.metadataVersion, + 'default' + ) + expect(result.result).toBe('success') + cache.getSession(oldSession.id) // ensure cache exists + // Pull store write into cache via refresh path used by mergeSessions. + const refreshedOld = store.sessions.getSession(oldSession.id) + expect(refreshedOld?.metadata && (refreshedOld.metadata as { lastModelError?: { atTs: number } }).lastModelError?.atTs).toBe(atTs) + + // Re-load old session into cache from store (mergeSessions reads stored rows). + await cache.mergeSessions(oldSession.id, newSession.id, 'default') + + const merged = cache.getSession(newSession.id)?.metadata?.lastModelError + expect(merged?.eventId).toBe(eventId) + expect(merged?.atTs).toBe(atTs) + expect(merged?.acknowledgedAt).toBe(1_800_000_000_002) + expect(merged?.notifiedAt).toBe(1_800_000_000_003) + expect(merged?.kind).toBe('quota_exhausted') + }) + + it('keeps the new row eventId when atTs is lower than the old row (clock skew)', async () => { + const { store, cache } = setup() + const { oldSession, newSession } = makeSessions(cache) + + store.sessions.updateSessionMetadata( + oldSession.id, + { + ...(oldSession.metadata ?? { path: '/tmp/project', host: 'localhost' }), + lastModelError: { + eventId: 'evt-old-high-clock', + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'old', + atTs: 9_000, + priorAssistantClaimsDone: false, + notifiedAt: 9_001 + } + }, + oldSession.metadataVersion, + 'default' + ) + store.sessions.updateSessionMetadata( + newSession.id, + { + ...(newSession.metadata ?? { path: '/tmp/project', host: 'localhost' }), + lastModelError: { + eventId: 'evt-new-low-clock', + kind: 'transport_closed', + transient: true, + rawSnippet: 'new', + atTs: 1_000, + priorAssistantClaimsDone: false + } + }, + newSession.metadataVersion, + 'default' + ) + + await cache.mergeSessions(oldSession.id, newSession.id, 'default') + + const merged = cache.getSession(newSession.id)?.metadata?.lastModelError + expect(merged?.eventId).toBe('evt-new-low-clock') + expect(merged?.atTs).toBe(1_000) + expect(merged?.kind).toBe('transport_closed') + }) +}) diff --git a/hub/src/sync/sessionCache-model-error-notified.test.ts b/hub/src/sync/sessionCache-model-error-notified.test.ts new file mode 100644 index 0000000000..2126382221 --- /dev/null +++ b/hub/src/sync/sessionCache-model-error-notified.test.ts @@ -0,0 +1,158 @@ +import { describe, expect, it, spyOn } from 'bun:test' +import type { SyncEvent } from '@hapi/protocol/types' +import { Store } from '../store' +import type { EventPublisher } from './eventPublisher' +import { SessionCache } from './sessionCache' +import { NotificationHub } from '../notifications/notificationHub' +import type { SyncEngine, Session } from './syncEngine' +import type { + ModelErrorNotification, + ModelErrorSendOutcome, + NotificationChannel +} from '../notifications/notificationTypes' + +function createCapturingPublisher(events: SyncEvent[]): EventPublisher { + return { + emit: (event: SyncEvent) => { + events.push(event) + } + } as unknown as EventPublisher +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +describe('SessionCache.acknowledgeModelError', () => { + it('retries version-mismatch then persists acknowledgedAt', async () => { + const store = new Store(':memory:') + const cache = new SessionCache(store, createCapturingPublisher([])) + const eventId = 'evt-ack-9002' + const atTs = 9_002 + const session = cache.getOrCreateSession( + 'model-error-ack-retry', + { + path: '/tmp/project', + host: 'localhost', + flavor: 'cursor', + lastModelError: { + eventId, + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'Error: T: [resource_exhausted]', + atTs, + priorAssistantClaimsDone: false + } + }, + null, + 'default' + ) + + let calls = 0 + const original = store.sessions.updateSessionMetadata.bind(store.sessions) + spyOn(store.sessions, 'updateSessionMetadata').mockImplementation((...args) => { + calls += 1 + if (calls === 1) { + return { + result: 'version-mismatch' as const, + version: session.metadataVersion, + value: session.metadata + } + } + return original(...args) + }) + + await cache.acknowledgeModelError(session.id, eventId) + + expect(calls).toBeGreaterThanOrEqual(2) + expect(cache.getSession(session.id)?.metadata?.lastModelError?.acknowledgedAt).toEqual( + expect.any(Number) + ) + }) +}) + +describe('SessionCache.markModelErrorNotified', () => { + it('retries version-mismatch then persists notifiedAt so a fresh hub does not redeliver', async () => { + const store = new Store(':memory:') + const cache = new SessionCache(store, createCapturingPublisher([])) + const eventId = 'evt-notified-9001' + const atTs = 9_001 + const session = cache.getOrCreateSession( + 'model-error-notified-retry', + { + path: '/tmp/project', + host: 'localhost', + flavor: 'cursor', + lastModelError: { + eventId, + kind: 'quota_exhausted', + transient: false, + rawSnippet: 'Error: T: [resource_exhausted]', + atTs, + priorAssistantClaimsDone: false + } + }, + null, + 'default' + ) + + let calls = 0 + const original = store.sessions.updateSessionMetadata.bind(store.sessions) + spyOn(store.sessions, 'updateSessionMetadata').mockImplementation((...args) => { + calls += 1 + if (calls === 1) { + return { + result: 'version-mismatch' as const, + version: session.metadataVersion, + value: session.metadata + } + } + return original(...args) + }) + + await cache.markModelErrorNotified(session.id, eventId) + + expect(calls).toBeGreaterThanOrEqual(2) + const refreshed = cache.getSession(session.id) + expect(refreshed?.metadata?.lastModelError?.notifiedAt).toEqual(expect.any(Number)) + expect(refreshed?.metadata?.lastModelError?.atTs).toBe(atTs) + + class Channel implements NotificationChannel { + readonly modelErrors: ModelErrorNotification[] = [] + async sendReady() {} + async sendPermissionRequest() {} + async sendTaskNotification() {} + async sendSessionCompletion() {} + async sendModelError( + _session: Session, + notification: ModelErrorNotification + ): Promise { + this.modelErrors.push(notification) + return 'delivered' + } + } + + const engine = { + listeners: new Set<(e: { type: string; sessionId: string }) => void>(), + getSession: (id: string) => cache.getSession(id), + getSessions: () => { + const s = cache.getSession(session.id) + return s ? [s] : [] + }, + subscribe(listener: (e: { type: string; sessionId: string }) => void) { + this.listeners.add(listener) + return () => this.listeners.delete(listener) + }, + async markModelErrorNotified() {}, + emit(sessionId: string) { + for (const listener of this.listeners) { + listener({ type: 'session-updated', sessionId }) + } + } + } + const channel = new Channel() + const hub = new NotificationHub(engine as unknown as SyncEngine, [channel]) + engine.emit(session.id) + await sleep(10) + expect(channel.modelErrors).toHaveLength(0) + hub.stop() + }) +}) diff --git a/hub/src/sync/sessionCache.ts b/hub/src/sync/sessionCache.ts index cbab45975f..2900fdb733 100644 --- a/hub/src/sync/sessionCache.ts +++ b/hub/src/sync/sessionCache.ts @@ -904,6 +904,150 @@ export class SessionCache { throw new Error('Session was modified concurrently. Please try again.') } + async markModelErrorBridged(sessionId: string, eventId: string): Promise { + const session = this.sessions.get(sessionId) ?? this.refreshSession(sessionId) + if (!session) { + throw new Error('Session not found') + } + + const currentMetadata = session.metadata ?? { path: '', host: '' } + const currentError = currentMetadata.lastModelError + if (!currentError || currentError.eventId !== eventId) { + return + } + if (currentError.bridgedForEventId === eventId) { + return + } + + const newMetadata = { + ...currentMetadata, + lastModelError: { + ...currentError, + bridgedForEventId: eventId + } + } + + const result = this.store.sessions.updateSessionMetadata( + sessionId, + newMetadata, + session.metadataVersion, + session.namespace, + { touchUpdatedAt: false } + ) + + if (result.result === 'error') { + throw new Error('Failed to update session metadata') + } + + if (result.result === 'version-mismatch') { + throw new Error('Session was modified concurrently. Please try again.') + } + + this.refreshSession(sessionId) + } + + async acknowledgeModelError(sessionId: string, eventId: string): Promise { + // Bind dismiss to the error the client actually showed. If a newer + // lastModelError replaced it between render and click, refuse so we + // don't silently ack the unseen error (banner/dot would vanish). + // Identity is eventId (not wall-clock atTs). Retry version-mismatch + // (CLI metadata race) like markModelErrorNotified. + for (let attempt = 0; attempt < METADATA_RETRY_ATTEMPTS; attempt += 1) { + const session = this.sessions.get(sessionId) ?? this.refreshSession(sessionId) + if (!session) { + throw new Error('Session not found') + } + + const currentMetadata = session.metadata ?? { path: '', host: '' } + const currentError = currentMetadata.lastModelError + if (!currentError) { + return + } + if (currentError.eventId !== eventId) { + throw new Error('Model error changed; refresh before acknowledging.') + } + if (typeof currentError.acknowledgedAt === 'number') { + return + } + + const result = this.store.sessions.updateSessionMetadata( + sessionId, + { + ...currentMetadata, + lastModelError: { + ...currentError, + acknowledgedAt: Date.now() + } + }, + session.metadataVersion, + session.namespace, + { touchUpdatedAt: false } + ) + + if (result.result === 'success') { + this.refreshSession(sessionId) + return + } + if (result.result === 'error') { + throw new Error('Failed to update session metadata') + } + + this.refreshSession(sessionId) + } + + throw new Error('Session was modified concurrently. Please try again.') + } + + /** + * Persist delivery watermark on lastModelError so hub restarts do not + * re-page the same unacknowledged eventId (in-memory Map alone is lost). + * No-ops when the error changed under us — a different eventId owns the page. + * Retries on version-mismatch (same pattern as renameSession / #919). + */ + async markModelErrorNotified(sessionId: string, eventId: string): Promise { + for (let attempt = 0; attempt < METADATA_RETRY_ATTEMPTS; attempt += 1) { + const session = this.sessions.get(sessionId) ?? this.refreshSession(sessionId) + if (!session) { + return + } + + const currentMetadata = session.metadata ?? { path: '', host: '' } + const currentError = currentMetadata.lastModelError + if (!currentError || currentError.eventId !== eventId) { + return + } + if (typeof currentError.notifiedAt === 'number') { + return + } + + const result = this.store.sessions.updateSessionMetadata( + sessionId, + { + ...currentMetadata, + lastModelError: { + ...currentError, + notifiedAt: Date.now() + } + }, + session.metadataVersion, + session.namespace, + { touchUpdatedAt: false } + ) + + if (result.result === 'success') { + this.refreshSession(sessionId) + return + } + if (result.result === 'error') { + throw new Error('Failed to persist model-error notification') + } + + this.refreshSession(sessionId) + } + + throw new Error('Model-error notification metadata stayed contended') + } + /** * Clear archive-related metadata on an archived session so it can be resumed. * - Removes `lifecycleState`, `archivedBy`, `archiveReason`, and stamps @@ -1182,24 +1326,25 @@ export class SessionCache { this.emitScratchlistChanged(oldSessionId) } - const mergedMetadata = this.mergeSessionMetadata(oldStored.metadata, newStored.metadata) - if (mergedMetadata !== null && mergedMetadata !== newStored.metadata) { - for (let attempt = 0; attempt < 2; attempt += 1) { - const latest = this.store.sessions.getSessionByNamespace(newSessionId, namespace) - if (!latest) break - const result = this.store.sessions.updateSessionMetadata( - newSessionId, - mergedMetadata, - latest.metadataVersion, - namespace, - { touchUpdatedAt: false } - ) - if (result.result === 'success') { - break - } - if (result.result === 'error') { - break - } + // Recompute merge against the live target row each attempt — a newer + // lastModelError (or other field) can land on newSessionId between the + // initial read and this write (version-mismatch retry). + for (let attempt = 0; attempt < 2; attempt += 1) { + const latest = this.store.sessions.getSessionByNamespace(newSessionId, namespace) + if (!latest) break + const mergedMetadata = this.mergeSessionMetadata(oldStored.metadata, latest.metadata) + if (mergedMetadata === null || mergedMetadata === latest.metadata) { + break + } + const result = this.store.sessions.updateSessionMetadata( + newSessionId, + mergedMetadata, + latest.metadataVersion, + namespace, + { touchUpdatedAt: false } + ) + if (result.result === 'success' || result.result === 'error') { + break } } @@ -1379,6 +1524,42 @@ export class SessionCache { changed = true } + // Preserve durable model-error alert state across resume/dedup row merges. + // Identity is eventId (wall-clock atTs is display-only and can go + // backwards after NTP/sleep). Carry old when new has none; when both + // share an eventId, merge hub watermarks. + type ModelErrorState = { + eventId?: string + atTs?: number + acknowledgedAt?: number + notifiedAt?: number + [key: string]: unknown + } + const oldError = oldObj.lastModelError as ModelErrorState | undefined + const newError = newObj.lastModelError as ModelErrorState | undefined + const oldId = typeof oldError?.eventId === 'string' ? oldError.eventId : null + const newId = typeof newError?.eventId === 'string' ? newError.eventId : null + if (oldError && oldId && !newError) { + merged.lastModelError = oldError + changed = true + } else if (oldError && newError && oldId && newId && oldId === newId) { + merged.lastModelError = { + ...oldError, + ...newError, + acknowledgedAt: newError.acknowledgedAt ?? oldError.acknowledgedAt, + notifiedAt: newError.notifiedAt ?? oldError.notifiedAt, + bridgedForEventId: newError.bridgedForEventId ?? oldError.bridgedForEventId, + retriedAndFailed: newError.retriedAndFailed === true || oldError.retriedAndFailed === true, + supersededByUserTurn: newError.supersededByUserTurn === true + || oldError.supersededByUserTurn === true, + bridgeable: newError.bridgeable === false || oldError.bridgeable === false + ? false + : (newError.bridgeable ?? oldError.bridgeable), + lastUserMessage: newError.lastUserMessage ?? oldError.lastUserMessage + } + changed = true + } + return changed ? merged : newMetadata } diff --git a/hub/src/sync/syncEngine.ts b/hub/src/sync/syncEngine.ts index 42067f5a10..7d183c55f8 100644 --- a/hub/src/sync/syncEngine.ts +++ b/hub/src/sync/syncEngine.ts @@ -60,6 +60,11 @@ import { } from './rpcGateway' import { SessionCache } from './sessionCache' import { ingestNotifySummaryFromMessage } from './workGraphNotifyIngest' +import { + readAutoBridgeTransientModelErrorsEnabled, + writeAutoBridgeTransientModelErrorsEnabled +} from '../config/autoBridgeTransientModelErrors' +import { getConfiguration } from '../configuration' type PiResumeAttempt = NonNullable['piResumeAttempt']> type PtyResumeAttempt = NonNullable['ptyResumeAttempt']> @@ -201,6 +206,15 @@ export class SyncEngine { * Defaults to "1" for unit tests; startHub overwrites with getOrCreateOwnerId(). */ private hubOwnerUserId: string = '1' + /** Test-only settings root so session-ready reconcile can run without full hub boot. */ + private settingsDataDirForTests: string | null = null + /** Serialize settings write/fanout with first-activation / session-ready reconcile. */ + private autoBridgeConfigTail: Promise = Promise.resolve() + /** + * Active Cursor sessions whose last auto-bridge Settings RPC failed. + * Heartbeats retry persisted desired state until reconcile succeeds. + */ + private readonly pendingAutoBridgeReconcile = new Set() constructor( private readonly store: Store, @@ -522,9 +536,16 @@ export class SyncEngine { serviceTier?: string | null collaborationMode?: CodexCollaborationMode }): void { + const before = this.sessionCache.getSession(payload.sid) + const wasActive = before?.active === true this.sessionCache.handleSessionAlive(payload) this.messageService.replayImmediateQueuedMessages(payload.sid) this.triggerDedupIfNeeded(payload.sid) + // First inactive → active: catch toggles while inactive (fanout is + // active-only). Also retry sessions whose prior fanout/rollback RPC failed. + if (!wasActive || this.pendingAutoBridgeReconcile.has(payload.sid)) { + void this.reconcileCursorAutoBridgeSetting(payload.sid) + } } handleSessionReady(payload: { sid: string; time: number }): void { @@ -538,9 +559,123 @@ export class SyncEngine { }) .catch(() => {}) } + // session-ready fires after set-session-config is registered (alive can + // race earlier). Re-read under the same lock as Settings fanout. + void this.reconcileCursorAutoBridgeSetting(payload.sid) this.triggerDedupIfNeeded(payload.sid) } + /** @internal test hook — hub settings dataDir without createConfiguration(). */ + setSettingsDataDirForTests(dataDir: string | null): void { + this.settingsDataDirForTests = dataDir + } + + /** + * Owner hub setting changed: push to every active default-namespace Cursor CLI. + * Serialized with first-activation / session-ready reconcile. + */ + async fanoutAutoBridgeTransientModelErrors(enabled: boolean): Promise { + await this.withAutoBridgeConfigLock(async () => { + await this.pushAutoBridgeSettingToActiveCursorSessions(enabled) + }) + } + + /** + * Persist the owner auto-bridge pref and fanout under one lock so concurrent + * Settings PUTs cannot interleave disk write / rollback / RPC push. + */ + async applyAutoBridgeTransientModelErrorsSetting( + dataDir: string, + enabled: boolean + ): Promise { + await this.withAutoBridgeConfigLock(async () => { + const previous = await readAutoBridgeTransientModelErrorsEnabled(dataDir) + await writeAutoBridgeTransientModelErrorsEnabled(dataDir, enabled) + try { + await this.pushAutoBridgeSettingToActiveCursorSessions(enabled) + } catch (error) { + await writeAutoBridgeTransientModelErrorsEnabled(dataDir, previous) + await this.pushAutoBridgeSettingToActiveCursorSessions(previous).catch(() => {}) + throw error + } + }) + } + + private async pushAutoBridgeSettingToActiveCursorSessions(enabled: boolean): Promise { + const targets = this.sessionCache.getSessions().filter( + (session) => session.active + && session.namespace === 'default' + && session.metadata?.flavor === 'cursor' + ) + const results = await Promise.allSettled( + targets.map((session) => this.rpcGateway.requestSessionConfig(session.id, { + autoBridgeTransientModelErrors: enabled + })) + ) + results.forEach((result, index) => { + const id = targets[index]!.id + if (result.status === 'rejected') { + this.pendingAutoBridgeReconcile.add(id) + } else { + this.pendingAutoBridgeReconcile.delete(id) + } + }) + if (results.some((result) => result.status === 'rejected')) { + throw new Error('Failed to update every active Cursor session') + } + } + + /** + * Push the current owner hub auto-bridge pref to one Cursor CLI. + * Best-effort: never throws into the socket path. + */ + async reconcileCursorAutoBridgeSetting(sessionId: string): Promise { + try { + await this.withAutoBridgeConfigLock(async () => { + const session = this.sessionCache.getSession(sessionId) + ?? this.sessionCache.refreshSession(sessionId) + if (!session) { + return + } + if (session.namespace !== 'default' || session.metadata?.flavor !== 'cursor') { + return + } + const dataDir = this.resolveSettingsDataDir() + if (!dataDir) { + return + } + const enabled = await readAutoBridgeTransientModelErrorsEnabled(dataDir) + await this.rpcGateway.requestSessionConfig(sessionId, { + autoBridgeTransientModelErrors: enabled + }) + this.pendingAutoBridgeReconcile.delete(sessionId) + }) + } catch (error) { + this.pendingAutoBridgeReconcile.add(sessionId) + console.warn( + `[sync] failed to reconcile auto-bridge setting for session ${sessionId}`, + error + ) + } + } + + private withAutoBridgeConfigLock(fn: () => Promise): Promise { + const run = this.autoBridgeConfigTail.then(fn, fn) + this.autoBridgeConfigTail = run.then(() => undefined, () => undefined) + return run + } + + private resolveSettingsDataDir(): string | null { + if (this.settingsDataDirForTests) { + return this.settingsDataDirForTests + } + try { + return getConfiguration().dataDir + } catch { + return null + } + } + clearQueuedThinkingGrace(sessionId: string): void { this.sessionCache.clearQueuedThinkingGrace(sessionId) } @@ -966,6 +1101,25 @@ export class SyncEngine { private reloadAll(): void { this.sessionCache.reloadAll() this.machineCache.reloadAll() + this.queuePersistedActiveCursorAutoBridgeReconcile() + } + + /** + * Hub restart: in-memory pendingAutoBridgeReconcile is empty. If SQLite + * restored a Cursor row as already active, the first heartbeat is not an + * inactive→active transition — seed those ids so handleSessionAlive still + * pushes persisted settings.json. + */ + private queuePersistedActiveCursorAutoBridgeReconcile(): void { + for (const session of this.sessionCache.getSessions()) { + if ( + session.active + && session.namespace === 'default' + && session.metadata?.flavor === 'cursor' + ) { + this.pendingAutoBridgeReconcile.add(session.id) + } + } } getOrCreateSession( @@ -1869,6 +2023,61 @@ export class SyncEngine { await this.sessionCache.updateSessionSummary(sessionId, text) } + async acknowledgeModelError(sessionId: string, eventId: string): Promise { + await this.sessionCache.acknowledgeModelError(sessionId, eventId) + } + + async markModelErrorNotified(sessionId: string, eventId: string): Promise { + await this.sessionCache.markModelErrorNotified(sessionId, eventId) + } + + async bridgeModelError(sessionId: string, eventId: string): Promise<{ ok: boolean; reason?: string }> { + const session = this.sessionCache.refreshSession(sessionId) + ?? this.sessionCache.getSession(sessionId) + if (!session) { + throw new Error('Session not found') + } + + const err = session.metadata?.lastModelError + if (!err) { + throw new Error('No model error to bridge') + } + if (err.eventId !== eventId) { + throw new Error('Model error changed; refresh before bridging.') + } + if (!err.transient) { + throw new Error('Model error is not transient') + } + if (err.bridgedForEventId === err.eventId) { + throw new Error('Model error was already bridged') + } + if (err.retriedAndFailed) { + throw new Error('Bridge already failed for this error') + } + if (err.supersededByUserTurn) { + throw new Error('Model error was superseded by a newer turn') + } + if (err.bridgeable === false) { + throw new Error('Model error is not bridgeable') + } + + // Do not mark bridgedForEventId here — CLI persists recovery only after + // the bridge prompt actually succeeds. + return await this.rpcGateway.bridgeModelError(sessionId, { + eventId: err.eventId, + atTs: err.atTs, + kind: err.kind, + rawSnippet: err.rawSnippet, + lastUserMessage: err.lastUserMessage, + priorAssistantClaimsDone: err.priorAssistantClaimsDone, + transient: err.transient, + bridgedForEventId: err.bridgedForEventId, + retriedAndFailed: err.retriedAndFailed, + supersededByUserTurn: err.supersededByUserTurn, + bridgeable: err.bridgeable + }) + } + async deleteSession(sessionId: string): Promise { await this.sessionCache.deleteSession(sessionId) } @@ -1883,6 +2092,7 @@ export class SyncEngine { serviceTier?: string | null collaborationMode?: CodexCollaborationMode copilotAgentMode?: CopilotAgentMode + autoBridgeTransientModelErrors?: boolean } ): Promise { const session = this.sessionCache.getSession(sessionId) diff --git a/hub/src/sync/syncEngineReopenPreservesPtyId.test.ts b/hub/src/sync/syncEngineReopenPreservesPtyId.test.ts index 1f328d3b3a..bde878a0e8 100644 --- a/hub/src/sync/syncEngineReopenPreservesPtyId.test.ts +++ b/hub/src/sync/syncEngineReopenPreservesPtyId.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it, beforeEach } from 'bun:test' +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' import type { Session } from '@hapi/protocol/types' import { Store } from '../store' import { RpcRegistry } from '../socket/rpcRegistry' @@ -28,6 +28,16 @@ describe('SyncEngine reopen/resume PTY session id preservation', () => { const NAMESPACE = 'default' + /** Minimal MachineCache stand-in; must include expireInactive (SyncEngine's 5s timer). */ + function fakeMachineCache() { + return { + getOnlineMachinesByNamespace: () => [ + { id: 'machine-x', metadata: { host: 'localhost' } } + ], + expireInactive: () => {} + } + } + function baseMetadata(overrides: Record = {}): Record { return { path: '/tmp/proj', @@ -50,13 +60,7 @@ describe('SyncEngine reopen/resume PTY session id preservation', () => { function installFakeRunner(): void { const cache = (engine as unknown as { sessionCache: import('./sessionCache').SessionCache }).sessionCache - ;(engine as unknown as { machineCache: unknown }).machineCache = { - getOnlineMachinesByNamespace: () => [ - { id: 'machine-x', metadata: { host: 'localhost' } } - ], - expireInactive: () => {} - } - ;(engine as unknown as { waitForSessionActive: unknown }).waitForSessionActive = async () => true + ;(engine as unknown as { machineCache: unknown }).machineCache = fakeMachineCache() ;(engine as unknown as { waitForSessionActive: unknown }).waitForSessionActive = async () => true ;(engine as unknown as { waitForSessionReady: unknown }).waitForSessionReady = async () => 'ready' ;(engine as unknown as { rpcGateway: { spawnSession: unknown } }).rpcGateway.spawnSession = async (...args: unknown[]) => { @@ -99,6 +103,10 @@ describe('SyncEngine reopen/resume PTY session id preservation', () => { installFakeRunner() }) + afterEach(() => { + engine.stop() + }) + it('clears stale readiness before spawning the same-id PTY replacement', async () => { const sessionId = insertSession( 'pty-session-stale-ready', @@ -236,11 +244,7 @@ describe('SyncEngine reopen/resume PTY session id preservation', () => { engine.handleSessionAlive({ sid: sessionId, time: Date.now() }) const restarted = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never) - ;(restarted as any).machineCache = { - getOnlineMachinesByNamespace: () => [{ id: 'machine-x', metadata: { host: 'localhost' } }], - expireInactive: () => {} - } - ;(restarted as any).rpcGateway.stopRunnerSession = async () => 'still_alive' + ;(restarted as any).machineCache = fakeMachineCache() ;(restarted as any).rpcGateway.stopRunnerSession = async () => 'still_alive' const result = await restarted.reopenSession(sessionId, NAMESPACE) @@ -262,11 +266,7 @@ describe('SyncEngine reopen/resume PTY session id preservation', () => { engine.handleSessionAlive({ sid: sessionId, time: Date.now() }) const restarted = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never) - ;(restarted as any).machineCache = { - getOnlineMachinesByNamespace: () => [{ id: 'machine-x', metadata: { host: 'localhost' } }], - expireInactive: () => {} - } - ;(restarted as any).rpcGateway.stopRunnerSession = async () => 'already_gone' + ;(restarted as any).machineCache = fakeMachineCache() ;(restarted as any).rpcGateway.stopRunnerSession = async () => 'already_gone' ;(restarted as any).rpcGateway.spawnSession = async () => { restarted.handleSessionAlive({ sid: sessionId, time: Date.now() }) restarted.handleSessionReady({ sid: sessionId, time: Date.now() }) diff --git a/hub/src/telegram/bot.ts b/hub/src/telegram/bot.ts index cddbe15223..4b249b06c0 100644 --- a/hub/src/telegram/bot.ts +++ b/hub/src/telegram/bot.ts @@ -9,8 +9,15 @@ import { Bot, Context, InlineKeyboard } from 'grammy' import { SyncEngine, Session, type Machine } from '../sync/syncEngine' import { handleCallback, CallbackContext } from './callbacks' import { formatReadyNotification, formatSessionNotification, createNotificationKeyboard } from './sessionView' -import { getAgentName } from '../notifications/sessionInfo' -import type { NotificationChannel, TaskNotification } from '../notifications/notificationTypes' +import { getAgentName, getSessionName } from '../notifications/sessionInfo' +import type { + ModelErrorNotification, + ModelErrorSendOutcome, + NotificationChannel, + TaskNotification +} from '../notifications/notificationTypes' +import type { NotificationSendContext } from '../notifications/notificationSendContext' +import { formatModelErrorBody, formatModelErrorTitle } from '../notifications/modelErrorCopy' import type { Store } from '../store' export interface BotContext extends Context { @@ -264,6 +271,43 @@ export class HappyBot implements NotificationChannel { } } + async sendModelError( + session: Session, + notification: ModelErrorNotification, + _ctx?: NotificationSendContext + ): Promise { + // No active-session guard: bounded retries must still reach Telegram + // if the session ended between the first attempt and the timer. + + const agentName = getAgentName(session) + const sessionName = getSessionName(session) + const title = formatModelErrorTitle(notification.kind) + const body = formatModelErrorBody(notification, { agentName, sessionName }) + // Plain text (no parse_mode): sessionName can contain + // Markdown metacharacters; Telegram drops the whole message on parse errors. + const text = `\u{1F6A8} Model error - ${title}\n\n${body}` + const url = buildMiniAppDeepLink(this.publicUrl, `session_${session.id}`) + const keyboard = new InlineKeyboard().webApp('Open Session', url) + + const chatIds = this.getBoundChatIds(session.namespace) + if (chatIds.length === 0) { + return 'unavailable' + } + + let delivered = 0 + for (const chatId of chatIds) { + try { + await this.bot.api.sendMessage(chatId, text, { + reply_markup: keyboard + }) + delivered++ + } catch (error) { + console.error(`[HAPIBot] Failed to send model-error notification to chat ${chatId}:`, error) + } + } + return delivered > 0 ? 'delivered' : 'failed' + } + async sendTaskNotification(session: Session, notification: TaskNotification): Promise { if (!session.active) { return diff --git a/hub/src/web/routes/cli.test.ts b/hub/src/web/routes/cli.test.ts index f903364320..f6d0f6984a 100644 --- a/hub/src/web/routes/cli.test.ts +++ b/hub/src/web/routes/cli.test.ts @@ -1,7 +1,8 @@ -import { beforeAll, describe, expect, it, mock } from 'bun:test' +import { afterEach, beforeAll, describe, expect, it, mock } from 'bun:test' import { Hono } from 'hono' import type { SyncEngine } from '../../sync/syncEngine' -import { createConfiguration } from '../../configuration' +import { createConfiguration, getConfiguration } from '../../configuration' +import { writeAutoBridgeTransientModelErrorsEnabled } from '../../config/autoBridgeTransientModelErrors' import { createCliRoutes } from './cli' import { SessionIdentityConflictError } from '../../store/sessions' @@ -289,3 +290,61 @@ describe('cli lazy session creation', () => { expect(response.status).toBe(409) }) }) + +describe('cli autoBridgeTransientModelErrors namespace gate', () => { + afterEach(async () => { + await writeAutoBridgeTransientModelErrorsEnabled(getConfiguration().dataDir, false) + }) + + it('returns hub auto-bridge only for the default namespace', async () => { + await writeAutoBridgeTransientModelErrorsEnabled(getConfiguration().dataDir, true) + const sessionId = '22222222-2222-4222-8222-222222222222' + const app = createApp({ + getOrCreateSession: () => ({ id: sessionId }), + resolveSessionAccess: () => ({ + ok: true, + session: { id: sessionId }, + sessionId + }) + } as never) + + const defaultCreate = await app.request('/cli/sessions', { + method: 'POST', + headers: { + ...authHeaders(), + 'content-type': 'application/json' + }, + body: JSON.stringify({ + id: sessionId, + tag: 'auto-bridge-default', + metadata: {} + }) + }) + expect(defaultCreate.status).toBe(200) + const defaultBody = await defaultCreate.json() as { autoBridgeTransientModelErrors?: boolean } + expect(defaultBody.autoBridgeTransientModelErrors).toBe(true) + + const tenantCreate = await app.request('/cli/sessions', { + method: 'POST', + headers: { + authorization: 'Bearer test-token:tenant-a', + 'content-type': 'application/json' + }, + body: JSON.stringify({ + id: sessionId, + tag: 'auto-bridge-tenant', + metadata: {} + }) + }) + expect(tenantCreate.status).toBe(200) + const tenantCreateBody = await tenantCreate.json() as { autoBridgeTransientModelErrors?: boolean } + expect(tenantCreateBody.autoBridgeTransientModelErrors).toBe(false) + + const tenantGet = await app.request(`/cli/sessions/${sessionId}`, { + headers: { authorization: 'Bearer test-token:tenant-a' } + }) + expect(tenantGet.status).toBe(200) + const tenantGetBody = await tenantGet.json() as { autoBridgeTransientModelErrors?: boolean } + expect(tenantGetBody.autoBridgeTransientModelErrors).toBe(false) + }) +}) diff --git a/hub/src/web/routes/cli.ts b/hub/src/web/routes/cli.ts index 5e2345860c..de2b53f3c6 100644 --- a/hub/src/web/routes/cli.ts +++ b/hub/src/web/routes/cli.ts @@ -8,6 +8,7 @@ import { PROTOCOL_VERSION } from '@hapi/protocol' import { getConfiguration } from '../../configuration' +import { readAutoBridgeTransientModelErrorsEnabled } from '../../config/autoBridgeTransientModelErrors' import { readSessionSummaryContractEnabled } from '../../config/sessionSummaryContract' import { constantTimeEquals } from '../../utils/crypto' import { parseAccessToken } from '../../utils/accessToken' @@ -129,10 +130,13 @@ export function createCliRoutes(getSyncEngine: () => SyncEngine | null): Hono SyncEngine | null): Hono { diff --git a/hub/src/web/routes/hubSettings.test.ts b/hub/src/web/routes/hubSettings.test.ts index 10ddfd6ae6..8c60decceb 100644 --- a/hub/src/web/routes/hubSettings.test.ts +++ b/hub/src/web/routes/hubSettings.test.ts @@ -5,6 +5,7 @@ import { join } from 'node:path' import { Hono } from 'hono' import type { WebAppEnv } from '../middleware/auth' import { createHubSettingsRoutes } from './hubSettings' +import { writeAutoBridgeTransientModelErrorsEnabled } from '../../config/autoBridgeTransientModelErrors' import { writeSessionSummaryContractEnabled } from '../../config/sessionSummaryContract' import { writeSessionSummaryInChatEnabled } from '../../config/sessionSummaryInChat' @@ -27,18 +28,19 @@ describe('GET/PUT /api/hub-settings', () => { return { app, dataDir } } - it('returns default off for emit and chat display', async () => { + it('returns default off for emit, chat display, and auto-bridge', async () => { const { app } = await createApp() const response = await app.request('/api/hub-settings') expect(response.status).toBe(200) expect(response.headers.get('cache-control')).toBe('no-store') expect(await response.json()).toEqual({ sessionSummaryContract: false, - sessionSummaryInChat: false + sessionSummaryInChat: false, + autoBridgeTransientModelErrors: false }) }) - it('persists emit toggle for owner without changing display', async () => { + it('persists emit toggle for owner without changing display or auto-bridge', async () => { const { app } = await createApp() const put = await app.request('/api/hub-settings', { method: 'PUT', @@ -48,13 +50,15 @@ describe('GET/PUT /api/hub-settings', () => { expect(put.status).toBe(200) expect(await put.json()).toEqual({ sessionSummaryContract: true, - sessionSummaryInChat: false + sessionSummaryInChat: false, + autoBridgeTransientModelErrors: false }) const get = await app.request('/api/hub-settings') expect(await get.json()).toEqual({ sessionSummaryContract: true, - sessionSummaryInChat: false + sessionSummaryInChat: false, + autoBridgeTransientModelErrors: false }) }) @@ -70,7 +74,23 @@ describe('GET/PUT /api/hub-settings', () => { expect(put.status).toBe(200) expect(await put.json()).toEqual({ sessionSummaryContract: true, - sessionSummaryInChat: true + sessionSummaryInChat: true, + autoBridgeTransientModelErrors: false + }) + }) + + it('persists autoBridgeTransientModelErrors toggle for owner', async () => { + const { app } = await createApp() + const put = await app.request('/api/hub-settings', { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ autoBridgeTransientModelErrors: true }) + }) + expect(put.status).toBe(200) + expect(await put.json()).toEqual({ + sessionSummaryContract: false, + sessionSummaryInChat: false, + autoBridgeTransientModelErrors: true }) }) @@ -94,6 +114,20 @@ describe('GET/PUT /api/hub-settings', () => { expect(response.status).toBe(400) }) + it('rejects combined two-field updates', async () => { + const { app } = await createApp() + const response = await app.request('/api/hub-settings', { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + sessionSummaryContract: true, + autoBridgeTransientModelErrors: true + }) + }) + expect(response.status).toBe(400) + expect(await response.json()).toEqual({ error: 'Update one hub setting per request' }) + }) + it('rejects non-default namespaces for PUT but allows GET', async () => { const { app, dataDir } = await createApp('default') await writeSessionSummaryInChatEnabled(dataDir, true) @@ -109,7 +143,8 @@ describe('GET/PUT /api/hub-settings', () => { expect(get.status).toBe(200) expect(await get.json()).toEqual({ sessionSummaryContract: false, - sessionSummaryInChat: true + sessionSummaryInChat: true, + autoBridgeTransientModelErrors: false }) const put = await tenantApp.request('/api/hub-settings', { @@ -124,10 +159,12 @@ describe('GET/PUT /api/hub-settings', () => { const { app, dataDir } = await createApp() await writeSessionSummaryContractEnabled(dataDir, true) await writeSessionSummaryInChatEnabled(dataDir, true) + await writeAutoBridgeTransientModelErrorsEnabled(dataDir, true) const response = await app.request('/api/hub-settings') expect(await response.json()).toEqual({ sessionSummaryContract: true, - sessionSummaryInChat: true + sessionSummaryInChat: true, + autoBridgeTransientModelErrors: true }) }) }) diff --git a/hub/src/web/routes/hubSettings.ts b/hub/src/web/routes/hubSettings.ts index 3d2a087789..cb23215b3a 100644 --- a/hub/src/web/routes/hubSettings.ts +++ b/hub/src/web/routes/hubSettings.ts @@ -1,31 +1,46 @@ import { Hono } from 'hono' import { UpdateHubSettingsRequestSchema, type HubSettingsResponse } from '@hapi/protocol' import { - getSettingsFile, - readSettingsOrThrow, - updateSettings, - type Settings -} from '../../config/settings' + readAutoBridgeTransientModelErrorsEnabled, + writeAutoBridgeTransientModelErrorsEnabled +} from '../../config/autoBridgeTransientModelErrors' +import { + readSessionSummaryContractEnabled, + writeSessionSummaryContractEnabled +} from '../../config/sessionSummaryContract' +import { + readSessionSummaryInChatEnabled, + writeSessionSummaryInChatEnabled +} from '../../config/sessionSummaryInChat' +import type { SyncEngine } from '../../sync/syncEngine' import type { WebAppEnv } from '../middleware/auth' const OWNER_ONLY_ERROR = 'Hub settings are only available to the hub owner' -function toHubSettings(settings: Settings): HubSettingsResponse { +async function readHubSettings(dataDir: string): Promise { + const [sessionSummaryContract, sessionSummaryInChat, autoBridgeTransientModelErrors] = await Promise.all([ + readSessionSummaryContractEnabled(dataDir), + readSessionSummaryInChatEnabled(dataDir), + readAutoBridgeTransientModelErrorsEnabled(dataDir) + ]) return { - sessionSummaryContract: settings.sessionSummaryContract === true, - sessionSummaryInChat: settings.sessionSummaryInChat === true + sessionSummaryContract, + sessionSummaryInChat, + autoBridgeTransientModelErrors } } -export function createHubSettingsRoutes(dataDir: string): Hono { +export function createHubSettingsRoutes( + dataDir: string, + getSyncEngine?: () => SyncEngine | null +): Hono { const app = new Hono() // Authenticated readers (any namespace) can observe hub-wide display/emit // flags. Mutations stay owner-only below. app.get('/hub-settings', async (c) => { c.header('Cache-Control', 'no-store') - const settings = await readSettingsOrThrow(getSettingsFile(dataDir)) - return c.json(toHubSettings(settings)) + return c.json(await readHubSettings(dataDir)) }) app.put('/hub-settings', async (c) => { @@ -37,21 +52,46 @@ export function createHubSettingsRoutes(dataDir: string): Hono { if (!parsed.success) { return c.json({ error: 'Invalid body' }, 400) } - const response = await updateSettings(getSettingsFile(dataDir), (current) => { - const settings: Settings = { ...current } - if (parsed.data.sessionSummaryContract !== undefined) { - settings.sessionSummaryContract = parsed.data.sessionSummaryContract - } - if (parsed.data.sessionSummaryInChat !== undefined) { - settings.sessionSummaryInChat = parsed.data.sessionSummaryInChat - } - return { - settings, - result: toHubSettings(settings) + const providedFieldCount = [ + parsed.data.sessionSummaryContract, + parsed.data.sessionSummaryInChat, + parsed.data.autoBridgeTransientModelErrors + ].filter((value) => value !== undefined).length + if (providedFieldCount > 1) { + return c.json({ error: 'Update one hub setting per request' }, 400) + } + if (parsed.data.sessionSummaryContract !== undefined) { + await writeSessionSummaryContractEnabled( + dataDir, + parsed.data.sessionSummaryContract + ) + } + if (parsed.data.sessionSummaryInChat !== undefined) { + await writeSessionSummaryInChatEnabled( + dataDir, + parsed.data.sessionSummaryInChat + ) + } + if (parsed.data.autoBridgeTransientModelErrors !== undefined) { + const enabled = parsed.data.autoBridgeTransientModelErrors + const engine = getSyncEngine?.() ?? null + if (engine) { + try { + // Disk write + fanout + rollback share SyncEngine's lock so + // concurrent PUTs cannot leave CLI prefs ahead of settings.json. + await engine.applyAutoBridgeTransientModelErrorsSetting(dataDir, enabled) + } catch (error) { + const message = error instanceof Error + ? error.message + : 'Failed to update every active Cursor session' + return c.json({ error: message }, 409) + } + } else { + await writeAutoBridgeTransientModelErrorsEnabled(dataDir, enabled) } - }) + } c.header('Cache-Control', 'no-store') - return c.json(response) + return c.json(await readHubSettings(dataDir)) }) return app diff --git a/hub/src/web/routes/sessions.test.ts b/hub/src/web/routes/sessions.test.ts index d690e0521c..9cf0df3726 100644 --- a/hub/src/web/routes/sessions.test.ts +++ b/hub/src/web/routes/sessions.test.ts @@ -63,6 +63,7 @@ function createApp(session: Session, opts?: { getSessionExport?: (sessionId: string, session: Session, options?: { force?: boolean }) => unknown sessionExists?: boolean archiveSession?: (sessionId: string) => Promise + acknowledgeModelError?: (sessionId: string, eventId: string) => Promise getCursorChatStoreStatus?: SyncEngine['getCursorChatStoreStatus'] listCodexModelsForSession?: SyncEngine['listCodexModelsForSession'] forkConversation?: SyncEngine['forkConversation'] @@ -71,6 +72,7 @@ function createApp(session: Session, opts?: { updateSessionSummary?: SyncEngine['updateSessionSummary'] setSessionPinned?: (sessionId: string, pinned: boolean) => void setSessionPinMode?: (sessionId: string, mode: 'none' | 'project' | 'global') => void + namespace?: string }) { const applySessionConfigCalls: Array<[string, Record]> = [] const applySessionConfig = async (sessionId: string, config: Record) => { @@ -124,6 +126,7 @@ function createApp(session: Session, opts?: { })) const sessionExists = opts?.sessionExists !== false const archiveSessionMock = opts?.archiveSession ?? (async () => {}) + const acknowledgeModelErrorMock = opts?.acknowledgeModelError ?? (async () => {}) const engine = { resolveSessionAccess: () => sessionExists ? { ok: true, sessionId: session.id, session } @@ -147,6 +150,7 @@ function createApp(session: Session, opts?: { archiveSession: archiveSessionMock, setSessionPinned: opts?.setSessionPinned ?? (() => {}), setSessionPinMode: opts?.setSessionPinMode ?? (() => {}), + acknowledgeModelError: acknowledgeModelErrorMock, getSessionExport: opts?.getSessionExport ?? (() => ({ type: 'success', payload: { @@ -169,7 +173,7 @@ function createApp(session: Session, opts?: { const app = new Hono() app.use('*', async (c, next) => { - c.set('namespace', 'default') + c.set('namespace', opts?.namespace ?? 'default') await next() }) app.route('/api', createSessionsRoutes(() => engine as SyncEngine)) @@ -1564,4 +1568,133 @@ describe('sessions routes', () => { expect(body.sessions.map((s) => s.id)).toEqual(['new-inactive']) }) + describe('POST /sessions/:id/model-error/acknowledge', () => { + it('forwards eventId to the engine when body is valid', async () => { + const calls: Array<[string, string]> = [] + const { app } = createApp(createSession(), { + acknowledgeModelError: async (sessionId, eventId) => { + calls.push([sessionId, eventId]) + } + }) + + const response = await app.request('/api/sessions/session-1/model-error/acknowledge', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ eventId: 'evt-ack-1' }) + }) + + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ ok: true }) + expect(calls).toEqual([['session-1', 'evt-ack-1']]) + }) + + it('returns 400 when eventId is missing', async () => { + let called = false + const { app } = createApp(createSession(), { + acknowledgeModelError: async () => { called = true } + }) + + const response = await app.request('/api/sessions/session-1/model-error/acknowledge', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({}) + }) + + expect(response.status).toBe(400) + expect(called).toBe(false) + }) + + it('returns 409 when the displayed error no longer matches', async () => { + const { app } = createApp(createSession(), { + acknowledgeModelError: async () => { + throw new Error('Model error changed; refresh before acknowledging.') + } + }) + + const response = await app.request('/api/sessions/session-1/model-error/acknowledge', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ eventId: 'evt-stale' }) + }) + + expect(response.status).toBe(409) + expect(await response.json()).toEqual({ + error: 'Model error changed; refresh before acknowledging.' + }) + }) + }) + + describe('POST /sessions/:id/model-error/bridge', () => { + it('returns 409 session_inactive when the session is not active', async () => { + const { app } = createApp(createSession({ + active: false, + metadata: { path: '/tmp/project', host: 'localhost', flavor: 'cursor' } + })) + + const response = await app.request('/api/sessions/session-1/model-error/bridge', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ eventId: 'evt-1' }) + }) + + expect(response.status).toBe(409) + expect(await response.json()).toEqual({ + error: 'Session is inactive', + code: 'session_inactive' + }) + }) + + it('returns 400 when eventId is missing', async () => { + const { app } = createApp(createSession({ + metadata: { path: '/tmp/project', host: 'localhost', flavor: 'cursor' } + })) + + const response = await app.request('/api/sessions/session-1/model-error/bridge', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({}) + }) + + expect(response.status).toBe(400) + }) + }) + + describe('POST /sessions/:id/model-error/auto-bridge-setting', () => { + it('rejects non-default namespaces with 403 and does not apply config', async () => { + const { app, applySessionConfigCalls } = createApp(createSession({ + metadata: { path: '/tmp/project', host: 'localhost', flavor: 'cursor' } + }), { namespace: 'tenant-a' }) + + const response = await app.request('/api/sessions/session-1/model-error/auto-bridge-setting', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ enabled: true }) + }) + + expect(response.status).toBe(403) + expect(await response.json()).toEqual({ + error: 'Model error auto-bridge is only available to the hub owner' + }) + expect(applySessionConfigCalls).toEqual([]) + }) + + it('applies the setting for the default namespace', async () => { + const { app, applySessionConfigCalls } = createApp(createSession({ + metadata: { path: '/tmp/project', host: 'localhost', flavor: 'cursor' } + })) + + const response = await app.request('/api/sessions/session-1/model-error/auto-bridge-setting', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ enabled: true }) + }) + + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ ok: true }) + expect(applySessionConfigCalls).toEqual([ + ['session-1', { autoBridgeTransientModelErrors: true }] + ]) + }) + }) + }) diff --git a/hub/src/web/routes/sessions.ts b/hub/src/web/routes/sessions.ts index 44238c9e4f..7ce6cb652a 100644 --- a/hub/src/web/routes/sessions.ts +++ b/hub/src/web/routes/sessions.ts @@ -1,4 +1,6 @@ import { + AcknowledgeModelErrorRequestSchema, + BridgeModelErrorRequestSchema, CursorMigrateToAcpRequestSchema, DeleteUploadRequestSchema, ForkConversationRequestSchema, @@ -460,6 +462,125 @@ export function createSessionsRoutes(getSyncEngine: () => SyncEngine | null): Ho return c.json({ ok: true }) }) + app.post('/sessions/:id/model-error/acknowledge', async (c) => { + const engine = requireSyncEngine(c, getSyncEngine) + if (engine instanceof Response) { + return engine + } + + const sessionResult = requireSessionFromParam(c, engine) + if (sessionResult instanceof Response) { + return sessionResult + } + + const body = await c.req.json().catch(() => null) + const parsed = AcknowledgeModelErrorRequestSchema.safeParse(body) + if (!parsed.success) { + return c.json({ error: 'Invalid body', issues: parsed.error.issues }, 400) + } + + try { + await engine.acknowledgeModelError(sessionResult.sessionId, parsed.data.eventId) + return c.json({ ok: true }) + } catch (error) { + const message = error instanceof Error ? error.message : 'Failed to acknowledge model error' + if ( + message.includes('concurrently') + || message.includes('version') + || message.includes('changed') + ) { + return c.json({ error: message }, 409) + } + return c.json({ error: message }, 500) + } + }) + + app.post('/sessions/:id/model-error/bridge', async (c) => { + const engine = requireSyncEngine(c, getSyncEngine) + if (engine instanceof Response) { + return engine + } + + // Bridge needs a live CLI RPC target; inactive → 409 session_inactive. + const sessionResult = requireSessionFromParam(c, engine, { requireActive: true }) + if (sessionResult instanceof Response) { + return sessionResult + } + + const flavor = sessionResult.session.metadata?.flavor ?? 'claude' + if (flavor !== 'cursor') { + return c.json({ error: 'Model error bridge is only supported for Cursor sessions' }, 400) + } + + const body = await c.req.json().catch(() => null) + const parsed = BridgeModelErrorRequestSchema.safeParse(body) + if (!parsed.success) { + return c.json({ error: 'Invalid body', issues: parsed.error.issues }, 400) + } + + try { + const result = await engine.bridgeModelError( + sessionResult.sessionId, + parsed.data.eventId + ) + if (!result.ok) { + return c.json({ ok: false, reason: result.reason ?? 'not_bridgeable' }, 409) + } + return c.json({ ok: true }) + } catch (error) { + const message = error instanceof Error ? error.message : 'Failed to bridge model error' + if ( + message.includes('not active') + || message.includes('not transient') + || message.includes('already bridged') + || message.includes('already failed') + || message.includes('changed') + || message.includes('superseded') + || message.includes('not bridgeable') + ) { + return c.json({ error: message }, 409) + } + return c.json({ error: message }, 500) + } + }) + + app.post('/sessions/:id/model-error/auto-bridge-setting', async (c) => { + // Owner-only — matches CLI create/get which force false for tenants. + if (c.get('namespace') !== 'default') { + return c.json({ error: 'Model error auto-bridge is only available to the hub owner' }, 403) + } + + const engine = requireSyncEngine(c, getSyncEngine) + if (engine instanceof Response) { + return engine + } + + const sessionResult = requireSessionFromParam(c, engine, { requireActive: true }) + if (sessionResult instanceof Response) { + return sessionResult + } + + const flavor = sessionResult.session.metadata?.flavor ?? 'claude' + if (flavor !== 'cursor') { + return c.json({ error: 'Model error auto-bridge is only supported for Cursor sessions' }, 400) + } + + const body = await c.req.json().catch(() => null) + if (!body || typeof body !== 'object' || typeof body.enabled !== 'boolean') { + return c.json({ error: 'Invalid body' }, 400) + } + + try { + await engine.applySessionConfig(sessionResult.sessionId, { + autoBridgeTransientModelErrors: body.enabled + }) + return c.json({ ok: true }) + } catch (error) { + const message = error instanceof Error ? error.message : 'Failed to apply auto-bridge setting' + return c.json({ error: message }, 409) + } + }) + app.post('/sessions/:id/migrate-to-acp', async (c) => { const engine = requireSyncEngine(c, getSyncEngine) if (engine instanceof Response) { diff --git a/hub/src/web/server.ts b/hub/src/web/server.ts index 3444b88ce6..ba1b8fa8fe 100644 --- a/hub/src/web/server.ts +++ b/hub/src/web/server.ts @@ -291,7 +291,7 @@ function createWebApp(options: { app.route('/api', createPermissionsRoutes(options.getSyncEngine)) app.route('/api', createMachinesRoutes(options.getSyncEngine)) app.route('/api', createStorageRoutes(configuration.dbPath)) - app.route('/api', createHubSettingsRoutes(configuration.dataDir)) + app.route('/api', createHubSettingsRoutes(configuration.dataDir, options.getSyncEngine)) app.route('/api', createUsageRoutes(options.store)) app.route('/api', createGitRoutes(options.getSyncEngine)) // 中文注释：这里提供两类 Codex 辅助能力：扫描本地 transcript 以导入到 Hapi，以及按需重启 Codex Desktop 客户端。 diff --git a/ios/HapiNotificationService/NotificationService.swift b/ios/HapiNotificationService/NotificationService.swift index 981df0813e..f23b0aac0f 100644 --- a/ios/HapiNotificationService/NotificationService.swift +++ b/ios/HapiNotificationService/NotificationService.swift @@ -97,7 +97,10 @@ final class NotificationService: UNNotificationServiceExtension { } // Coalescing (Android `type-` tag): a newer push of the // same type for the same session replaces the previous one. - content.threadIdentifier = "\(type.isEmpty ? "unknown" : type)-\(fields["sessionId"] ?? "")" + content.threadIdentifier = { + if let tag = fields["tag"], !isBlank(tag) { return tag } + return "\(type.isEmpty ? "unknown" : type)-\(fields["sessionId"] ?? "")" + }() // Decrypted plaintext for the app-side tap/action handlers — key name // is `PushCoordinator.decryptedUserInfoKey`. var userInfo = content.userInfo diff --git a/ios/Packages/HapiKit/Sources/HapiClient/Push/PushPayload.swift b/ios/Packages/HapiKit/Sources/HapiClient/Push/PushPayload.swift index 710ce8a2a1..2a0213a375 100644 --- a/ios/Packages/HapiKit/Sources/HapiClient/Push/PushPayload.swift +++ b/ios/Packages/HapiKit/Sources/HapiClient/Push/PushPayload.swift @@ -7,6 +7,7 @@ public enum PushType: String, Sendable { case ready case permissionRequest = "permission-request" case taskNotification = "task-notification" + case modelError = "model-error" } /// `severity` — visual urgency accent (`hub/src/fcm/fcmService.ts`). @@ -71,6 +72,8 @@ public struct PushPayload: Equatable, Sendable { public var severity: PushSeverity? public var contractVersion: String? public var notifySummary: PushNotifySummary? + /// Hub coalescing tag (`FcmSendPayload.tag`), when present. + public var tag: String? /// The contract version this client implements. public static let contractVersion = "1" @@ -91,7 +94,7 @@ public struct PushPayload: Equatable, Sendable { switch type { case .permissionRequest: return requestId != nil case .ready, .taskNotification: return true - case nil: return false + case .modelError, nil: return false } } @@ -103,6 +106,13 @@ public struct PushPayload: Equatable, Sendable { supportsActions ? rawType : nil } + /// Coalescing identity: prefer the hub-supplied `tag` (event-specific + /// for `model-error`) before falling back to `type-`. + public var notificationTag: String { + if let tag, !tag.isPushBlank { return tag } + return "\(rawType.isEmpty ? "unknown" : rawType)-\(sessionId)" + } + /// Title to render; falls back to the session name, then a constant. public var displayTitle: String { if let title, !title.isPushBlank { return title } @@ -134,7 +144,8 @@ public struct PushPayload: Equatable, Sendable { requestId: String? = nil, severity: PushSeverity? = nil, contractVersion: String? = nil, - notifySummary: PushNotifySummary? = nil + notifySummary: PushNotifySummary? = nil, + tag: String? = nil ) { self.type = type self.rawType = rawType @@ -147,6 +158,7 @@ public struct PushPayload: Equatable, Sendable { self.severity = severity self.contractVersion = contractVersion self.notifySummary = notifySummary + self.tag = tag } // MARK: - Parsing @@ -170,7 +182,8 @@ public struct PushPayload: Equatable, Sendable { requestId: nonBlank(data["requestId"]), severity: data["severity"].flatMap(PushSeverity.init(rawValue:)), contractVersion: nonBlank(data["contractVersion"]), - notifySummary: data["notifySummary"].flatMap(parseNotifySummary) + notifySummary: data["notifySummary"].flatMap(parseNotifySummary), + tag: nonBlank(data["tag"]) ) } diff --git a/ios/Packages/HapiKit/Tests/HapiClientTests/Push/PushPayloadTests.swift b/ios/Packages/HapiKit/Tests/HapiClientTests/Push/PushPayloadTests.swift index e8a614de46..e5650e65b5 100644 --- a/ios/Packages/HapiKit/Tests/HapiClientTests/Push/PushPayloadTests.swift +++ b/ios/Packages/HapiKit/Tests/HapiClientTests/Push/PushPayloadTests.swift @@ -74,6 +74,34 @@ struct PushPayloadTests { #expect(payload.displayTitle == "Permission needed") } + @Test func modelErrorIsKnownTypeAndPrefersHubTag() throws { + let eventTag = "model-error-sess-1-evt-1710000000000" + let payload = try #require(PushPayload.parse(dictionary: [ + "type": "model-error", + "sessionId": "sess-1", + "title": "Rate limited", + "body": "status 429", + "severity": "error", + "contractVersion": "1", + "tag": eventTag, + ])) + #expect(payload.type == .modelError) + #expect(payload.rawType == "model-error") + #expect(payload.notificationTag == eventTag) + #expect(!payload.supportsActions) + #expect(payload.categoryIdentifier == nil) + #expect(payload.severity == .error) + } + + @Test func modelErrorWithoutHubTagFallsBackToTypeSessionId() throws { + let payload = try #require(PushPayload.parse(dictionary: [ + "type": "model-error", + "sessionId": "sess-1", + "contractVersion": "1", + ])) + #expect(payload.notificationTag == "model-error-sess-1") + } + @Test func absentContractVersionCountsAsKnown() throws { var fields = fullFields fields["contractVersion"] = nil diff --git a/ios/README.md b/ios/README.md index 962a36de4d..4114889a39 100644 --- a/ios/README.md +++ b/ios/README.md @@ -663,7 +663,9 @@ End-to-end encrypted APNs push, mirroring the Android FCM stack carries a ~60-line copy of the HapiKit `PushEnvelope` decrypt, kept honest by the shared test vector. - **Actions.** `permission-request` → Allow / Deny; `ready` and - `task-notification` → inline Reply. Handlers run in the notification + `task-notification` → inline Reply. `model-error` is tap-to-open only + (HIGH / heads-up channel on Android; event-specific coalescing tag). + Handlers run in the notification delegate's async completion and resolve the owning hub Android-style (active hub first, then the roster; 404 "Session not found" / 403 = try the next hub) — approve/deny post `{}`, reply posts `{text, localId}`. diff --git a/shared/src/apiTypes.ts b/shared/src/apiTypes.ts index 735d8a5ad9..0c0102ab45 100644 --- a/shared/src/apiTypes.ts +++ b/shared/src/apiTypes.ts @@ -52,7 +52,9 @@ export type CliMessagesResponse = z.infer export const CreateSessionResponseSchema = z.object({ session: SessionSchema, /** Hub opt-in for AGENT_NOTIFY_SUMMARY prompt injection (default off when omitted). */ - sessionSummaryContract: z.boolean().optional() + sessionSummaryContract: z.boolean().optional(), + /** Hub opt-in: Cursor auto-bridge after transient model errors (default off). */ + autoBridgeTransientModelErrors: z.boolean().optional() }) export type CreateSessionResponse = z.infer @@ -60,7 +62,8 @@ export type CreateSessionResponse = z.infer export const HubSettingsResponseSchema = z.object({ sessionSummaryContract: z.boolean(), /** Show compact AGENT_NOTIFY_SUMMARY in chat (default off / hide). */ - sessionSummaryInChat: z.boolean() + sessionSummaryInChat: z.boolean(), + autoBridgeTransientModelErrors: z.boolean() }) export type HubSettingsResponse = z.infer @@ -68,10 +71,13 @@ export type HubSettingsResponse = z.infer export const UpdateHubSettingsRequestSchema = z .object({ sessionSummaryContract: z.boolean().optional(), - sessionSummaryInChat: z.boolean().optional() + sessionSummaryInChat: z.boolean().optional(), + autoBridgeTransientModelErrors: z.boolean().optional() }) .refine( - (data) => data.sessionSummaryContract !== undefined || data.sessionSummaryInChat !== undefined, + (data) => data.sessionSummaryContract !== undefined + || data.sessionSummaryInChat !== undefined + || data.autoBridgeTransientModelErrors !== undefined, { message: 'At least one hub setting field is required' } ) @@ -431,6 +437,20 @@ export const ScratchlistEntryUpdateRequestSchema = z.object({ export type ScratchlistEntryUpdateRequest = z.infer +/** Dismiss the model-error banner for a specific displayed error (by eventId). */ +export const AcknowledgeModelErrorRequestSchema = z.object({ + eventId: z.string().min(1) +}) + +export type AcknowledgeModelErrorRequest = z.infer + +/** Bridge & retry for the specific displayed model error (by eventId). */ +export const BridgeModelErrorRequestSchema = z.object({ + eventId: z.string().min(1) +}) + +export type BridgeModelErrorRequest = z.infer + /** Per-session legacy stream-json → ACP migrator request. See tiann/hapi#824. */ export const CursorMigrateToAcpRequestSchema = z.object({ /** Skip removing the legacy ~/.cursor/chats source store.db even after verify passes. */ diff --git a/shared/src/rpcMethods.ts b/shared/src/rpcMethods.ts index 2619ca27fb..6fe1a19c89 100644 --- a/shared/src/rpcMethods.ts +++ b/shared/src/rpcMethods.ts @@ -47,6 +47,7 @@ export const RPC_METHODS = { SteerQueuedMessage: 'steer-queued-message', ForkConversation: 'fork-conversation', RewindConversation: 'rewind-conversation', + BridgeModelError: 'bridge-model-error' } as const export const RPC_TARGET_MISSING_ERROR_CODE = 'rpc_target_missing' as const diff --git a/shared/src/schemas.ts b/shared/src/schemas.ts index 167f95d6f4..3fb1827df4 100644 --- a/shared/src/schemas.ts +++ b/shared/src/schemas.ts @@ -159,7 +159,33 @@ export const MetadataSchema = z.object({ // field stores only modelId (shared across all flavors); this preserves // the provider so web can resolve the exact model when two providers // share a modelId. - piSelectedModel: z.object({ provider: z.string(), modelId: z.string() }).nullable().optional() + piSelectedModel: z.object({ provider: z.string(), modelId: z.string() }).nullable().optional(), + lastModelError: z.object({ + /** Stable identity for this error event (not wall-clock order). */ + eventId: z.string().min(1), + kind: z.string(), + transient: z.boolean(), + rawSnippet: z.string(), + /** Display / telemetry timestamp only — not used for notify/ack identity. */ + atTs: z.number(), + priorAssistantClaimsDone: z.boolean(), + lastUserMessage: z.string().optional(), + bridgedForEventId: z.string().optional(), + retriedAndFailed: z.boolean().optional(), + /** + * Set when a non-bridge user turn starts after this error was recorded. + * Blocks Bridge so a later retry cannot replay work the newer turn already handled. + */ + supersededByUserTurn: z.boolean().optional(), + /** + * Explicit false blocks Bridge (idle stderr after a successful turn). + * Omitted / true keeps the existing transient gate. + */ + bridgeable: z.boolean().optional(), + acknowledgedAt: z.number().optional(), + /** Hub-owned: successful push/FCM/Telegram delivery watermark. */ + notifiedAt: z.number().optional() + }).optional() }) export type Metadata = z.infer diff --git a/shared/src/sessionSummary.test.ts b/shared/src/sessionSummary.test.ts index 379b79ddcb..be1c990454 100644 --- a/shared/src/sessionSummary.test.ts +++ b/shared/src/sessionSummary.test.ts @@ -330,4 +330,22 @@ describe('summary derivation helpers', () => { }) expect(summary?.agentSessionId).toBe('cursor-xyz') }) + + it('toSessionSummaryMetadata omits lastModelError.lastUserMessage', () => { + const summary = toSessionSummaryMetadata({ + path: '/p', + host: 'h', + lastModelError: { + eventId: 'evt-1', + kind: 'transport_closed', + transient: true, + rawSnippet: 'WritableIterable is closed', + atTs: 1, + priorAssistantClaimsDone: false, + lastUserMessage: 'secret prompt text that must not leak into list payloads' + } + }) + expect(summary?.lastModelError?.eventId).toBe('evt-1') + expect(summary?.lastModelError).not.toHaveProperty('lastUserMessage') + }) }) diff --git a/shared/src/sessionSummary.ts b/shared/src/sessionSummary.ts index bf8bc48561..9b5aa220cd 100644 --- a/shared/src/sessionSummary.ts +++ b/shared/src/sessionSummary.ts @@ -43,6 +43,20 @@ export type SessionSummaryMetadata = { lifecycleState?: string /** Loopback MCP URL when session CLI happy server is running (#956). */ hapiMcpUrl?: string + lastModelError?: { + eventId: string + kind: string + transient: boolean + rawSnippet: string + atTs: number + priorAssistantClaimsDone: boolean + bridgedForEventId?: string + retriedAndFailed?: boolean + supersededByUserTurn?: boolean + bridgeable?: boolean + acknowledgedAt?: number + notifiedAt?: number + } } export type SessionSummary = { @@ -197,7 +211,18 @@ export function toSessionSummaryMetadata(metadata: Metadata | null | undefined): worktree: metadata.worktree, agentSessionId: getSummaryAgentSessionId(metadata), lifecycleState: metadata.lifecycleState, - hapiMcpUrl: metadata.hapiMcpUrl ?? undefined + hapiMcpUrl: metadata.hapiMcpUrl ?? undefined, + // Omit lastUserMessage — bridge recovery text stays in full session + // metadata only; list/SSE summaries must not ship up to 32 KB of prompt. + lastModelError: metadata.lastModelError + ? (() => { + const { + lastUserMessage: _omitLastUserMessage, + ...summaryError + } = metadata.lastModelError + return summaryError + })() + : undefined } } diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 6ab7a91989..81bbb32b8d 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -606,6 +606,66 @@ export class ApiClient { }) } + async acknowledgeModelError(sessionId: string, eventId: string): Promise { + await this.request(`/api/sessions/${encodeURIComponent(sessionId)}/model-error/acknowledge`, { + method: 'POST', + body: JSON.stringify({ eventId }) + }) + } + + async bridgeModelError(sessionId: string, eventId: string): Promise<{ ok: boolean; reason?: string }> { + const path = `/api/sessions/${encodeURIComponent(sessionId)}/model-error/bridge` + const tryOnce = async (overrideToken: string | null): Promise => { + const headers = new Headers({ 'content-type': 'application/json' }) + const liveToken = this.getToken ? this.getToken() : null + const authToken = overrideToken ?? liveToken ?? this.token + if (authToken) { + headers.set('authorization', `Bearer ${authToken}`) + } + return fetch(this.buildUrl(path), { + method: 'POST', + headers, + body: JSON.stringify({ eventId }) + }) + } + + let res = await tryOnce(null) + if (res.status === 401 && this.onUnauthorized) { + const refreshed = await this.onUnauthorized() + if (refreshed) { + this.token = refreshed + res = await tryOnce(refreshed) + } + } + if (res.status === 401) { + throw new Error('Session expired. Please sign in again.') + } + + const body = await res.json().catch(() => null) as { ok?: boolean; reason?: string; error?: string } | null + if (res.status === 409 && body && typeof body.ok === 'boolean') { + return { ok: body.ok, reason: body.reason } + } + + if (!res.ok) { + const detail = body?.error ?? (typeof body === 'object' ? JSON.stringify(body) : '') + throw new ApiError( + `HTTP ${res.status} ${res.statusText}: ${detail}`, + res.status, + undefined, + detail || undefined + ) + } + + return { ok: true } + } + + async setModelErrorAutoBridge(sessionId: string, enabled: boolean): Promise { + await this.request(`/api/sessions/${encodeURIComponent(sessionId)}/model-error/auto-bridge-setting`, { + method: 'POST', + body: JSON.stringify({ enabled }) + }) + } + async reopenSession(sessionId: string): Promise { return await this.request( `/api/sessions/${encodeURIComponent(sessionId)}/reopen`, diff --git a/web/src/chat/presentation.test.ts b/web/src/chat/presentation.test.ts index 12fcce0f1c..f0c432d692 100644 --- a/web/src/chat/presentation.test.ts +++ b/web/src/chat/presentation.test.ts @@ -107,6 +107,22 @@ describe('getEventPresentation — api-error', () => { }) }) +describe('getEventPresentation — modelError', () => { + it('renders a short kind label without rawSnippet JSON dump', () => { + const result = getEventPresentation({ + type: 'modelError', + kind: 'quota_exhausted', + transient: true, + rawSnippet: 'Error: RetriableError: [resource_exhausted] Error', + priorAssistantClaimsDone: false + }) + + expect(result.icon).toBe('⚠️') + expect(result.text).toBe('Model error (quota_exhausted, transient)') + expect(result.text).not.toContain('RetriableError') + }) +}) + describe('getEventPresentation — limit-warning', () => { it('formats five_hour warning', () => { const result = getEventPresentation({ diff --git a/web/src/chat/presentation.ts b/web/src/chat/presentation.ts index 60d847cf89..e07ae44557 100644 --- a/web/src/chat/presentation.ts +++ b/web/src/chat/presentation.ts @@ -276,6 +276,28 @@ export function getEventPresentation(event: AgentEvent): EventPresentation { if (event.type === 'token-count') { return formatTokenCountEvent(event) } + if (event.type === 'modelError') { + // Banner already surfaces the full model-error UI; avoid dumping + // rawSnippet via the JSON.stringify fallback into the chat thread. + const kind = typeof event.kind === 'string' ? event.kind : 'unknown' + const transient = event.transient === true + return { + icon: '⚠️', + text: transient + ? `Model error (${kind}, transient)` + : `Model error (${kind})` + } + } + if (event.type === 'modelErrorBridged') { + const kind = typeof event.kind === 'string' ? event.kind : 'unknown' + const auto = event.auto === true + return { + icon: '✓', + text: auto + ? `HAPI auto-bridged after ${kind} — re-sent last user message` + : `HAPI bridged after ${kind} — re-sent last user message` + } + } try { return { icon: null, text: JSON.stringify(event) } } catch { diff --git a/web/src/components/ModelErrorBanner.test.ts b/web/src/components/ModelErrorBanner.test.ts new file mode 100644 index 0000000000..81af99b270 --- /dev/null +++ b/web/src/components/ModelErrorBanner.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, it } from 'vitest' +import { + canShowModelErrorBridge, + getModelErrorUiState, + hasActiveModelError, + hasRecoveredModelError, + hasUrgentModelError, + isBridgeSettling, + shouldKeepPendingBridge, + visibleBridgeFailureReason, + type ModelErrorHolder +} from './ModelErrorBanner' +import { getEventPresentation } from '@/chat/presentation' + +function holder(partial: NonNullable): ModelErrorHolder { + return { lastModelError: partial } +} + +describe('model error UI states', () => { + const base = { + eventId: 'evt-1000', + kind: 'transport_closed', + transient: true, + rawSnippet: 'WritableIterable is closed', + atTs: 1000, + priorAssistantClaimsDone: false + } + + it('treats unacked unrecovered errors as urgent', () => { + const metadata = holder(base) + expect(getModelErrorUiState(metadata)).toBe('unrecovered') + expect(hasUrgentModelError(metadata)).toBe(true) + expect(hasRecoveredModelError(metadata)).toBe(false) + expect(canShowModelErrorBridge(metadata)).toBe(true) + }) + + it('treats bridged errors as recovered (not urgent)', () => { + const metadata = holder({ ...base, bridgedForEventId: 'evt-1000' }) + expect(getModelErrorUiState(metadata)).toBe('recovered') + expect(hasUrgentModelError(metadata)).toBe(false) + expect(hasRecoveredModelError(metadata)).toBe(true) + expect(hasActiveModelError(metadata)).toBe(true) + expect(canShowModelErrorBridge(metadata)).toBe(false) + }) + + it('treats retriedAndFailed as bridge_failed / urgent', () => { + const metadata = holder({ ...base, bridgedForEventId: 'evt-1000', retriedAndFailed: true }) + expect(getModelErrorUiState(metadata)).toBe('bridge_failed') + expect(hasUrgentModelError(metadata)).toBe(true) + expect(canShowModelErrorBridge(metadata)).toBe(false) + }) + + it('hides acknowledged errors', () => { + const metadata = holder({ ...base, acknowledgedAt: 2000 }) + expect(getModelErrorUiState(metadata)).toBeNull() + expect(hasActiveModelError(metadata)).toBe(false) + }) + + it('hides Bridge when a newer turn superseded the error', () => { + const metadata = holder({ ...base, supersededByUserTurn: true }) + expect(getModelErrorUiState(metadata)).toBe('unrecovered') + expect(canShowModelErrorBridge(metadata)).toBe(false) + }) + + it('hides Bridge when bridgeable is explicitly false', () => { + const metadata = holder({ ...base, bridgeable: false }) + expect(getModelErrorUiState(metadata)).toBe('unrecovered') + expect(canShowModelErrorBridge(metadata)).toBe(false) + }) + + it('keeps Bridge settling after enqueue until metadata records an outcome', () => { + const unrecovered = holder(base) + expect(isBridgeSettling(unrecovered, 'evt-1000')).toBe(true) + expect(isBridgeSettling(unrecovered, 'evt-other')).toBe(false) + expect(isBridgeSettling(holder({ ...base, bridgedForEventId: 'evt-1000' }), 'evt-1000')).toBe(false) + expect(isBridgeSettling(holder({ ...base, retriedAndFailed: true }), 'evt-1000')).toBe(false) + expect(isBridgeSettling(holder({ ...base, supersededByUserTurn: true }), 'evt-1000')).toBe(false) + expect(isBridgeSettling(holder({ ...base, acknowledgedAt: 2000 }), 'evt-1000')).toBe(false) + expect(isBridgeSettling(unrecovered, null)).toBe(false) + }) + + it('scopes a Bridge failure reason to the requested eventId', () => { + const failure = { eventId: 'evt-1000', reason: 'not_bridgeable' } + expect(visibleBridgeFailureReason(failure, 'evt-1000')).toBe('not_bridgeable') + expect(visibleBridgeFailureReason(failure, 'evt-2000')).toBeNull() + expect(visibleBridgeFailureReason(null, 'evt-1000')).toBeNull() + }) + + it('drops local Bridge pending across a session disconnect', () => { + expect(shouldKeepPendingBridge(true, true)).toBe(true) + expect(shouldKeepPendingBridge(false, true)).toBe(false) + expect(shouldKeepPendingBridge(true, false)).toBe(false) + }) + + it('drops local Bridge pending after Abort even while still active and unrecovered', () => { + expect(shouldKeepPendingBridge(true, true, true)).toBe(false) + }) +}) + +describe('model error chat event labels', () => { + it('renders modelError and modelErrorBridged labels', () => { + expect(getEventPresentation({ + type: 'modelError', + kind: 'transport_closed', + transient: true + } as never).text).toContain('transport_closed') + + expect(getEventPresentation({ + type: 'modelErrorBridged', + kind: 'transport_closed', + auto: true, + eventId: 'evt-1' + } as never).text).toContain('auto-bridged') + + expect(getEventPresentation({ + type: 'modelErrorBridged', + kind: 'transport_closed', + auto: false, + eventId: 'evt-1' + } as never).text).toMatch(/HAPI bridged after/) + }) +}) diff --git a/web/src/components/ModelErrorBanner.tsx b/web/src/components/ModelErrorBanner.tsx new file mode 100644 index 0000000000..cb9b88a564 --- /dev/null +++ b/web/src/components/ModelErrorBanner.tsx @@ -0,0 +1,220 @@ +import { useState } from 'react' +import { useTranslation } from '@/lib/use-translation' + +// Minimal shape that both Metadata and SessionSummaryMetadata satisfy +export type ModelErrorHolder = { + lastModelError?: { + eventId: string + kind: string + transient: boolean + rawSnippet: string + atTs: number + priorAssistantClaimsDone: boolean + bridgedForEventId?: string + retriedAndFailed?: boolean + supersededByUserTurn?: boolean + bridgeable?: boolean + acknowledgedAt?: number + } + [key: string]: unknown +} + +export type ModelErrorUiState = 'unrecovered' | 'recovered' | 'bridge_failed' + +export function getModelErrorUiState(metadata: ModelErrorHolder | null | undefined): ModelErrorUiState | null { + const err = metadata?.lastModelError + if (!err || err.acknowledgedAt) { + return null + } + if (err.retriedAndFailed) { + return 'bridge_failed' + } + if (err.bridgedForEventId === err.eventId) { + return 'recovered' + } + return 'unrecovered' +} + +export function canShowModelErrorBridge(metadata: ModelErrorHolder | null | undefined): boolean { + return getModelErrorUiState(metadata) === 'unrecovered' + && Boolean(metadata?.lastModelError?.transient) + && !metadata?.lastModelError?.supersededByUserTurn + && metadata?.lastModelError?.bridgeable !== false +} + +/** + * True after the CLI accepted a Bridge enqueue for `pendingEventId` and before + * metadata records recovered / failed / superseded / acknowledged / a new error. + */ +export function isBridgeSettling( + metadata: ModelErrorHolder | null | undefined, + pendingEventId: string | null +): boolean { + const err = metadata?.lastModelError + if (!pendingEventId || !err || err.eventId !== pendingEventId) { + return false + } + if (err.acknowledgedAt || err.retriedAndFailed || err.supersededByUserTurn) { + return false + } + if (err.bridgedForEventId === pendingEventId) { + return false + } + return true +} + +export function shouldKeepPendingBridge( + sessionActive: boolean, + settling: boolean, + aborted = false +): boolean { + return sessionActive && settling && !aborted +} + +export function visibleBridgeFailureReason( + failure: { eventId: string; reason: string } | null, + currentEventId: string | undefined +): string | null { + if (!failure || !currentEventId || failure.eventId !== currentEventId) { + return null + } + return failure.reason +} + +/** Any unacknowledged model-error surface (error, recovered, or bridge failed). */ +export function hasActiveModelError(metadata: ModelErrorHolder | null | undefined): boolean { + return getModelErrorUiState(metadata) !== null +} + +/** Amber pulse: unrecovered or bridge-failed. Not recovered. */ +export function hasUrgentModelError(metadata: ModelErrorHolder | null | undefined): boolean { + const state = getModelErrorUiState(metadata) + return state === 'unrecovered' || state === 'bridge_failed' +} + +export function hasRecoveredModelError(metadata: ModelErrorHolder | null | undefined): boolean { + return getModelErrorUiState(metadata) === 'recovered' +} + +export function ModelErrorBanner({ + metadata, + onDismiss, + onBridge, + isBridging = false, + bridgeErrorReason = null +}: { + metadata: ModelErrorHolder | null | undefined + onDismiss: () => void + onBridge?: () => void + isBridging?: boolean + bridgeErrorReason?: string | null +}) { + const { t } = useTranslation() + const [showRaw, setShowRaw] = useState(false) + + const err = metadata?.lastModelError + const uiState = getModelErrorUiState(metadata) + if (!err || !uiState) { + return null + } + + const transientLabel = err.transient + ? t('session.modelError.banner.subtitle.transient') + : t('session.modelError.banner.subtitle.nonTransient') + + const isRecovered = uiState === 'recovered' + const isBridgeFailed = uiState === 'bridge_failed' + + const title = isRecovered + ? t('session.modelError.banner.recoveredTitle', { kind: err.kind }) + : isBridgeFailed + ? t('session.modelError.banner.bridgeFailedTitle', { kind: err.kind }) + : t('session.modelError.banner.title', { kind: err.kind }) + + const bodyText = isRecovered + ? t('session.modelError.banner.recoveredBody') + : isBridgeFailed + ? t('session.modelError.banner.bridgeFailedBody') + : err.priorAssistantClaimsDone + ? t('session.modelError.banner.claimedDone') + : t('session.modelError.banner.midExecution') + + const showBridge = canShowModelErrorBridge(metadata) && onBridge + + const shellClass = isRecovered + ? 'border-emerald-500/40 bg-emerald-500/10' + : 'border-amber-500/40 bg-amber-500/10' + + const titleClass = isRecovered + ? 'text-emerald-700 dark:text-emerald-400' + : 'text-amber-600 dark:text-amber-400' + + const icon = isRecovered ? '\u2713' : '\u26A0' + + return ( +
+
+
+ +
+
+ {title}{' '} + {!isRecovered ? ( + + ({transientLabel}) + + ) : null} +
+
+ {bodyText} +
+ {showRaw && ( + 
+                                {err.rawSnippet}
+
+ )} +
+
+
+ {showBridge ? ( + + ) : null} + {bridgeErrorReason ? ( + + {t('session.modelError.banner.bridgeFailed')} + + ) : null} + + +
+
+
+ ) +} diff --git a/web/src/components/SessionChat.tsx b/web/src/components/SessionChat.tsx index d57f18e7ae..916e1904df 100644 --- a/web/src/components/SessionChat.tsx +++ b/web/src/components/SessionChat.tsx @@ -80,6 +80,7 @@ import type { SendMessageAcceptance, SendMessageSettlement } from '@/hooks/mutat import { handoffComposerDraft, transferComposerDraftThenNavigate } from '@/lib/composer-draft-transfer' import { SessionHeader } from '@/components/SessionHeader' import { CursorMigrationBanner } from '@/components/CursorMigrationBanner' +import { ModelErrorBanner, hasActiveModelError, isBridgeSettling, shouldKeepPendingBridge, visibleBridgeFailureReason } from '@/components/ModelErrorBanner' import { TeamPanel } from '@/components/TeamPanel' import { SessionStatusPanel } from '@/components/SessionStatusPanel' import { buildSessionStatusData } from '@/chat/sessionStatus' @@ -1115,6 +1116,63 @@ function SessionChatInner(props: SessionChatProps) { codexCollaborationModeSupported ) + const handleAcknowledgeModelError = useCallback(async () => { + const eventId = props.session.metadata?.lastModelError?.eventId + if (typeof eventId !== 'string' || eventId.length === 0) { + props.onRefresh() + return + } + await props.api.acknowledgeModelError(props.session.id, eventId).catch(() => {}) + props.onRefresh() + }, [props.api, props.session.id, props.session.metadata?.lastModelError?.eventId, props.onRefresh]) + + const [isBridgingModelError, setIsBridgingModelError] = useState(false) + const [pendingBridgeEventId, setPendingBridgeEventId] = useState(null) + const [bridgeFailure, setBridgeFailure] = useState<{ eventId: string; reason: string } | null>(null) + const currentModelError = props.session.metadata?.lastModelError + const bridgePending = isBridgeSettling(props.session.metadata, pendingBridgeEventId) + + useEffect(() => { + if (pendingBridgeEventId && !shouldKeepPendingBridge(props.session.active, bridgePending)) { + setPendingBridgeEventId(null) + } + }, [props.session.active, pendingBridgeEventId, bridgePending]) + + const handleBridgeModelError = useCallback(async () => { + if (isBridgingModelError || bridgePending) { + return + } + const eventId = currentModelError?.eventId + if (typeof eventId !== 'string' || eventId.length === 0) { + props.onRefresh() + return + } + setIsBridgingModelError(true) + setBridgeFailure(null) + try { + const result = await props.api.bridgeModelError(props.session.id, eventId) + if (result.ok) { + setPendingBridgeEventId(eventId) + } else { + setBridgeFailure({ eventId, reason: result.reason ?? 'not_bridgeable' }) + } + props.onRefresh() + } catch (error) { + const message = error instanceof Error ? error.message : 'bridge_failed' + setBridgeFailure({ eventId, reason: message }) + console.warn('[SessionChat] model error bridge failed:', error) + } finally { + setIsBridgingModelError(false) + } + }, [ + isBridgingModelError, + bridgePending, + currentModelError?.eventId, + props.api, + props.session.id, + props.onRefresh + ]) + // Voice assistant integration const voice = useVoiceOptional() const [voiceBackendReady, setVoiceBackendReady] = useState(false) @@ -1514,6 +1572,10 @@ function SessionChatInner(props: SessionChatProps) { // Abort handler const handleAbort = useCallback(async () => { await abortSession() + // Abort cancels pending/in-flight Bridge without recovered/failed/superseded + // metadata, and the session stays active — drop the local latch so Bridge + // is not stuck as "Bridging…" until remount. + setPendingBridgeEventId(null) props.onRefresh() }, [abortSession, props.onRefresh]) @@ -1749,6 +1811,16 @@ function SessionChatInner(props: SessionChatProps) { {sessionStatus ? : null} + +
{props.session.teamState && ( diff --git a/web/src/components/SessionRowSummary.test.tsx b/web/src/components/SessionRowSummary.test.tsx index 8e6cbeca27..af0a74ca74 100644 --- a/web/src/components/SessionRowSummary.test.tsx +++ b/web/src/components/SessionRowSummary.test.tsx @@ -1,11 +1,16 @@ import { cleanup, render, screen } from '@testing-library/react' import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import type { ReactNode } from 'react' import type { SessionSummary } from '@/types/api' import { I18nProvider } from '@/lib/i18n-context' import { SessionRowSummary } from './SessionRowSummary' afterEach(() => cleanup()) +function renderWithI18n(children: ReactNode) { + return render({children}) +} + function makeSummary(overrides: Partial = {}): SessionSummary { return { id: 'background-demo', @@ -155,3 +160,99 @@ describe('SessionRowSummary background status', () => { expect(screen.getByRole('tooltip', { hidden: true })).toHaveTextContent('New activity') }) }) + +describe('SessionRowSummary model-error + attention', () => { + it('shows model-error and permission attention together', () => { + const summary = makeSummary({ + id: 's-both', + backgroundTaskCount: 0, + pendingRequestsCount: 1, + pendingRequestKinds: ['permission'], + pendingRequests: [{ id: 'r1', kind: 'permission', tool: 'Bash', since: 0 }], + metadata: { + path: '/tmp/proj', + lastModelError: { + eventId: 'evt-row-1', + kind: 'model_not_found', + transient: false, + rawSnippet: 'Unknown model', + atTs: 1, + priorAssistantClaimsDone: false, + }, + }, + }) + + renderWithI18n( + + ) + + expect(screen.getByLabelText(/Model error/i)).toBeTruthy() + expect(screen.getByLabelText('Permission required')).toBeTruthy() + }) + + it('keeps the model-error pulse while thinking (auto-bridge in flight)', () => { + const summary = makeSummary({ + id: 's-bridge', + thinking: true, + backgroundTaskCount: 0, + metadata: { + path: '/tmp/proj', + lastModelError: { + eventId: 'evt-row-bridge', + kind: 'rate_limited', + transient: true, + rawSnippet: 'rate limited', + atTs: 1, + priorAssistantClaimsDone: false, + }, + }, + }) + + renderWithI18n( + + ) + + expect(screen.getByLabelText(/Model error/i)).toBeTruthy() + }) + + it('hides the model-error pulse after a successful bridge even if still thinking', () => { + const summary = makeSummary({ + id: 's-recovered', + thinking: true, + backgroundTaskCount: 0, + metadata: { + path: '/tmp/proj', + lastModelError: { + eventId: 'evt-row-ok', + kind: 'rate_limited', + transient: true, + rawSnippet: 'rate limited', + atTs: 1, + priorAssistantClaimsDone: false, + bridgedForEventId: 'evt-row-ok', + }, + }, + }) + + renderWithI18n( + + ) + + expect(screen.queryByLabelText(/Model error/i)).toBeNull() + }) +}) diff --git a/web/src/components/SessionRowSummary.tsx b/web/src/components/SessionRowSummary.tsx index 0602f7ae45..0b76c0f68f 100644 --- a/web/src/components/SessionRowSummary.tsx +++ b/web/src/components/SessionRowSummary.tsx @@ -12,6 +12,7 @@ import { getCodexImportedAt } from '@/lib/codexImportedSessions' import { getSessionTitle } from '@/lib/sessionTitle' import { useTranslation } from '@/lib/use-translation' import { getWorktreeSessionLabel } from '@/lib/sessionWorktreeLabel' +import { hasUrgentModelError } from '@/components/ModelErrorBanner' function LoaderIcon(props: { className?: string }) { return ( @@ -151,6 +152,8 @@ export function SessionRowSummary(props: { const attentionLabel = attention ? getAttentionLabel(attention, t) : null const urgentAttention = attention !== null && (attention.kind === 'permission' || attention.kind === 'input') + const modelErrorActive = hasUrgentModelError(s.metadata) + const modelErrorLabel = t('session.modelError.listIndicator') const scheduledLabel = s.futureScheduledMessageCount > 1 ? t('session.item.scheduledMessages', { count: s.futureScheduledMessageCount }) : t('session.item.scheduledMessage') @@ -174,6 +177,16 @@ export function SessionRowSummary(props: { > {sessionName}
+ {modelErrorActive ? ( + + + + + ) : null} {attention?.kind === 'unread' && nestedTooltips && attentionId ? ( = {}): HappyChatC sessionId: 'session-1', metadata: { path: '/home/ada/coding/hapi', host: 'local' }, terminalToolDisplayMode: 'compact', + showSessionSummaryInChat: false, disabled: false, onRefresh: () => {}, hasMoreMessages: false, isSyncingTail: false, isLoadingMoreMessages: false, - showSessionSummaryInChat: false, loadOlderMessagesPreservingScroll: async () => 'loaded', ...overrides, } diff --git a/web/src/hooks/useAutoBridgeTransientModelErrors.ts b/web/src/hooks/useAutoBridgeTransientModelErrors.ts new file mode 100644 index 0000000000..efbbca9d51 --- /dev/null +++ b/web/src/hooks/useAutoBridgeTransientModelErrors.ts @@ -0,0 +1,29 @@ +import { useCallback, useEffect, useState } from 'react' +import { + readAutoBridgeTransientModelErrors, + writeAutoBridgeTransientModelErrors +} from '@/lib/modelErrorBridgePrefs' + +export function useAutoBridgeTransientModelErrors(): { + enabled: boolean + setEnabled: (enabled: boolean) => void +} { + const [enabled, setEnabledState] = useState(() => readAutoBridgeTransientModelErrors()) + + useEffect(() => { + const onStorage = (event: StorageEvent) => { + if (event.key === 'hapi-auto-bridge-transient-model-errors') { + setEnabledState(readAutoBridgeTransientModelErrors()) + } + } + window.addEventListener('storage', onStorage) + return () => window.removeEventListener('storage', onStorage) + }, []) + + const setEnabled = useCallback((next: boolean) => { + writeAutoBridgeTransientModelErrors(next) + setEnabledState(next) + }, []) + + return { enabled, setEnabled } +} diff --git a/web/src/lib/locales/en.ts b/web/src/lib/locales/en.ts index e1e51a96fc..2d243f9c73 100644 --- a/web/src/lib/locales/en.ts +++ b/web/src/lib/locales/en.ts @@ -210,6 +210,23 @@ export default { 'session.cursorMigration.bannerAmbiguous.title': 'Cursor session upgrade needs manual review', 'session.cursorMigration.bannerAmbiguous.body': 'This chat exists on disk in multiple workspace folders or the on-disk size does not match the synced history, so HAPI refused to transplant it automatically. Check hub logs for the candidate list (search for "[migrator] ambiguous legacy store" or "[migrator] size sanity check refused"), delete the stale drawers under ~/.cursor/chats/, and reopen the session.', + // Model error banner + 'session.modelError.banner.title': 'MODEL ERROR \u2014 {kind}', + 'session.modelError.banner.subtitle.transient': 'transient', + 'session.modelError.banner.subtitle.nonTransient': 'non-transient', + 'session.modelError.banner.claimedDone': 'The agent claimed completion before this error. The work is likely INCOMPLETE.', + 'session.modelError.banner.midExecution': 'The last agent turn failed mid-execution.', + 'session.modelError.banner.recoveredTitle': 'RECOVERED \u2014 continued after {kind}', + 'session.modelError.banner.recoveredBody': 'HAPI bridged the failure and re-sent your last message. Dismiss when you no longer need this notice.', + 'session.modelError.banner.bridgeFailedTitle': 'BRIDGE FAILED \u2014 {kind}', + 'session.modelError.banner.bridgeFailedBody': 'The automatic or manual bridge retry failed. Check the session or dismiss this notice.', + 'session.modelError.banner.bridgeRetry': 'Bridge & retry', + 'session.modelError.banner.bridging': 'Bridging\u2026', + 'session.modelError.banner.bridgeFailed': 'Bridge failed — try again or dismiss.', + 'session.modelError.banner.dismiss': 'Dismiss', + 'session.modelError.banner.viewRaw': 'View raw error', + 'session.modelError.listIndicator': 'Model error \u2014 click to view', + // Session inactive 'session.inactive.autoResume': 'This session is inactive. Send a message to resume.', 'session.inactive.cannotResume': 'This session is inactive and cannot be resumed.', @@ -868,6 +885,10 @@ export default { 'settings.chat.input': 'Input', 'settings.chat.tools': 'Tool cards', 'settings.chat.colors': 'Conversation colors', + 'settings.chat.modelErrors': 'Model errors', + 'settings.chat.autoBridgeTransientModelErrors': 'Automatically bridge transient model errors', + 'settings.chat.autoBridgeTransientModelErrors.description': 'When enabled, Cursor sessions in the default namespace re-send your last message once after a recoverable model error. Owner hub setting (default off); other namespaces stay off.', + 'settings.chat.autoBridgeTransientModelErrors.syncFailed': 'Could not update every active Cursor session. Setting was left unchanged.', 'settings.chat.enterBehavior': 'Enter Key', 'settings.chat.enterBehavior.send': 'Send message', 'settings.chat.enterBehavior.newline': 'Insert newline', diff --git a/web/src/lib/locales/zh-CN.ts b/web/src/lib/locales/zh-CN.ts index 94ffbad9dc..17e6168f21 100644 --- a/web/src/lib/locales/zh-CN.ts +++ b/web/src/lib/locales/zh-CN.ts @@ -210,6 +210,23 @@ export default { 'session.cursorMigration.bannerAmbiguous.title': 'Cursor 会话升级需要人工处理', 'session.cursorMigration.bannerAmbiguous.body': '此会话在磁盘上的多个工作区下存在，或本地存储大小与已同步的历史记录不匹配，因此 HAPI 拒绝自动迁移。请在 hub 日志中搜索 "[migrator] ambiguous legacy store" 或 "[migrator] size sanity check refused" 获取候选列表，删除 ~/.cursor/chats/ 下的陈旧目录后，再重新打开此会话。', + // Model error banner + 'session.modelError.banner.title': '模型错误 — {kind}', + 'session.modelError.banner.subtitle.transient': '暂时性', + 'session.modelError.banner.subtitle.nonTransient': '非暂时性', + 'session.modelError.banner.claimedDone': '代理在此错误之前声称任务已完成。工作可能未完成。', + 'session.modelError.banner.midExecution': '上一轮代理在执行过程中失败。', + 'session.modelError.banner.recoveredTitle': '已恢复 — 已在 {kind} 后继续', + 'session.modelError.banner.recoveredBody': 'HAPI 已桥接该失败并重新发送你的上一条消息。不再需要此提示时请忽略。', + 'session.modelError.banner.bridgeFailedTitle': '桥接失败 — {kind}', + 'session.modelError.banner.bridgeFailedBody': '自动或手动桥接重试失败。请检查会话，或忽略此提示。', + 'session.modelError.banner.bridgeRetry': '桥接并重试', + 'session.modelError.banner.bridging': '桥接中…', + 'session.modelError.banner.bridgeFailed': '桥接失败 — 请重试或忽略。', + 'session.modelError.banner.dismiss': '忽略', + 'session.modelError.banner.viewRaw': '查看原始错误', + 'session.modelError.listIndicator': '模型错误 — 点击查看', + // Session inactive 'session.inactive.autoResume': '此会话已停止。发送消息即可恢复。', 'session.inactive.cannotResume': '此会话已停止，无法恢复。', @@ -867,6 +884,10 @@ export default { 'settings.chat.input': '输入', 'settings.chat.tools': '工具卡片', 'settings.chat.colors': '对话颜色', + 'settings.chat.modelErrors': '模型错误', + 'settings.chat.autoBridgeTransientModelErrors': '自动桥接暂时性模型错误', + 'settings.chat.autoBridgeTransientModelErrors.description': '启用后，默认命名空间下的 Cursor 会话在可恢复的模型错误后会自动重发上一条用户消息一次。属 owner 的 hub 设置（默认关闭）；其他命名空间保持关闭。', + 'settings.chat.autoBridgeTransientModelErrors.syncFailed': '未能同步到所有活跃的 Cursor 会话，设置未更改。', 'settings.chat.enterBehavior': '回车键行为', 'settings.chat.enterBehavior.send': '发送消息', 'settings.chat.enterBehavior.newline': '插入换行', diff --git a/web/src/lib/modelErrorBridgePrefs.ts b/web/src/lib/modelErrorBridgePrefs.ts new file mode 100644 index 0000000000..b2242f4549 --- /dev/null +++ b/web/src/lib/modelErrorBridgePrefs.ts @@ -0,0 +1,21 @@ +const STORAGE_KEY = 'hapi-auto-bridge-transient-model-errors'; + +export function readAutoBridgeTransientModelErrors(): boolean { + try { + return localStorage.getItem(STORAGE_KEY) === 'true'; + } catch { + return false; + } +} + +export function writeAutoBridgeTransientModelErrors(enabled: boolean): void { + try { + if (enabled) { + localStorage.setItem(STORAGE_KEY, 'true'); + } else { + localStorage.removeItem(STORAGE_KEY); + } + } catch { + // ignore quota / privacy mode failures + } +} diff --git a/web/src/routes/settings/chat.tsx b/web/src/routes/settings/chat.tsx index ec61765510..f6ed02ab31 100644 --- a/web/src/routes/settings/chat.tsx +++ b/web/src/routes/settings/chat.tsx @@ -1,4 +1,6 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import { useTranslation } from '@/lib/use-translation' +import { useAppContext } from '@/lib/app-context' import { getComposerEnterBehaviorOptions, useComposerEnterBehavior } from '@/hooks/useComposerEnterBehavior' import { getTerminalToolDisplayModeOptions, useTerminalToolDisplayMode } from '@/hooks/useTerminalToolDisplayMode' import { useCodexExplorationCollapse } from '@/hooks/useCodexExplorationCollapse' @@ -13,7 +15,10 @@ import { type ChatSurfaceColorPreset, } from '@/hooks/useChatSurfaceColors' import { SettingsChoiceGroup, SettingsFieldLabel, SettingsPageContent, SettingsSection, SettingsSwitch } from '@/components/settings/SettingsPrimitives' +import { getNamespaceFromToken } from '@/components/settings/SettingsNav' import { ComposerToolbarLayoutControl } from '@/components/settings/ComposerToolbarLayoutControl' +import { queryKeys } from '@/lib/query-keys' +import { writeAutoBridgeTransientModelErrors } from '@/lib/modelErrorBridgePrefs' function ChatSurfaceColorControl(props: { label: string @@ -48,11 +53,43 @@ function ChatSurfaceColorControl(props: { export default function SettingsChatPage() { const { t } = useTranslation() + const { api, token } = useAppContext() + const isOwner = Boolean(token) && getNamespaceFromToken(token) === 'default' + const queryClient = useQueryClient() const { composerEnterBehavior, setComposerEnterBehavior } = useComposerEnterBehavior() const { terminalToolDisplayMode, setTerminalToolDisplayMode } = useTerminalToolDisplayMode() const { codexExplorationCollapsed, setCodexExplorationCollapsed } = useCodexExplorationCollapse() const { reasoningCollapsed, setReasoningCollapsed } = useReasoningCollapse() const { toolGroupBackground, userMessageBackground, setToolGroupBackground, setUserMessageBackground } = useChatSurfaceColors() + + const hubSettingsQuery = useQuery({ + queryKey: queryKeys.hubSettings, + queryFn: async () => { + if (!api) throw new Error('API unavailable') + return await api.getHubSettings() + }, + enabled: Boolean(api) && isOwner, + staleTime: 30_000, + retry: false, + }) + + const autoBridgeMutation = useMutation({ + mutationFn: async (enabled: boolean) => { + if (!api) throw new Error('API unavailable') + // Hub persists + fans out to active Cursor CLIs under one lock + // (and rolls back on fanout failure). Web only mirrors the result. + const next = await api.updateHubSettings({ autoBridgeTransientModelErrors: enabled }) + writeAutoBridgeTransientModelErrors(enabled) + return next + }, + onSuccess: (data) => { + queryClient.setQueryData(queryKeys.hubSettings, data) + }, + onError: async () => { + await queryClient.invalidateQueries({ queryKey: queryKeys.hubSettings }) + }, + }) + return ( @@ -88,6 +125,28 @@ export default function SettingsChatPage() { setToolGroupBackground(toPresetChatSurfaceColorPreference(preset))} onCustomChange={(value) => setToolGroupBackground(toCustomChatSurfaceColorPreference(value))} /> setUserMessageBackground(toPresetChatSurfaceColorPreference(preset))} onCustomChange={(value) => setUserMessageBackground(toCustomChatSurfaceColorPreference(value))} /> + {isOwner ? ( + + {hubSettingsQuery.data ? ( + <> + { + if (autoBridgeMutation.isPending) return + autoBridgeMutation.mutate(next) + }} + /> + {autoBridgeMutation.isError ? ( +

+ {t('settings.chat.autoBridgeTransientModelErrors.syncFailed')} +

+ ) : null} + + ) : null} + + ) : null} ) } diff --git a/web/src/routes/settings/index.test.tsx b/web/src/routes/settings/index.test.tsx index 881786f4e3..1ca9927a89 100644 --- a/web/src/routes/settings/index.test.tsx +++ b/web/src/routes/settings/index.test.tsx @@ -23,8 +23,16 @@ const { context, navigate, setAppearance, setColorTheme, setFontScale, setTermin setVoice: vi.fn(), })) -const getHubSettings = vi.fn().mockResolvedValue({ sessionSummaryContract: false, sessionSummaryInChat: false }) -const updateHubSettings = vi.fn().mockResolvedValue({ sessionSummaryContract: true, sessionSummaryInChat: false }) +const getHubSettings = vi.fn().mockResolvedValue({ + sessionSummaryContract: false, + sessionSummaryInChat: false, + autoBridgeTransientModelErrors: false +}) +const updateHubSettings = vi.fn().mockResolvedValue({ + sessionSummaryContract: true, + sessionSummaryInChat: false, + autoBridgeTransientModelErrors: false +}) vi.mock('@/hooks/useColorTheme', () => ({ useColorTheme: () => ({ colorTheme: 'default', setColorTheme }), @@ -160,7 +168,7 @@ vi.mock('@/hooks/useChatSurfaceColors', () => ({ vi.mock('@/lib/app-context', () => ({ useAppContext: () => ({ - api: { getHubSettings, updateHubSettings }, + api: { getHubSettings, updateHubSettings, getSessions: vi.fn().mockResolvedValue({ sessions: [] }) }, baseUrl: 'http://127.0.0.1:3006', token: context.token, }), @@ -216,8 +224,16 @@ describe('responsive settings pages', () => { beforeEach(() => { vi.clearAllMocks() localStorage.clear() - getHubSettings.mockResolvedValue({ sessionSummaryContract: false, sessionSummaryInChat: false }) - updateHubSettings.mockResolvedValue({ sessionSummaryContract: true, sessionSummaryInChat: false }) + getHubSettings.mockResolvedValue({ + sessionSummaryContract: false, + sessionSummaryInChat: false, + autoBridgeTransientModelErrors: false + }) + updateHubSettings.mockResolvedValue({ + sessionSummaryContract: true, + sessionSummaryInChat: false, + autoBridgeTransientModelErrors: false + }) context.token = `x.${btoa(JSON.stringify({ ns: 'default' }))}.x` }) @@ -278,11 +294,19 @@ describe('responsive settings pages', () => { expect(description.compareDocumentPosition(choices) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy() }) - it('keeps chat enum choices inline', () => { + it('keeps chat enum choices inline', async () => { renderPage() fireEvent.click(screen.getByRole('radio', { name: 'Insert newline' })) expect(setComposerEnterBehavior).toHaveBeenCalledWith('newline') expect(screen.getByText('Grouped Tool Use Background')).toBeInTheDocument() + expect(await screen.findByRole('checkbox', { name: 'Automatically bridge transient model errors' })).toBeInTheDocument() + }) + + it('hides the auto-bridge setting from tenant namespaces', () => { + context.token = `x.${btoa(JSON.stringify({ ns: 'tenant' }))}.x` + renderPage() + expect(screen.queryByRole('checkbox', { name: 'Automatically bridge transient model errors' })).not.toBeInTheDocument() + expect(getHubSettings).not.toHaveBeenCalled() }) it('renders the default-collapse switch for Codex exploration groups', () => {