diff --git a/AgentSourceCore/src/codex-source-turn.test.ts b/AgentSourceCore/src/codex-source-turn.test.ts new file mode 100644 index 000000000..70b9c28c8 --- /dev/null +++ b/AgentSourceCore/src/codex-source-turn.test.ts @@ -0,0 +1,134 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { readCodexRollout, readCodexSourceTurn, sourceTurnFailureReason, sourceTurnFromMessages, buildSourceTurnRequest, orderedTurns, type ConversationMessage } from "./index.js"; + +const dirs: string[] = []; +afterEach(() => { for (const path of dirs.splice(0)) rmSync(path, { recursive: true, force: true }); }); +const time = "2026-09-09T10:00:00.000Z"; +const event = (type: string, payload: Record) => ({ type, timestamp: time, payload }); +const message = (role: string, text: string) => event("response_item", { type: "message", role, content: [{ text }] }); +function fixture(records: unknown[]) { + const dir = mkdtempSync(join(tmpdir(), "source-turn-")); dirs.push(dir); + const path = join(dir, "rollout-file.jsonl"); writeFileSync(path, records.map(r => JSON.stringify(r)).join("\n") + "\n"); return path; +} +async function collect(values: AsyncIterable): Promise { const all: T[] = []; for await (const value of values) all.push(value); return all; } +async function read(records: unknown[]) { return collect(readCodexRollout(fixture(records))); } +const prefix = [event("session_meta", { id: "file-artifact", session_id: "conversation", cwd: "/project" }), event("event_msg", { type: "task_started", turn_id: "turn-1" }), message("user", "Fix the issue")]; +const end = [message("assistant", "Done"), event("event_msg", { type: "task_complete", turn_id: "turn-1" })]; + +describe("Codex native source turns", () => { + it("retains steering in one native turn and pairs missing/reversed results only by call id", async () => { + const messages = await read([...prefix, + event("response_item", { type: "function_call", call_id: "a", name: "read", arguments: { path: "a" } }), + message("user", "Also run tests"), + event("response_item", { type: "custom_tool_call", call_id: "b", name: "test", input: "npm test" }), + event("response_item", { type: "custom_tool_call_output", call_id: "b", output: "FAILED", status: "failed" }), + event("response_item", { type: "web_search_call", id: "web", action: { query: "fix" }, status: "completed" }), ...end]); + const turn = sourceTurnFromMessages(messages); + expect(turn).toMatchObject({ conversationId: "conversation", turnId: "turn-1", query: "Fix the issue\n\nAlso run tests", answer: "Done", completionEvidence: "task_complete:turn-1" }); + expect(turn?.toolCalls).toEqual([ + expect.objectContaining({ id: "a", name: "read", input: { path: "a" } }), + expect.objectContaining({ id: "b", name: "test", input: "npm test", output: "FAILED", status: "failed", success: false }), + expect.objectContaining({ id: "web", name: "web_search", input: { query: "fix" }, status: "completed" }) + ]); + expect(turn?.toolCalls[0]).not.toHaveProperty("output"); + const staged = messages.map(m => ({ ...m, sourceId: "codex", workspacePath: null, gitRoot: null })) as ConversationMessage[]; + expect(await collect(orderedTurns((async function* () { yield* staged; })()))).toHaveLength(1); + expect(buildSourceTurnRequest(turn!, "hook").sourceTurn).toEqual(buildSourceTurnRequest(turn!, "agent_source_scan").sourceTurn); + }); + + it("does not create a canonical turn without native identity, completion, or matching completion", async () => { + for (const records of [[message("user", "Q"), message("assistant", "A")], [...prefix, message("assistant", "still running")], [...prefix, ...end.slice(0, 1), event("event_msg", { type: "task_complete", turn_id: "other" })]]) { + const messages = await read(records); + expect(sourceTurnFromMessages(messages)).toBeNull(); + expect(messages.at(-1)?.rawMeta.sourceTurnState).not.toBe("complete"); + } + }); + + it("never matches a result to another turn and keeps unresolved turns for scan retry", async () => { + const messages = await read([...prefix, event("response_item", { type: "function_call", call_id: "same", name: "old", arguments: "x" }), ...end, + event("event_msg", { type: "task_started", turn_id: "turn-2" }), message("user", "Second"), event("response_item", { type: "function_call_output", call_id: "same", output: "not for old" })]); + const first = sourceTurnFromMessages(messages.filter(m => m.rawMeta.sourceTurnId === "turn-1")); + expect(first?.toolCalls[0]).not.toHaveProperty("output"); + const staged = messages.map(m => ({ ...m, sourceId: "codex", workspacePath: null, gitRoot: null })) as ConversationMessage[]; + const turns = await collect(orderedTurns((async function* () { yield* staged; })())); + expect(turns).toHaveLength(2); expect(sourceTurnFromMessages(turns[1]!.messages)).toBeNull(); + }); + + it("applies the existing redactor equally to canonical text and structured tool values", async () => { + const token = `sk-${"fixture".repeat(8)}`; + const messages = await read([...prefix, event("response_item", { type: "custom_tool_call", call_id: "a", name: "write", input: { token } }), event("response_item", { type: "custom_tool_call_output", call_id: "a", output: { token } }), ...end]); + const turn = sourceTurnFromMessages(messages)!; + expect(JSON.stringify(turn)).not.toContain(token); + expect(turn.toolCalls[0]?.input).toEqual({ token: "[REDACTED:openai_api_key]" }); + }); + it("keeps the native completion timestamp visible to incremental scanning", async () => { + const completedAt = "2026-09-09T10:05:00.000Z"; + const messages = await read([...prefix, message("assistant", "Done"), { ...event("event_msg", { type: "task_complete", turn_id: "turn-1" }), timestamp: completedAt }]); + expect(messages.at(-1)).toMatchObject({ role: "system", createdAt: completedAt, rawMeta: { sourceTurnState: "complete" } }); + expect(sourceTurnFromMessages(messages)?.completedAt).toBe(completedAt); + }); + + it("accepts duplicate evidence for the same canonical turn but does not merge conflicting content", async () => { + const messages = await read([...prefix, ...end]); + const turn = sourceTurnFromMessages(messages)!; + expect(sourceTurnFromMessages([...messages, ...messages])).toEqual(turn); + expect(sourceTurnFromMessages([...messages, { rawMeta: { sourceTurn: { ...turn, answer: "different" } } }])).toBeNull(); + }); + + it("retains numeric record order even when many events have the same timestamp", async () => { + const messages = await read([...prefix, ...Array.from({ length: 15 }, (_, i) => message("assistant", `progress ${i}`)), ...end]); + expect([...messages].sort((a, b) => a.messageId.localeCompare(b.messageId))).toEqual(messages); + }); + + it("resolves an initial user message followed by turn_context without inventing an ID", async () => { + const messages = await read([prefix[0], message("user", "Question"), event("turn_context", { turn_id: "turn-1" }), ...end]); + expect(sourceTurnFromMessages(messages)).toMatchObject({ turnId: "turn-1", query: "Question" }); + }); + + it("keeps a turn pending when a malformed record might hide a tool call or result", async () => { + const path = fixture([...prefix, ...end]); + const records = [...prefix, ...end].map(record => JSON.stringify(record)); + records.splice(3, 0, '{"type":"response_item","payload":'); + writeFileSync(path, records.join("\n") + "\n"); + const messages = await collect(readCodexRollout(path)); + expect(sourceTurnFromMessages(messages)).toBeNull(); + expect(messages.at(-1)?.rawMeta.sourceTurnReason).toBe("source_record_invalid"); + }); + + it("does not assign one result to two calls with the same native call id", async () => { + const messages = await read([...prefix, + event("response_item", { type: "function_call", call_id: "a", name: "read", arguments: "first" }), + event("response_item", { type: "function_call", call_id: "a", name: "write", arguments: "second" }), + event("response_item", { type: "function_call_output", call_id: "a", output: "ambiguous" }), ...end]); + expect(sourceTurnFromMessages(messages)?.toolCalls.every(call => call.output === undefined)).toBe(true); + }); + + it("keeps conflicting or incomplete repeated native turns pending in both channels", async () => { + const completed = [...prefix, ...end]; + for (const tail of [[...prefix, message("assistant", "changed"), end[1]], [...prefix, message("assistant", "still working")]]) { + const path = fixture([...completed, ...tail]); + expect(sourceTurnFromMessages(await collect(readCodexRollout(path)))).toBeNull(); + expect((await readCodexSourceTurn(path, { turnId: "turn-1", conversationId: "conversation", stop: true })).turn).toBeNull(); + } + }); + + it("does not emit startup or trailing system-only context as a pending memory turn", async () => { + const messages = await read([prefix[0], message("developer", "startup instructions"), ...prefix.slice(1), ...end, message("developer", "context for next turn")]); + expect(sourceTurnFromMessages(messages)).not.toBeNull(); + expect(messages.every(message => message.rawMeta.sourceTurnState === "complete")).toBe(true); + expect(await read([prefix[0], message("system", "startup only")])).toEqual([]); + }); + + it("reports pending evidence before content conflicts and distinguishes conflicts from missing identity", async () => { + const messages = await read([...prefix, ...end]); + const turn = sourceTurnFromMessages(messages)!; + const conflicting = [...messages, { rawMeta: { sourceTurn: { ...turn, answer: "different content" } } }]; + expect(sourceTurnFailureReason(conflicting)).toBe("source_turn_content_conflict"); + expect(sourceTurnFailureReason([...conflicting, { rawMeta: { sourceTurnState: "turn_incomplete", sourceTurnReason: "source_record_invalid" } }])).toBe("source_record_invalid"); + expect(sourceTurnFailureReason([{ rawMeta: {} }])).toBe("identity_unresolved"); + }); + +}); diff --git a/AgentSourceCore/src/codex-source-turn.ts b/AgentSourceCore/src/codex-source-turn.ts new file mode 100644 index 000000000..751fd3947 --- /dev/null +++ b/AgentSourceCore/src/codex-source-turn.ts @@ -0,0 +1,284 @@ +import { basename } from "node:path"; +import { readJsonlObjects } from "./jsonl-lines.js"; +import { redactSecrets } from "./secret-redactor.js"; + +export interface SourceToolResult { + id?: string; + output?: unknown; + status?: string; + success?: boolean; + error?: unknown; +} +export interface SourceToolCall extends SourceToolResult { + name: string; + input?: unknown; +} +export interface SourceTurn { + source: "codex"; + conversationId: string; + turnId: string; + startedAt: string; + completedAt: string; + sequence: number; + completionEvidence: string; + query: string; + answer: string; + status: "succeeded" | "failed"; + toolCalls: SourceToolCall[]; + toolResults: SourceToolResult[]; + workspacePath?: string; +} +export interface RawCodexMessage { + messageId: string; + conversationId: string; + role: "user" | "assistant" | "tool" | "system"; + content: string; + createdAt: string; + ordinal: number; + rawMeta: Readonly>; +} + +/** Canonical turn is stored once, on the final staged message, including its native completion evidence. */ +export function sourceTurnFromMessages(messages: readonly { rawMeta: Readonly> }[]): SourceTurn | null { + if (messages.some(message => message.rawMeta.sourceTurnState && message.rawMeta.sourceTurnState !== "complete")) return null; + const turns = messages.map(message => message.rawMeta.sourceTurn).filter(isRecord); + if (turns.length === 0) return null; + if (turns.some(turn => canonicalTurnContent(turn) !== canonicalTurnContent(turns[0]!))) return null; + const turn = turns[0]!; + if (turn.source !== "codex" || !text(turn.conversationId) || !text(turn.turnId) || !text(turn.completionEvidence)) return null; + return turn as unknown as SourceTurn; +} + +/** Preserve the reason a staged native turn cannot be submitted, including conflicting complete evidence. */ +export function sourceTurnFailureReason(messages: readonly { rawMeta: Readonly> }[]): string { + for (const message of messages) { + const state = text(message.rawMeta.sourceTurnState); + if (state && state !== "complete") return text(message.rawMeta.sourceTurnReason) || state; + } + const turns = messages.map(message => message.rawMeta.sourceTurn).filter(isRecord); + if (turns.length > 1 && turns.some(turn => canonicalTurnContent(turn) !== canonicalTurnContent(turns[0]!))) { + return "source_turn_content_conflict"; + } + return "identity_unresolved"; +} + +export function buildSourceTurnRequest(turn: SourceTurn, channel: "hook" | "agent_source_scan", profileId = "default") { + return { + sourceTurn: { + source: turn.source, profileId, conversationId: turn.conversationId, turnId: turn.turnId, + startedAt: turn.startedAt, completedAt: turn.completedAt, sequence: turn.sequence, completionEvidence: turn.completionEvidence + }, + source: turn.source, profileId, channel, query: turn.query, answer: turn.answer, + status: turn.status, toolCalls: turn.toolCalls, toolResults: turn.toolResults, + workspacePath: turn.workspacePath + }; +} + +/** Read one native turn at a time; neither assistant text alone nor EOF proves completion. */ +export async function* readCodexRollout( + filePath: string, + signal?: AbortSignal, + stopEvidence?: { conversationId?: string; turnId: string } +): AsyncIterable { + const fileId = rolloutFileId(filePath); + let conversationId = ""; + let workspacePath: string | undefined; + let current: RawCodexMessage[] = []; + let turnId = ""; + let startedAt = ""; + let sequence = 0; + let lineNumber = 0; + let invalidReason = ""; + let toolCalls: SourceToolCall[] = []; + let toolResults: SourceToolResult[] = []; + const toolNames = new Map(); + + function finish(completedAt = "", completionId = "", status: "succeeded" | "failed" = "succeeded", completionKind = "task_complete"): RawCodexMessage[] { + if (current.length === 0) return []; + if (current.every(message => message.role === "system")) { + current = []; invalidReason = ""; + return []; + } + const query = current.filter(message => message.role === "user").map(message => message.content).join("\n\n"); + const answer = current.filter(message => message.role === "assistant").map(message => message.content).join("\n\n"); + let reason = invalidReason; + if (!conversationId || !turnId) reason ||= "identity_unresolved"; + else if (!completedAt || completionId !== turnId) reason ||= "turn_incomplete"; + else if (!startedAt || !Number.isFinite(Date.parse(startedAt)) || !Number.isFinite(Date.parse(completedAt))) reason ||= "timestamp_unresolved"; + else if (!query.trim() || !answer.trim()) reason ||= "turn_content_incomplete"; + const finalMessage = [...current].reverse().find(message => message.role !== "system"); + const hasFinalAnswer = finalMessage?.role === "assistant" && finalMessage.rawMeta.sourcePhase === "final_answer"; + const canonicalCompletedAt = hasFinalAnswer ? finalMessage.createdAt : completedAt; + const completionEvidence = hasFinalAnswer + ? `final_answer:${text(finalMessage.rawMeta.sourceRecordId) || turnId}` + : `task_complete:${turnId}`; + if (completedAt && completionKind === "task_complete") { + current.push({ messageId: `${fileId}:${String(lineNumber).padStart(12, "0")}`, conversationId: conversationId || fileId, + role: "system", content: "Codex task_complete", createdAt: completedAt, ordinal: lineNumber, + rawMeta: { sourceFile: filePath, sourceTurnId: turnId || undefined, sourceTurnSequence: sequence, sourceTurnStartedAt: startedAt || undefined } + }); + } + const output = current.map(message => ({ ...message, rawMeta: { ...message.rawMeta, sourceTurnId: turnId || undefined, sourceTurnState: reason || "complete", sourceTurnReason: reason || undefined } })); + if (!reason) { + const resultsById = new Map(); + const duplicateIds = new Set(); + for (const result of toolResults) { + if (!result.id) continue; + if (resultsById.has(result.id)) duplicateIds.add(result.id); + else resultsById.set(result.id, result); + } + const callCounts = new Map(); + for (const call of toolCalls) { + if (call.id) callCounts.set(call.id, (callCounts.get(call.id) ?? 0) + 1); + } + const paired = toolCalls.map(call => { + const result = call.id && callCounts.get(call.id) === 1 && !duplicateIds.has(call.id) ? resultsById.get(call.id) : undefined; + return result ? { ...call, ...result, name: call.name, input: call.input } : call; + }); + const turn: SourceTurn = { + source: "codex", conversationId, turnId, startedAt, completedAt: canonicalCompletedAt, sequence, + completionEvidence, query: redactSecrets(query), answer: redactSecrets(answer), status, + toolCalls: paired.map(redactCall), toolResults: toolResults.map(redactResult), workspacePath + }; + const last = output[output.length - 1]!; + last.rawMeta = { ...last.rawMeta, sourceTurn: turn } as typeof last.rawMeta; + } + current = []; toolCalls = []; toolResults = []; toolNames.clear(); invalidReason = ""; + return output; + } + + for await (const record of readJsonlObjects(filePath, signal, reason => { invalidReason = reason; })) { + lineNumber += 1; + const payload = isRecord(record.payload) ? record.payload : {}; + const timestamp = iso(record.timestamp); + if (record.type === "session_meta") { + const nativeId = text(payload.session_id) || text(payload.id); + if (conversationId && nativeId && nativeId !== conversationId) { + invalidReason = "identity_conflict"; + yield* finish(); turnId = ""; + } + conversationId = nativeId || conversationId; + workspacePath = text(payload.cwd) || workspacePath; + continue; + } + if (record.type === "turn_context" || (record.type === "event_msg" && payload.type === "task_started")) { + const nextId = text(payload.turn_id); + if (nextId && nextId !== turnId) { + if (turnId || current.some(message => message.role === "assistant" || message.role === "tool")) yield* finish(); + turnId = nextId; + startedAt = current.find(message => message.role === "user")?.createdAt || timestamp; + sequence = lineNumber; + } + workspacePath = text(payload.cwd) || workspacePath; + continue; + } + if (record.type === "event_msg" && payload.type === "task_complete") { + const completeId = text(payload.turn_id); + if (completeId !== turnId) invalidReason = "identity_conflict"; + yield* finish(timestamp, completeId, payload.status === "failed" ? "failed" : "succeeded"); + turnId = ""; startedAt = ""; + continue; + } + if (record.type === "event_msg" && (payload.type === "turn_aborted" || payload.type === "task_aborted")) { + invalidReason = "turn_cancelled"; yield* finish(); turnId = ""; startedAt = ""; continue; + } + if (record.type !== "response_item") continue; + if (text(payload.turn_id) && text(payload.turn_id) !== turnId) invalidReason = "identity_conflict"; + let role: RawCodexMessage["role"]; + let content: string; + if (payload.type === "message") { + const rawRole = payload.role; + if (rawRole !== "user" && rawRole !== "assistant" && rawRole !== "developer" && rawRole !== "system") continue; + role = rawRole === "developer" ? "system" : rawRole; + content = Array.isArray(payload.content) ? payload.content.map(item => isRecord(item) ? text(item.text) : "").filter(Boolean).join("\n") : ""; + } else { + role = "tool"; + const id = text(payload.call_id) || text(payload.id) || undefined; + const status = text(payload.status) || undefined; + const success = toolSuccess(payload); + const error = payload.error; + if (payload.type === "function_call" || payload.type === "custom_tool_call" || payload.type === "web_search_call") { + const name = payload.type === "web_search_call" ? "web_search" : text(payload.name) || "tool"; + const input = payload.type === "web_search_call" ? payload.action : payload.arguments ?? payload.input; + const call = compact({ id, name, status, success, error, input, output: payload.output ?? payload.result }) as unknown as SourceToolCall; + toolCalls.push(call); if (id) toolNames.set(id, name); + content = renderTool(call); + } else if (payload.type === "function_call_output" || payload.type === "custom_tool_call_output") { + const result = compact({ id, output: payload.output ?? payload.result, status, success, error }) as SourceToolResult; + toolResults.push(result); content = renderTool({ ...result, name: id ? toolNames.get(id) ?? "tool" : "tool" }); + } else continue; + } + if (!content) continue; + if (!timestamp && role !== "system") invalidReason = "timestamp_unresolved"; + current.push({ messageId: `${fileId}:${String(lineNumber).padStart(12, "0")}`, conversationId: conversationId || fileId, role, content, createdAt: timestamp || new Date(0).toISOString(), ordinal: lineNumber, + rawMeta: { sourceFile: filePath, sourceRecordId: text(payload.id) || undefined, sourcePhase: text(payload.phase) || undefined, sourceTurnId: turnId || undefined, sourceTurnSequence: sequence, sourceTurnStartedAt: startedAt || undefined } }); + } + const lastResponse = [...current].reverse().find(message => message.role !== "system"); + const stopMatches = stopEvidence?.turnId === turnId && + (!stopEvidence.conversationId || stopEvidence.conversationId === conversationId); + if (stopMatches && lastResponse?.role === "assistant" && lastResponse.rawMeta.sourcePhase === "final_answer") { + yield* finish(lastResponse.createdAt, turnId, "succeeded", "stop"); + } else { + yield* finish(); + } +} + +/** Hook uses the exact same streaming parser as the scan adapter. */ +export async function readCodexSourceTurn(filePath: string, expected: { conversationId?: string; turnId?: string; stop?: boolean } = {}): Promise<{ turn: SourceTurn | null; reason?: string }> { + let latest: SourceTurn | null = null; + let observed: SourceTurn | null = null; + let conflict = false; + let unresolved = false; + let reason = "identity_unresolved"; + let latestTurnId: unknown; + const stopEvidence = expected.stop && expected.turnId + ? { conversationId: expected.conversationId, turnId: expected.turnId } + : undefined; + for await (const message of readCodexRollout(filePath, undefined, stopEvidence)) { + if (expected.conversationId && message.conversationId !== expected.conversationId) return { turn: null, reason: "identity_conflict" }; + if (expected.turnId && message.rawMeta.sourceTurnId !== expected.turnId) continue; + if (message.rawMeta.sourceTurnId !== latestTurnId) { + latest = null; observed = null; conflict = false; unresolved = false; + } + latestTurnId = message.rawMeta.sourceTurnId; + reason = text(message.rawMeta.sourceTurnReason) || "turn_incomplete"; + if (message.rawMeta.sourceTurnState !== "complete") { latest = null; unresolved = true; } + const turn = sourceTurnFromMessages([message]); + if (turn) { + if (observed && canonicalTurnContent(observed) !== canonicalTurnContent(turn)) conflict = true; + observed = turn; + latest = turn; + } + } + if (conflict) return { turn: null, reason: "source_turn_content_conflict" }; + if (unresolved) return { turn: null, reason }; + return latest ? { turn: latest } : { turn: null, reason }; +} + +function canonicalTurnContent(turn: Record | SourceTurn): string { + const { sequence: _sequence, ...content } = turn; + return JSON.stringify(content); +} + +function redactCall(call: SourceToolCall): SourceToolCall { return { ...call, name: redactSecrets(call.name), ...redactResult(call), ...(call.input !== undefined ? { input: redactValue(call.input) } : {}) }; } +function redactResult(result: SourceToolResult): SourceToolResult { return { ...result, ...(result.output !== undefined ? { output: redactValue(result.output) } : {}), ...(result.error !== undefined ? { error: redactValue(result.error) } : {}) }; } +function redactValue(value: unknown): unknown { + if (typeof value === "string") return redactSecrets(value); + if (Array.isArray(value)) return value.map(redactValue); + if (isRecord(value)) return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, redactValue(entry)])); + return value; +} +function toolSuccess(payload: Record): boolean | undefined { + if (typeof payload.success === "boolean") return payload.success; + if (typeof payload.is_error === "boolean") return !payload.is_error; + if ((payload.error !== undefined && payload.error !== null) || payload.status === "failed" || payload.status === "cancelled") return false; + if (payload.status === "completed" || payload.status === "succeeded") return true; + return undefined; +} +function renderTool(tool: SourceToolCall): string { return [`Tool: ${tool.name}`, tool.id ? `Call ID: ${tool.id}` : undefined, tool.status ? `Status: ${tool.status}` : undefined, tool.input !== undefined ? `Input:\n${format(tool.input)}` : undefined, tool.output !== undefined ? `Output:\n${format(tool.output)}` : undefined].filter(Boolean).join("\n\n"); } +function format(value: unknown): string { return typeof value === "string" ? value.trim() : JSON.stringify(value, null, 2); } +function compact(value: Record): Record { return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined)); } +function text(value: unknown): string { return typeof value === "string" ? value : ""; } +function iso(value: unknown): string { const parsed = typeof value === "string" ? Date.parse(value) : NaN; return Number.isFinite(parsed) ? new Date(parsed).toISOString() : ""; } +function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } +function rolloutFileId(path: string): string { const name = basename(path).replace(/\.jsonl$/u, ""); return name.match(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu)?.[0] ?? name; } diff --git a/AgentSourceCore/src/index.ts b/AgentSourceCore/src/index.ts index 4132edb27..b00d32933 100644 --- a/AgentSourceCore/src/index.ts +++ b/AgentSourceCore/src/index.ts @@ -1,3 +1,6 @@ +export * from "./codex-source-turn.js"; +export * from "./secret-redactor.js"; +export * from "./jsonl-lines.js"; import { createHash } from "node:crypto"; export interface ConversationMessage { @@ -157,19 +160,32 @@ export async function* orderedTurns(messages: AsyncIterable let turnIndex = 0; for await (const message of messages) { if (message.conversationId !== conversationId) { - if (isCompleteTurn(current)) yield { sourceId: current[0]!.sourceId, conversationId, turnIndex, messages: current }; + if (shouldEmitTurn(current)) yield { sourceId: current[0]!.sourceId, conversationId, turnIndex, messages: current }; current = []; conversationId = message.conversationId; turnIndex = 0; } - if (message.role === "user" && current.length > 0) { - if (isCompleteTurn(current)) yield { sourceId: current[0]!.sourceId, conversationId, turnIndex, messages: current }; + if (current.length > 0 && beginsNextTurn(current, message)) { + if (shouldEmitTurn(current)) yield { sourceId: current[0]!.sourceId, conversationId, turnIndex, messages: current }; turnIndex += 1; current = []; } current.push(message); } - if (isCompleteTurn(current)) yield { sourceId: current[0]!.sourceId, conversationId, turnIndex, messages: current }; + if (shouldEmitTurn(current)) yield { sourceId: current[0]!.sourceId, conversationId, turnIndex, messages: current }; +} + +function shouldEmitTurn(messages: readonly ConversationMessage[]): boolean { + return messages.length > 0 && (messages[0]!.sourceId === "codex" || isCompleteTurn(messages)); +} + +function beginsNextTurn(current: readonly ConversationMessage[], next: ConversationMessage): boolean { + if (next.sourceId === "codex") { + const currentId = current[0]!.rawMeta.sourceTurnId; + const nextId = next.rawMeta.sourceTurnId; + if (currentId || nextId) return currentId !== nextId; + } + return next.role === "user"; } export function isCompleteTurn(messages: readonly ConversationMessage[]): boolean { @@ -212,6 +228,10 @@ export function conversationContentHash(messages: Iterable) } export function stableTurnIdentity(turn: ImportedTurn): string { + const nativeId = turn.messages[0]?.rawMeta.sourceTurnId; + if (turn.sourceId === "codex") { + return `${turn.sourceId}::${turn.conversationId}::${typeof nativeId === "string" ? nativeId : turn.messages[0]?.messageId ?? "unresolved"}`; + } const firstUser = turn.messages.find((message) => message.role === "user"); if (!firstUser) throw new Error("turn is missing user message"); return `${turn.sourceId}::${turn.conversationId}::${firstUser.messageId}`; diff --git a/AgentSourceCore/src/jsonl-lines.ts b/AgentSourceCore/src/jsonl-lines.ts new file mode 100644 index 000000000..d02b5e216 --- /dev/null +++ b/AgentSourceCore/src/jsonl-lines.ts @@ -0,0 +1,101 @@ +/** Jsonl lines module. */ +import { createReadStream } from "node:fs"; + +export type JsonPrimitive = string | number | boolean | null; +export type JsonValue = JsonPrimitive | JsonObject | JsonValue[]; +export type JsonObject = { readonly [key: string]: JsonValue }; + +/** + * Streams valid object rows from a JSONL file. + * Malformed and non-object rows are skipped without interrupting the stream. + * + * @param filePath JSONL file path. + * @param signal Optional abort signal. + * @returns The JSON objects parsed line by line. + */ +export async function* readJsonlObjects(filePath: string, signal?: AbortSignal, onInvalidRecord?: (reason: string) => void): AsyncIterable { + const stream = createReadStream(filePath); + const maxRecordBytes = 64 * 1024 * 1024; + let segments: Buffer[] = []; + let recordBytes = 0; + let overLimit = false; + const append = (segment: Buffer): void => { + if (overLimit || segment.length === 0) return; + recordBytes += segment.length; + if (recordBytes > maxRecordBytes) { + segments = []; + overLimit = true; + return; + } + segments.push(segment); + }; + const reset = (): void => { + segments = []; + recordBytes = 0; + overLimit = false; + }; + const parseSegments = (): JsonObject | null => { + if (overLimit) { + onInvalidRecord?.("source_record_over_limit"); + return null; + } + const line = segments.length === 1 ? segments[0]! : Buffer.concat(segments, recordBytes); + const text = line.toString("utf8").trim(); + if (!text) return null; + try { + const parsed = JSON.parse(text) as unknown; + if (isJsonObject(parsed)) return parsed; + onInvalidRecord?.("source_record_invalid"); + return null; + } catch { + onInvalidRecord?.("source_record_invalid"); + return null; + } + }; + + try { + for await (const chunk of stream) { + throwIfAborted(signal, filePath); + const buffer = chunk as Buffer; + let start = 0; + while (start <= buffer.length) { + const newline = buffer.indexOf(0x0a, start); + if (newline < 0) { + append(buffer.subarray(start)); + break; + } + append(buffer.subarray(start, newline)); + const parsed = parseSegments(); + reset(); + if (parsed) yield parsed; + start = newline + 1; + } + } + const parsed = parseSegments(); + if (parsed) yield parsed; + } finally { + stream.destroy(); + } +} + +/** + * Abort-signal check. + * + * @param signal Optional abort signal. + * @param filePath Current file path. + */ +function throwIfAborted(signal: AbortSignal | undefined, filePath: string): void { + if (signal?.aborted) { + throw new DOMException(`JSONL read aborted: ${filePath}`, "AbortError"); + } +} + +/** + * JSON object type guard. + * + * @param value Unknown value. + * @returns Whether it is a non-array object. + */ +function isJsonObject(value: unknown): value is JsonObject { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/AgentSourceCore/src/secret-redactor.ts b/AgentSourceCore/src/secret-redactor.ts new file mode 100644 index 000000000..49a9a4ef5 --- /dev/null +++ b/AgentSourceCore/src/secret-redactor.ts @@ -0,0 +1,120 @@ +/** Type definition for redaction rule. */ +type RedactionRule = { + /** Pattern. */ + pattern: RegExp; + /** Token. */ + token: string; + /** Replace. */ + replace?: (match: string, ...groups: string[]) => string; +}; + +const REDACTION_RULES: readonly RedactionRule[] = [ + { + pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, + token: "[REDACTED:ssh_private_key]" + }, + { + pattern: /\b(Authorization\s*:\s*Bearer\s+)[A-Za-z0-9._~+/=-]+/gi, + token: "[REDACTED:authorization_bearer]", + replace: (_match, prefix: string) => `${prefix}[REDACTED:authorization_bearer]` + }, + { + pattern: /\bsk-ant-api\d{2}-[A-Za-z0-9_-]{40,}\b/g, + token: "[REDACTED:anthropic_api_key]" + }, + { + pattern: /\bsk-(?:proj-)?[A-Za-z0-9_-]{40,}\b/g, + token: "[REDACTED:openai_api_key]" + }, + { + pattern: /\bAIza[A-Za-z0-9_-]{32,}\b/g, + token: "[REDACTED:google_api_key]" + }, + { + pattern: /\b([A-Za-z0-9_]*password[A-Za-z0-9_]*\s*[:=]\s*)(?:"[^"\n]+"|'[^'\n]+'|[^\s#&]+)/gi, + token: "[REDACTED:password]", + replace: (_match, prefix: string) => `${prefix}[REDACTED:password]` + } +]; + +const BASE64_SECRET_TOKEN = "[REDACTED:base64_secret]"; +const BASE64_SECRET_MIN_LENGTH = 32; +const LARGE_BASE64_PAYLOAD_MIN_LENGTH = 4096; + +/** + * Redacts common secrets from text. + * + * @param input Raw message text from an external Agent. + * @returns The plain text with secrets replaced, or the original text when no rule matches. + */ +export function redactSecrets(input: string): string { + const withoutLargeBinaryPayloads = redactBase64Runs(input, LARGE_BASE64_PAYLOAD_MIN_LENGTH); + const redacted = REDACTION_RULES.reduce((current, rule) => { + if (rule.replace) { + return current.replace(rule.pattern, rule.replace); + } + + return current.replace(rule.pattern, rule.token); + }, withoutLargeBinaryPayloads); + + return redactBase64Runs(redacted, BASE64_SECRET_MIN_LENGTH); +} + +function redactBase64Runs(input: string, minLength: number): string { + let output = ""; + let cursor = 0; + let index = 0; + + while (index < input.length) { + if (!isBase64CoreChar(input.charCodeAt(index))) { + index += 1; + continue; + } + + const start = index; + while (index < input.length && isBase64CoreChar(input.charCodeAt(index))) { + index += 1; + } + const coreEnd = index; + let padding = 0; + while (padding < 2 && input.charCodeAt(index) === 61) { + index += 1; + padding += 1; + } + + if (coreEnd - start >= minLength && hasBase64Boundary(input, start, index)) { + output += input.slice(cursor, start); + output += BASE64_SECRET_TOKEN; + cursor = index; + } + } + + if (cursor === 0) { + return input; + } + + return output + input.slice(cursor); +} + +function hasBase64Boundary(input: string, start: number, end: number): boolean { + return !isAsciiWord(input.charCodeAt(start - 1)) && !isAsciiWord(input.charCodeAt(end)); +} + +function isBase64CoreChar(code: number): boolean { + return ( + (code >= 65 && code <= 90) || + (code >= 97 && code <= 122) || + (code >= 48 && code <= 57) || + code === 43 || + code === 47 + ); +} + +function isAsciiWord(code: number): boolean { + return ( + (code >= 65 && code <= 90) || + (code >= 97 && code <= 122) || + (code >= 48 && code <= 57) || + code === 95 + ); +} diff --git a/App/backend/README.md b/App/backend/README.md index f4da7b39c..0de90c276 100644 --- a/App/backend/README.md +++ b/App/backend/README.md @@ -115,7 +115,7 @@ Every route in this table requires the local runtime token. | Claude Code | `~/.claude/projects/**/*.jsonl` | `~/.claude/CLAUDE.md`, `skills/memmy-memory/`, hooks, and the resume command | | Codex | `~/.codex/sessions/**/rollout-*.jsonl` | `~/.codex/AGENTS.md`, `skills/memmy-memory/`, and hooks | | OpenCode | `${XDG_DATA_HOME:-~/.local/share}/opencode/opencode.db` | `${XDG_CONFIG_HOME:-~/.config}/opencode/AGENTS.md`, `skills/memmy-memory/`, plugin, and resume command | -| OpenClaw | SQLite databases under `~/.openclaw/` | Workspace `AGENTS.md`, `~/.openclaw/skills/memmy-memory/`, and the Memory extension | +| OpenClaw | SQLite databases under `~/.openclaw/` | Workspace `../../AGENTS.md`, `~/.openclaw/skills/memmy-memory/`, and the Memory extension | | Hermes | `~/.hermes/sessions/**/*.jsonl` and `~/.hermes/state.db` | `~/.hermes/SOUL.md`, `skills/memmy-memory/`, and Memory/resume plugins | | WorkBuddy | `~/.workbuddy/projects/**/*.jsonl` | `~/.workbuddy/skills/memmy-memory/` | | Pi | `~/.pi/agent/sessions/**/*.jsonl` | `~/.pi/agent/skills/memmy-memory/` | diff --git a/App/backend/local-api-contracts/src/memory-runtime.ts b/App/backend/local-api-contracts/src/memory-runtime.ts index 8a15f98be..cee5eff6e 100644 --- a/App/backend/local-api-contracts/src/memory-runtime.ts +++ b/App/backend/local-api-contracts/src/memory-runtime.ts @@ -474,6 +474,31 @@ export const CompleteTurnOutputSchema = z.object({ }); export type CompleteTurnOutput = z.infer; +/** Completed native Agent turn shared by Hook and automatic scanning. */ +export const SourceTurnCompleteInputSchema = CompleteTurnInputSchema.omit({ sessionId: true }).extend({ + sessionId: NonEmptyStringSchema.optional(), + sourceTurn: z.object({ + source: NonEmptyStringSchema, + profileId: NonEmptyStringSchema, + conversationId: NonEmptyStringSchema, + turnId: NonEmptyStringSchema, + startedAt: IsoTimeSchema, + completedAt: IsoTimeSchema, + sequence: z.number().int().nonnegative().optional(), + completionEvidence: NonEmptyStringSchema + }), + channel: z.enum(["hook", "agent_source_scan"]), + workspacePath: z.string().optional() +}); +export type SourceTurnCompleteInput = z.infer; + +export const SourceTurnCompleteOutputSchema = z.object({ + status: z.enum(["stored", "existing", "rejected", "pending", "conflict"]), + reason: z.string().optional(), + result: CompleteTurnOutputSchema.partial({ changeSeq: true }).optional() +}); +export type SourceTurnCompleteOutput = z.infer; + /** Definition for search input. */ export const SearchInputSchema = RuntimeRequestFieldsSchema.extend({ query: NonEmptyStringSchema, diff --git a/App/backend/src/adapters/outbound/agent-source/codex/adapter.ts b/App/backend/src/adapters/outbound/agent-source/codex/adapter.ts index 62b5dcab9..fd11dfd00 100644 --- a/App/backend/src/adapters/outbound/agent-source/codex/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/codex/adapter.ts @@ -102,7 +102,8 @@ function toConversationMessage( createdAt: rawMessage.createdAt, workspacePath, gitRoot, - rawMeta: Object.freeze({}) + ordinal: rawMessage.ordinal, + rawMeta: rawMessage.rawMeta }; } diff --git a/App/backend/src/adapters/outbound/agent-source/codex/rollout-reader.ts b/App/backend/src/adapters/outbound/agent-source/codex/rollout-reader.ts index 6dd193392..9b2de0524 100644 --- a/App/backend/src/adapters/outbound/agent-source/codex/rollout-reader.ts +++ b/App/backend/src/adapters/outbound/agent-source/codex/rollout-reader.ts @@ -1,208 +1 @@ -/** Rollout reader module. */ -import { basename } from "node:path"; -import { readJsonlObjects, type JsonObject } from "../jsonl-lines.js"; - -const MAX_TOOL_NAME_ENTRIES = 4096; - -export interface RawCodexMessage { - /** Message id. */ - messageId: string; - conversationId: string; - role: "user" | "assistant" | "tool" | "system"; - content: string; - createdAt: string; -} - -/** Rollout reader module. */ -export async function* readCodexRollout(filePath: string, signal?: AbortSignal): AsyncIterable { - const rolloutId = parseRolloutId(filePath); - const toolNamesByCallId = new Map(); - let lineNumber = 0; - - for await (const record of readJsonlObjects(filePath, signal)) { - lineNumber += 1; - const message = toRawCodexMessage(record, rolloutId, lineNumber, toolNamesByCallId); - if (message) { - yield message; - } - } -} - -/** Handles to raw codex message. */ -function toRawCodexMessage( - record: JsonObject, - rolloutId: string, - lineNumber: number, - toolNamesByCallId: Map -): RawCodexMessage | null { - if (record.type !== "response_item" || !isRecord(record.payload)) { - return null; - } - - if (record.payload.type !== "message") { - return toToolMessage(record.payload, rolloutId, lineNumber, normalizeTimestamp(record.timestamp), toolNamesByCallId); - } - - const role = record.payload.role; - if (role !== "user" && role !== "assistant" && role !== "developer" && role !== "system") { - return null; - } - - const content = getContentText(record.payload.content); - if (!content) { - return null; - } - - return { - messageId: `${rolloutId}:${lineNumber}`, - conversationId: rolloutId, - role: role === "developer" ? "system" : role, - content, - createdAt: normalizeTimestamp(record.timestamp) - }; -} - -function toToolMessage( - payload: Record, - rolloutId: string, - lineNumber: number, - createdAt: string, - toolNamesByCallId: Map -): RawCodexMessage | null { - const type = payload.type; - if (type === "function_call" || type === "custom_tool_call") { - const callId = getString(payload.call_id) ?? getString(payload.id); - const name = getString(payload.name) ?? "tool"; - if (callId) { - toolNamesByCallId.set(callId, name); - if (toolNamesByCallId.size > MAX_TOOL_NAME_ENTRIES) { - const oldest = toolNamesByCallId.keys().next().value; - if (typeof oldest === "string") toolNamesByCallId.delete(oldest); - } - } - return { - messageId: `${rolloutId}:${lineNumber}`, - conversationId: rolloutId, - role: "tool", - content: renderToolMessage({ - name, - callId, - status: getString(payload.status), - input: firstDefined(payload.arguments, payload.input) - }), - createdAt - }; - } - - if (type === "function_call_output" || type === "custom_tool_call_output") { - const callId = getString(payload.call_id) ?? getString(payload.id); - return { - messageId: `${rolloutId}:${lineNumber}`, - conversationId: rolloutId, - role: "tool", - content: renderToolMessage({ - name: callId ? toolNamesByCallId.get(callId) ?? "tool" : "tool", - callId, - status: getString(payload.status), - output: firstDefined(payload.output, payload.result) - }), - createdAt - }; - } - - if (type === "web_search_call") { - return { - messageId: `${rolloutId}:${lineNumber}`, - conversationId: rolloutId, - role: "tool", - content: renderToolMessage({ - name: "web_search", - callId: getString(payload.call_id) ?? getString(payload.id), - status: getString(payload.status), - input: payload.action - }), - createdAt - }; - } - - return null; -} - -/** - * Parses the rollout uuid from the file name. - * - * @param filePath Rollout path. - * @returns The uuid, falling back to the file name. - */ -function parseRolloutId(filePath: string): string { - const name = basename(filePath).replace(/\.jsonl$/, ""); - const uuid = name.match(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i)?.[0]; - return uuid ?? name; -} - -/** - * Extracts the text from Codex content. - * - * @param content Raw payload.content value. - * @returns The merged text, or null. - */ -function getContentText(content: unknown): string | null { - if (!Array.isArray(content)) { - return null; - } - - const text = content - .filter(isRecord) - .map((item) => (typeof item.text === "string" ? item.text : null)) - .filter((item): item is string => Boolean(item)) - .join("\n"); - return text.length > 0 ? text : null; -} - -function renderToolMessage(input: { - name: string; - callId?: string; - status?: string; - input?: unknown; - output?: unknown; -}): string { - return [ - `Tool: ${input.name}`, - input.callId ? `Call ID: ${input.callId}` : undefined, - input.status ? `Status: ${input.status}` : undefined, - input.input !== undefined ? `Input:\n${formatToolPayload(input.input)}` : undefined, - input.output !== undefined ? `Output:\n${formatToolPayload(input.output)}` : undefined - ].filter(Boolean).join("\n\n"); -} - -function formatToolPayload(value: unknown): string { - if (typeof value === "string") { - return value.trim(); - } - try { - return JSON.stringify(value, null, 2); - } catch { - return String(value); - } -} - -function firstDefined(...values: unknown[]): unknown { - return values.find((value) => value !== undefined && value !== null); -} - -function getString(value: unknown): string | undefined { - return typeof value === "string" && value.length > 0 ? value : undefined; -} - -function normalizeTimestamp(value: unknown): string { - if (typeof value === "string") { - const date = new Date(value); - return Number.isNaN(date.getTime()) ? new Date(0).toISOString() : date.toISOString(); - } - - return new Date(0).toISOString(); -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} +export { readCodexRollout, type RawCodexMessage } from "@memmy/agent-source-core"; diff --git a/App/backend/src/adapters/outbound/agent-source/codex/tests/adapter.test.ts b/App/backend/src/adapters/outbound/agent-source/codex/tests/adapter.test.ts index bf6d549f7..fe299fafb 100644 --- a/App/backend/src/adapters/outbound/agent-source/codex/tests/adapter.test.ts +++ b/App/backend/src/adapters/outbound/agent-source/codex/tests/adapter.test.ts @@ -27,23 +27,23 @@ describe("codex source adapter", () => { expect(messages).toEqual([ expect.objectContaining({ - messageId: "019e72be-500b-7f02-9400-112c5a194e5c:2", + messageId: "019e72be-500b-7f02-9400-112c5a194e5c:000000000002", conversationId: "019e72be-500b-7f02-9400-112c5a194e5c", role: "user", content: `Use OPENAI_API_KEY=${SYNTHETIC_API_KEY}` }), expect.objectContaining({ - messageId: "019e72be-500b-7f02-9400-112c5a194e5c:3", + messageId: "019e72be-500b-7f02-9400-112c5a194e5c:000000000003", role: "tool", content: expect.stringContaining("Tool: shell") }), expect.objectContaining({ - messageId: "019e72be-500b-7f02-9400-112c5a194e5c:4", + messageId: "019e72be-500b-7f02-9400-112c5a194e5c:000000000004", role: "tool", content: expect.stringContaining("Output:") }), expect.objectContaining({ - messageId: "019e72be-500b-7f02-9400-112c5a194e5c:5", + messageId: "019e72be-500b-7f02-9400-112c5a194e5c:000000000005", role: "assistant", content: "Done" }) diff --git a/App/backend/src/adapters/outbound/agent-source/jsonl-lines.ts b/App/backend/src/adapters/outbound/agent-source/jsonl-lines.ts index c0fd7cbd6..a01d98e59 100644 --- a/App/backend/src/adapters/outbound/agent-source/jsonl-lines.ts +++ b/App/backend/src/adapters/outbound/agent-source/jsonl-lines.ts @@ -1,98 +1 @@ -/** Jsonl lines module. */ -import { createReadStream } from "node:fs"; - -export type JsonPrimitive = string | number | boolean | null; -export type JsonValue = JsonPrimitive | JsonObject | JsonValue[]; -export type JsonObject = { readonly [key: string]: JsonValue }; - -/** - * Streams valid object rows from a JSONL file. - * Malformed and non-object rows are skipped without interrupting the stream. - * - * @param filePath JSONL file path. - * @param signal Optional abort signal. - * @returns The JSON objects parsed line by line. - */ -export async function* readJsonlObjects(filePath: string, signal?: AbortSignal): AsyncIterable { - const stream = createReadStream(filePath); - const maxRecordBytes = 64 * 1024 * 1024; - let segments: Buffer[] = []; - let recordBytes = 0; - let overLimit = false; - - const append = (segment: Buffer): void => { - if (overLimit || segment.length === 0) return; - recordBytes += segment.length; - if (recordBytes > maxRecordBytes) { - segments = []; - overLimit = true; - return; - } - segments.push(segment); - }; - - const reset = (): void => { - segments = []; - recordBytes = 0; - overLimit = false; - }; - - const parseSegments = (): JsonObject | null => { - if (overLimit) return null; - const line = segments.length === 1 ? segments[0]! : Buffer.concat(segments, recordBytes); - const text = line.toString("utf8").trim(); - if (!text) return null; - try { - const parsed = JSON.parse(text) as unknown; - return isJsonObject(parsed) ? parsed : null; - } catch { - return null; - } - }; - - try { - for await (const chunk of stream) { - throwIfAborted(signal, filePath); - const buffer = chunk as Buffer; - let start = 0; - while (start <= buffer.length) { - const newline = buffer.indexOf(0x0a, start); - if (newline < 0) { - append(buffer.subarray(start)); - break; - } - append(buffer.subarray(start, newline)); - const parsed = parseSegments(); - reset(); - if (parsed) yield parsed; - start = newline + 1; - } - } - const parsed = parseSegments(); - if (parsed) yield parsed; - } finally { - stream.destroy(); - } -} - -/** - * Abort-signal check. - * - * @param signal Optional abort signal. - * @param filePath Current file path. - */ -function throwIfAborted(signal: AbortSignal | undefined, filePath: string): void { - if (signal?.aborted) { - throw new DOMException(`JSONL read aborted: ${filePath}`, "AbortError"); - } -} - -/** - * JSON object type guard. - * - * @param value Unknown value. - * @returns Whether it is a non-array object. - */ -function isJsonObject(value: unknown): value is JsonObject { - return typeof value === "object" && value !== null && !Array.isArray(value); -} +export { readJsonlObjects, type JsonObject, type JsonValue, type JsonPrimitive } from "@memmy/agent-source-core"; diff --git a/App/backend/src/adapters/outbound/agent-source/secret-redactor.ts b/App/backend/src/adapters/outbound/agent-source/secret-redactor.ts index 49a9a4ef5..7038a55b1 100644 --- a/App/backend/src/adapters/outbound/agent-source/secret-redactor.ts +++ b/App/backend/src/adapters/outbound/agent-source/secret-redactor.ts @@ -1,120 +1 @@ -/** Type definition for redaction rule. */ -type RedactionRule = { - /** Pattern. */ - pattern: RegExp; - /** Token. */ - token: string; - /** Replace. */ - replace?: (match: string, ...groups: string[]) => string; -}; - -const REDACTION_RULES: readonly RedactionRule[] = [ - { - pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, - token: "[REDACTED:ssh_private_key]" - }, - { - pattern: /\b(Authorization\s*:\s*Bearer\s+)[A-Za-z0-9._~+/=-]+/gi, - token: "[REDACTED:authorization_bearer]", - replace: (_match, prefix: string) => `${prefix}[REDACTED:authorization_bearer]` - }, - { - pattern: /\bsk-ant-api\d{2}-[A-Za-z0-9_-]{40,}\b/g, - token: "[REDACTED:anthropic_api_key]" - }, - { - pattern: /\bsk-(?:proj-)?[A-Za-z0-9_-]{40,}\b/g, - token: "[REDACTED:openai_api_key]" - }, - { - pattern: /\bAIza[A-Za-z0-9_-]{32,}\b/g, - token: "[REDACTED:google_api_key]" - }, - { - pattern: /\b([A-Za-z0-9_]*password[A-Za-z0-9_]*\s*[:=]\s*)(?:"[^"\n]+"|'[^'\n]+'|[^\s#&]+)/gi, - token: "[REDACTED:password]", - replace: (_match, prefix: string) => `${prefix}[REDACTED:password]` - } -]; - -const BASE64_SECRET_TOKEN = "[REDACTED:base64_secret]"; -const BASE64_SECRET_MIN_LENGTH = 32; -const LARGE_BASE64_PAYLOAD_MIN_LENGTH = 4096; - -/** - * Redacts common secrets from text. - * - * @param input Raw message text from an external Agent. - * @returns The plain text with secrets replaced, or the original text when no rule matches. - */ -export function redactSecrets(input: string): string { - const withoutLargeBinaryPayloads = redactBase64Runs(input, LARGE_BASE64_PAYLOAD_MIN_LENGTH); - const redacted = REDACTION_RULES.reduce((current, rule) => { - if (rule.replace) { - return current.replace(rule.pattern, rule.replace); - } - - return current.replace(rule.pattern, rule.token); - }, withoutLargeBinaryPayloads); - - return redactBase64Runs(redacted, BASE64_SECRET_MIN_LENGTH); -} - -function redactBase64Runs(input: string, minLength: number): string { - let output = ""; - let cursor = 0; - let index = 0; - - while (index < input.length) { - if (!isBase64CoreChar(input.charCodeAt(index))) { - index += 1; - continue; - } - - const start = index; - while (index < input.length && isBase64CoreChar(input.charCodeAt(index))) { - index += 1; - } - const coreEnd = index; - let padding = 0; - while (padding < 2 && input.charCodeAt(index) === 61) { - index += 1; - padding += 1; - } - - if (coreEnd - start >= minLength && hasBase64Boundary(input, start, index)) { - output += input.slice(cursor, start); - output += BASE64_SECRET_TOKEN; - cursor = index; - } - } - - if (cursor === 0) { - return input; - } - - return output + input.slice(cursor); -} - -function hasBase64Boundary(input: string, start: number, end: number): boolean { - return !isAsciiWord(input.charCodeAt(start - 1)) && !isAsciiWord(input.charCodeAt(end)); -} - -function isBase64CoreChar(code: number): boolean { - return ( - (code >= 65 && code <= 90) || - (code >= 97 && code <= 122) || - (code >= 48 && code <= 57) || - code === 43 || - code === 47 - ); -} - -function isAsciiWord(code: number): boolean { - return ( - (code >= 65 && code <= 90) || - (code >= 97 && code <= 122) || - (code >= 48 && code <= 57) || - code === 95 - ); -} +export { redactSecrets } from "@memmy/agent-source-core"; diff --git a/App/backend/src/adapters/outbound/memory-client/http-memory-client.ts b/App/backend/src/adapters/outbound/memory-client/http-memory-client.ts index 739d6876e..31dc609a6 100644 --- a/App/backend/src/adapters/outbound/memory-client/http-memory-client.ts +++ b/App/backend/src/adapters/outbound/memory-client/http-memory-client.ts @@ -4,6 +4,7 @@ import { ApiErrorBodySchema, CloseSessionOutputSchema, CompleteTurnOutputSchema, + SourceTurnCompleteOutputSchema, DeleteMemoryOutputSchema, DeletePanelTaskOutputSchema, EnqueueImportSummariesOutputSchema, @@ -166,6 +167,10 @@ export function createHttpMemoryClient( }); }, + async completeSourceTurn(input, context) { + return request("POST", "completeSourceTurn", SourceTurnCompleteOutputSchema, { body: input, context }); + }, + async search(input, context) { return request("POST", "search", SearchOutputSchema, { body: input, context }); }, diff --git a/App/backend/src/adapters/outbound/memory-client/memory-layer-endpoints.ts b/App/backend/src/adapters/outbound/memory-client/memory-layer-endpoints.ts index a8b46b335..22a51e0c3 100644 --- a/App/backend/src/adapters/outbound/memory-client/memory-layer-endpoints.ts +++ b/App/backend/src/adapters/outbound/memory-client/memory-layer-endpoints.ts @@ -9,6 +9,7 @@ export const MEMORY_LAYER_PATHS = Object.freeze({ closeSession: "/api/v1/sessions/:sessionId/close", startTurn: "/api/v1/turns/start", completeTurn: "/api/v1/turns/:turnId/complete", + completeSourceTurn: "/api/v1/source-turns/complete", search: "/api/v1/memory/search", addMemory: "/api/v1/memory/add", getMemory: "/api/v1/memory/:id", diff --git a/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts b/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts index 75cb6e228..d658e9bb6 100644 --- a/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts +++ b/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts @@ -12,6 +12,24 @@ afterEach(async () => { }); describe("HttpMemoryClient", () => { + it("posts a native turn without a Runtime Session and preserves pending reasons", async () => { + const calls: Array<{ url: string; init?: RequestInit }> = []; + const client = createHttpMemoryClient({ baseUrl: "http://memory.test", token: "fixture-token", timeoutMs: 500, maxRetries: 0 }, { + fetchImpl: (async (url, init) => { + calls.push({ url: String(url), init }); + return new Response(JSON.stringify({ status: "pending", reason: "source_episode_closed" }), { status: 200, headers: { "content-type": "application/json" } }); + }) as typeof fetch + }); + const input = { + sourceTurn: { source: "codex", profileId: "default", conversationId: "native-session", turnId: "native-turn", startedAt: "2099-01-01T00:00:00.000Z", completedAt: "2099-01-01T00:01:00.000Z", completionEvidence: "final_answer:native-turn" }, + channel: "agent_source_scan" as const, query: "Run tests", answer: "Tests passed", toolCalls: [{ id: "call-a", name: "test", input: "npm test", output: "passed" }] + }; + expect(await client.completeSourceTurn(input, { userId: "fixture-user" })).toEqual({ status: "pending", reason: "source_episode_closed" }); + expect(calls).toHaveLength(1); + expect(calls[0]?.url).toBe("http://memory.test/api/v1/source-turns/complete"); + expect(JSON.parse(String(calls[0]?.init?.body))).toEqual(input); + expect(new Headers(calls[0]?.init?.headers).get("x-memmy-user-id")).toBe("fixture-user"); + }); it("only defines path templates for the final memory HTTP APIs", () => { expect(Object.values(MEMORY_LAYER_PATHS)).toEqual([ "/api/v1/health", @@ -22,6 +40,7 @@ describe("HttpMemoryClient", () => { "/api/v1/sessions/:sessionId/close", "/api/v1/turns/start", "/api/v1/turns/:turnId/complete", + "/api/v1/source-turns/complete", "/api/v1/memory/search", "/api/v1/memory/add", "/api/v1/memory/:id", diff --git a/App/backend/src/adapters/outbound/memory-client/tests/mock-memory-client.test.ts b/App/backend/src/adapters/outbound/memory-client/tests/mock-memory-client.test.ts index f9adf1f02..7e8ea06ed 100644 --- a/App/backend/src/adapters/outbound/memory-client/tests/mock-memory-client.test.ts +++ b/App/backend/src/adapters/outbound/memory-client/tests/mock-memory-client.test.ts @@ -76,6 +76,7 @@ describe("createMockMemoryClient", () => { "addMemory", "closeSession", "completeTurn", + "completeSourceTurn", "deleteMemory", "enqueueImportSummaries", "getMemory", diff --git a/App/backend/src/adapters/outbound/memory-client/types.ts b/App/backend/src/adapters/outbound/memory-client/types.ts index 40040dd8a..8467a6819 100644 --- a/App/backend/src/adapters/outbound/memory-client/types.ts +++ b/App/backend/src/adapters/outbound/memory-client/types.ts @@ -9,6 +9,8 @@ import type { DeletePanelTaskOutput, CompleteTurnInput, CompleteTurnOutput, + SourceTurnCompleteInput, + SourceTurnCompleteOutput, EnqueueImportSummariesOutput, GetMemoryOutput, MemoryApiLogsInput, @@ -52,6 +54,8 @@ export interface MemoryClient { startTurn(input: StartTurnInput, context?: MemoryRequestContext): Promise; completeTurn(input: CompleteTurnInput & { turnId: string }, context?: MemoryRequestContext): Promise; + completeSourceTurn(input: SourceTurnCompleteInput, context?: MemoryRequestContext): Promise; + search(input: SearchInput, context?: MemoryRequestContext): Promise; addMemory(input: AddMemoryInput, context?: MemoryRequestContext): Promise; getMemory(input: { memoryId: string }, context?: MemoryRequestContext): Promise; diff --git a/App/backend/src/adapters/outbound/skill-writer/codex/tests/target.test.ts b/App/backend/src/adapters/outbound/skill-writer/codex/tests/target.test.ts index 3b3300a15..bc31cdb17 100644 --- a/App/backend/src/adapters/outbound/skill-writer/codex/tests/target.test.ts +++ b/App/backend/src/adapters/outbound/skill-writer/codex/tests/target.test.ts @@ -273,7 +273,7 @@ describe("codex skill target", () => { } }); - it("uses turn.complete as the only write phase for a completed Codex turn", async () => { + it("uses native source completion as the only write phase for a completed Codex turn", async () => { const { rootDirectory, memmyConfigPath } = createFixture(); const requests: Array<{ body: Record; path: string }> = []; const server = createServer(async (request: IncomingMessage, response: ServerResponse) => { @@ -296,8 +296,8 @@ describe("codex skill target", () => { }); return; } - if (request.method === "POST" && url.pathname === "/api/v1/turns/turn-stop-1/complete") { - writeJsonResponse(response, 200, { turnId: "turn-stop-1", l1MemoryId: "trace_1" }); + if (request.method === "POST" && url.pathname === "/api/v1/source-turns/complete") { + writeJsonResponse(response, 200, { status: "stored", result: { turnId: "turn-stop-1", l1MemoryId: "trace_1" } }); return; } writeJsonResponse(response, 404, {}); @@ -331,10 +331,18 @@ describe("codex skill target", () => { } }); + const transcriptPath = join(rootDirectory, "rollout-stop.jsonl"); + writeFileSync(transcriptPath, [ + { type: "session_meta", payload: { id: "codex-session-1", cwd: "/tmp/memmy-project" } }, + { type: "event_msg", payload: { type: "task_started", turn_id: "turn-stop-1" } }, + { type: "response_item", payload: { type: "message", role: "user", content: [{ text: "请继续完成数据分析报告" }] } }, + { type: "response_item", payload: { type: "message", role: "assistant", phase: "final_answer", content: [{ text: "已经完成数据分析报告" }] } } + ].map(record => JSON.stringify({ ...record, timestamp: "2026-09-09T10:00:00.000Z" })).join("\n") + "\n"); const run = await runNodeHook( hookScriptPath, JSON.stringify({ hook_event_name: "Stop", + transcript_path: transcriptPath, session_id: "codex-session-1", turn_id: "turn-stop-1", cwd: "/tmp/memmy-project", @@ -349,9 +357,7 @@ describe("codex skill target", () => { "/api/v1/health", "/api/v1/sessions/open", "/api/v1/turns/start", - "/api/v1/health", - "/api/v1/sessions/open", - "/api/v1/turns/turn-stop-1/complete" + "/api/v1/source-turns/complete" ]); expect(requests[1]?.body).toMatchObject({ sessionId: "codex-memory-codex-session-1", @@ -365,9 +371,10 @@ describe("codex skill target", () => { turnId: "turn-stop-1", query: "请继续完成数据分析报告" }); - expect(requests[5]?.body).toMatchObject({ + expect(requests[3]?.body).toMatchObject({ adapterId: "memmy-codex-hook", - requestId: expect.stringMatching(/^codex-complete:turn-stop-1:/u), + channel: "hook", + sourceTurn: expect.objectContaining({ conversationId: "codex-session-1", turnId: "turn-stop-1" }), sessionId: "memmy-session-1", query: "请继续完成数据分析报告", answer: "已经完成数据分析报告", @@ -375,7 +382,7 @@ describe("codex skill target", () => { source: "codex", sourceMemoryIds: ["memory-1"] }); - expect(requests[5]?.body).not.toHaveProperty("episodeId"); + expect(requests[3]?.body).not.toHaveProperty("episodeId"); } finally { await close(server); } diff --git a/App/backend/src/adapters/outbound/skill-writer/templates/memmy-resume-hook.ts b/App/backend/src/adapters/outbound/skill-writer/templates/memmy-resume-hook.ts index 716677ef6..a3d38ee8a 100644 --- a/App/backend/src/adapters/outbound/skill-writer/templates/memmy-resume-hook.ts +++ b/App/backend/src/adapters/outbound/skill-writer/templates/memmy-resume-hook.ts @@ -16,6 +16,8 @@ import { join } from "node:path"; import { closeRuntimeSession, completeRuntimeTurn, + completeSourceTurn, + readCodexSourceTurn, loadRuntimeL3, notifyRuntimeBoundary, openRuntimeSession, @@ -57,7 +59,8 @@ async function main() { if (isStopEvent(payload)) { try { await captureCompletedTurn(payload); - } catch { + } catch (error) { + reportCaptureFailure("request_failed", error, payload); // Memory capture must not interrupt host turn completion. } writeStopOutput(); @@ -88,7 +91,8 @@ async function main() { try { const started = await startCapturedTurn(payload, prompt); writeTurnStartOutput(started); - } catch { + } catch (error) { + reportCaptureFailure("start_failed", error, payload); writeAllowOutput(); } return; @@ -223,6 +227,10 @@ function isAgentResponseEvent(payload) { } async function captureCompletedTurn(payload) { + if (MODE === "codex") { + await captureCodexSourceTurn(payload); + return; + } const pending = await readTurnState(payload); const status = completedTurnStatus(payload); if (status === "cancelled") { @@ -264,6 +272,59 @@ async function captureCompletedTurn(payload) { await clearTurnState(payload); } +async function captureCodexSourceTurn(payload) { + const status = completedTurnStatus(payload); + if (status === "cancelled") { + await clearTurnState(payload); + return; + } + const transcriptPath = normalizeText(payload.transcript_path || payload.transcriptPath); + if (!transcriptPath) { + reportCaptureFailure("transcript_unavailable", undefined, payload); + return; + } + const pending = await readTurnState(payload); + const expectedTurnId = platformTurnId(payload); + const expectedConversationId = normalizeText(payload.session_id || payload.sessionId || payload.conversation_id || payload.conversationId || payload.thread_id || payload.threadId); + const parsed = await readCodexSourceTurn(transcriptPath, { + turnId: expectedTurnId || undefined, + conversationId: expectedConversationId || undefined, + stop: status === "succeeded" + }); + if (!parsed.turn) { + reportCaptureFailure(parsed.reason || "identity_unresolved", undefined, payload); + return; + } + if (status === "failed" && parsed.turn.status !== "failed") { + reportCaptureFailure("turn_status_unresolved", undefined, payload); + return; + } + if (isResumeCommand(parsed.turn.query)) { + await clearTurnState(payload); + return; + } + const result = await completeSourceTurn({ + configUrl: CONFIG_URL, + turn: parsed.turn, + sessionId: normalizeText(pending && pending.sessionId) || undefined, + sourceMemoryIds: Array.isArray(pending && pending.sourceMemoryIds) ? pending.sourceMemoryIds : undefined + }); + if (result.status === "stored" || result.status === "existing" || result.status === "rejected") { + await clearTurnState(payload); + return; + } + reportCaptureFailure(normalizeText(result.reason) || normalizeText(result.status) || "unexpected_response", undefined, payload); +} + +function reportCaptureFailure(reason, error, payload = {}) { + process.stderr.write(JSON.stringify({ + event: "memmy.hook.capture_failed", source: SOURCE, reason, + sourceSessionId: sessionStateKey(payload), sourceTurnId: platformTurnId(payload) || undefined, + transcriptPath: normalizeText(payload.transcript_path || payload.transcriptPath) || undefined, + error: error ? formatError(error) : undefined + }) + "\n"); +} + async function startCapturedTurn(payload, prompt) { const query = sanitizeCaptureText(prompt); if (!query) { diff --git a/App/backend/src/adapters/outbound/skill-writer/workspace-bridge/runtime.test.ts b/App/backend/src/adapters/outbound/skill-writer/workspace-bridge/runtime.test.ts index b8bc695bd..604c72e99 100644 --- a/App/backend/src/adapters/outbound/skill-writer/workspace-bridge/runtime.test.ts +++ b/App/backend/src/adapters/outbound/skill-writer/workspace-bridge/runtime.test.ts @@ -5,7 +5,9 @@ import { join } from "node:path"; import { pathToFileURL } from "node:url"; import { afterEach, describe, expect, it } from "vitest"; import { loadMemmyWorkspaceBridgeRuntimeAsset } from "./runtime-loader.js"; +import type { SourceTurn } from "@memmy/agent-source-core"; import { + completeSourceTurn, notifyRuntimeBoundary, openRuntimeSession, readRuntimeConfig, @@ -48,6 +50,47 @@ describe("Memory lifecycle runtime", () => { }); }); + it.each([undefined, "pending-session"])("submits the pinned owner with pending Session %s across account changes", async (sessionId) => { + const fixture = createFixture(); + const requests: Array<{ path: string; body: Record }> = []; + const server = createServer(async (request, response) => { + requests.push({ path: request.url ?? "", body: await requestBody(request) }); + return json(response, 200, { status: "stored" }); + }); + const endpoint = await listen(server); + const configUrl = runtimeConfig(fixture, endpoint); + const turn: SourceTurn = { + source: "codex", conversationId: "native-conversation", turnId: "native-turn", + startedAt: "2026-09-09T10:00:00.000Z", completedAt: "2026-09-09T10:00:01.000Z", + sequence: 1, completionEvidence: "final_answer:native-turn", query: "Fix parser", answer: "Fixed", + status: "succeeded", toolCalls: [], toolResults: [], workspacePath: fixture, + }; + try { + await completeSourceTurn({ configUrl, turn, sessionId, profileId: "work", sourceMemoryIds: ["recalled-1"] }); + writeFileSync(join(fixture, "missing.yaml"), [ + "app:", " userId: switched-app-owner", "memmyMemory:", " userId: switched-memory-owner", "", + ].join("\n")); + await completeSourceTurn({ configUrl, turn, sessionId, profileId: "work", sourceMemoryIds: ["recalled-1"] }); + + expect(requests).toHaveLength(2); + expect(requests[1]).toEqual(requests[0]); + for (const request of requests) { + expect(request.path).toBe("/api/v1/source-turns/complete"); + expect(request.body.namespace).toEqual({ + source: "codex", profileId: "work", userId: "installed-owner", sessionKey: "native-conversation", + }); + expect(request.body).toMatchObject({ + sourceTurn: { source: "codex", profileId: "work", conversationId: "native-conversation" }, + sourceMemoryIds: ["recalled-1"], workspacePath: fixture, + }); + if (sessionId) expect(request.body.sessionId).toBe(sessionId); + else expect(request.body).not.toHaveProperty("sessionId"); + } + } finally { + await close(server); + } + }); + it("opens a v2 project Session with only canonical workspace identity", async () => { const fixture = createFixture(); const requests: Array<{ path: string; body: Record }> = []; diff --git a/App/backend/src/adapters/outbound/skill-writer/workspace-bridge/runtime.ts b/App/backend/src/adapters/outbound/skill-writer/workspace-bridge/runtime.ts index 0a8e13b78..8306cd2dc 100644 --- a/App/backend/src/adapters/outbound/skill-writer/workspace-bridge/runtime.ts +++ b/App/backend/src/adapters/outbound/skill-writer/workspace-bridge/runtime.ts @@ -1,3 +1,5 @@ +import { buildSourceTurnRequest, type SourceTurn } from "@memmy/agent-source-core"; +export { readCodexSourceTurn } from "@memmy/agent-source-core"; import { createHash, randomUUID } from "node:crypto"; import { lstat, readFile, realpath, stat } from "node:fs/promises"; import { homedir } from "node:os"; @@ -232,6 +234,31 @@ export async function completeRuntimeTurn( await client.post(`/api/v1/turns/${encodeURIComponent(input.turnId)}/complete`, compact(body)); } +/** Submit a completed native turn without opening a new runtime Session before deduplication. */ +export async function completeSourceTurn(input: { + configUrl: URL; + turn: SourceTurn; + sessionId?: string; + sourceMemoryIds?: string[]; + profileId?: string; +}): Promise> { + const config = await readRuntimeConfig(input.configUrl, true); + const client = new RuntimeHttpClient(config); + const profileId = input.profileId || "default"; + return objectValue(await client.post("/api/v1/source-turns/complete", compact({ + ...buildSourceTurnRequest(input.turn, "hook", profileId), + namespace: { + source: input.turn.source, + profileId, + userId: config.userId, + sessionKey: input.turn.conversationId, + }, + sessionId: input.sessionId, + sourceMemoryIds: input.sourceMemoryIds, + adapterId: "memmy-codex-hook", + }))); +} + class RuntimeHttpClient { constructor(private readonly config: RuntimeConfig) {} diff --git a/App/backend/src/infrastructure/agent-source-scan-store/index.ts b/App/backend/src/infrastructure/agent-source-scan-store/index.ts index 5a44dfef2..574cd5301 100644 --- a/App/backend/src/infrastructure/agent-source-scan-store/index.ts +++ b/App/backend/src/infrastructure/agent-source-scan-store/index.ts @@ -96,13 +96,22 @@ export function openAppAgentSourceScanStore(path: string, job: AppScanJobMeta): } let ordinal = Number((db.prepare("SELECT COALESCE(MAX(ordinal), -1) AS value FROM staged_messages WHERE job_id=?").get(job.jobId) as { value: number }).value) + 1; const insert = db.prepare("INSERT OR IGNORE INTO staged_messages(job_id,source_id,conversation_id,message_id,role,content,created_at,workspace_path,git_root,raw_meta_json,ordinal) VALUES(?,?,?,?,?,?,?,?,?,?,?)"); + const refreshCodex = db.prepare(`UPDATE staged_messages + SET conversation_id=?,role=?,content=?,created_at=?,workspace_path=?,git_root=?,raw_meta_json=? + WHERE job_id=? AND source_id=? AND message_id=?`); const store: AppAgentSourceScanStore = { path, stage(message) { const bytes = Buffer.byteLength(JSON.stringify(message)); if (bytes > MAX_RECORD_BYTES) throw new Error(`scan record exceeds 64 MiB limit (${bytes} bytes)`); - const result = insert.run(job.jobId, message.sourceId, message.conversationId, message.messageId, message.role, message.content, message.createdAt, message.workspacePath, message.gitRoot, JSON.stringify(message.rawMeta), ordinal++); - return Number(result.changes) > 0; + const rawMetaJson = JSON.stringify(message.rawMeta); + const inserted = Number(insert.run(job.jobId, message.sourceId, message.conversationId, message.messageId, message.role, message.content, message.createdAt, message.workspacePath, message.gitRoot, rawMetaJson, ordinal++).changes) > 0; + if (!inserted && message.sourceId === "codex" && typeof message.rawMeta.sourceTurnState === "string") { + // Retrying a staged turn can add native identity or completion evidence + // to an existing message. Preserve its ordinal and the insertion count. + refreshCodex.run(message.conversationId, message.role, message.content, message.createdAt, message.workspacePath, message.gitRoot, rawMetaJson, job.jobId, message.sourceId, message.messageId); + } + return inserted; }, stageBatch(messages) { let inserted = 0; diff --git a/App/backend/src/infrastructure/agent-source-scan-store/tests/repository.test.ts b/App/backend/src/infrastructure/agent-source-scan-store/tests/repository.test.ts index 1c3076700..65fb68bb7 100644 --- a/App/backend/src/infrastructure/agent-source-scan-store/tests/repository.test.ts +++ b/App/backend/src/infrastructure/agent-source-scan-store/tests/repository.test.ts @@ -8,6 +8,45 @@ let directory: string | undefined; afterEach(() => { if (directory) rmSync(directory, { recursive: true, force: true }); directory = undefined; }); describe("durable scan store", () => { + it("refreshes unresolved Codex rows in the same job without counting them as new messages", () => { + directory = mkdtempSync(join(tmpdir(), "memmy-codex-scan-retry-")); + const path = join(directory, "job.sqlite"); + const job = { jobId: "job", sourceId: "codex", mode: "incremental", phase: "failed", createdAt: "2026-09-09", updatedAt: "2026-09-09" }; + let store = openAppAgentSourceScanStore(path, job); + const pending = { messageId: "rollout:000000000009", sourceId: "codex", conversationId: "fallback-conversation", role: "assistant" as const, content: "Draft", createdAt: "2026-09-09T00:00:00Z", workspacePath: null, gitRoot: null, rawMeta: { sourceTurnState: "identity_unresolved", sourceTurnReason: "identity_unresolved" } }; + expect(store.stage(pending)).toBe(true); + const stagedOrdinal = [...store.messages("codex")][0]!.ordinal; + store.saveResult({ sourceId: "codex", conversationId: pending.conversationId, error: "identity_unresolved" }); + store.close(); + store = openAppAgentSourceScanStore(path, job); + const completed = { ...pending, conversationId: "native-conversation", content: "Final answer", workspacePath: "/tmp/project", rawMeta: { sourceTurnState: "complete", sourceTurnId: "turn-native", sourceTurn: { turnId: "turn-native", completionEvidence: "task_complete:turn-native" } } }; + const next = { ...completed, messageId: "rollout:000000000010", content: "Next answer" }; + expect(store.stageBatch([completed, next])).toBe(1); + expect(store.stage(completed)).toBe(false); + expect(store.count("codex")).toBe(2); + const rows = [...store.messages("codex")]; + expect(rows[0]).toMatchObject({ ...completed, ordinal: stagedOrdinal }); + expect(rows[0]!.rawMeta).not.toHaveProperty("sourceTurnReason"); + expect([...store.results("codex")]).toEqual([{ sourceId: "codex", conversationId: pending.conversationId, error: "identity_unresolved" }]); + store.close(); + store = openAppAgentSourceScanStore(path, job); + expect([...store.messages("codex")][0]).toMatchObject(completed); + store.close(); + }); + + it("keeps existing non-Codex staged rows and unrelated rows unchanged", () => { + directory = mkdtempSync(join(tmpdir(), "memmy-scan-legacy-dedup-")); + const store = openAppAgentSourceScanStore(join(directory, "job.sqlite"), { jobId: "job", sourceId: "all", mode: "full", phase: "stage", createdAt: "2026-09-09", updatedAt: "2026-09-09" }); + const message = { messageId: "shared-id", sourceId: "fixture", conversationId: "conversation", role: "user" as const, content: "Original", createdAt: "2026-09-09T00:00:00Z", workspacePath: null, gitRoot: null, rawMeta: {} }; + const codex = { ...message, sourceId: "codex", rawMeta: { sourceTurnState: "turn_incomplete" } }; + expect(store.stageBatch([message, codex])).toBe(2); + expect(store.stage({ ...message, content: "Revised" })).toBe(false); + expect(store.stage({ ...codex, content: "Complete", rawMeta: { sourceTurnState: "complete" } })).toBe(false); + expect([...store.messages("fixture")][0]!.content).toBe("Original"); + expect([...store.messages("codex")][0]!.content).toBe("Complete"); + expect(store.count()).toBe(2); + store.close(); + }); it("deduplicates staged rows and reads keyset pages", () => { directory = mkdtempSync(join(tmpdir(), "memmy-scan-store-")); const store = openAppAgentSourceScanStore(join(directory, "job.sqlite"), { jobId: "job", sourceId: "fixture", mode: "full", phase: "stage", createdAt: "2026-01-01", updatedAt: "2026-01-01" }); diff --git a/App/backend/src/services/agent-source-service.ts b/App/backend/src/services/agent-source-service.ts index da0e5971a..97baa71e9 100644 --- a/App/backend/src/services/agent-source-service.ts +++ b/App/backend/src/services/agent-source-service.ts @@ -46,6 +46,9 @@ import { } from "./managed-agent-history.js"; import { orderedTurns, + sourceTurnFromMessages, + sourceTurnFailureReason, + buildSourceTurnRequest, renderTurnClipped, stableTurnIdentity, isCompleteTurn, @@ -114,7 +117,7 @@ export interface CreateAgentSourceServiceOptions { sourceRegistry: SourceRegistry; agentSourceRepository: AgentSourceRepository; ingestionService: IngestionService; - memoryClient: Pick; + memoryClient: Pick; skillDistributionService: SkillDistributionService; agentSourceAnalytics?: AgentSourceLifecycleAnalytics; getScanPermission?: () => Promise; @@ -139,7 +142,7 @@ export function createAgentSourceService(options: CreateAgentSourceServiceOption if (!scanOptions.scanJobId && !options.scanStoreDirectory) { const collected = await this.collectAll(scanOptions); const results = await this.ingestCollected(collected, scanOptions); - const failures = await this.processImportSummaries(results.flatMap((result) => result.memoryIds ?? []), { ...scanOptions, progressSourceId: "all" }); + const failures = await this.processImportSummaries(results.filter((result) => result.sourceId !== "codex").flatMap((result) => result.memoryIds ?? []), { ...scanOptions, progressSourceId: "all" }); appendProcessingFailuresToResults(results, failures); return results; } @@ -177,7 +180,7 @@ export function createAgentSourceService(options: CreateAgentSourceServiceOption if (!scanOptions.scanJobId && !options.scanStoreDirectory) { const collected = await this.collectOne(sourceId, scanOptions); const result = await ingestCollectedSource(options, collected, scanOptions, now); - const failures = await processPendingImportSummaries(options, result.memoryIds ?? [], { ...scanOptions, progressSourceId: sourceId }); + const failures = sourceId === "codex" ? [] : await processPendingImportSummaries(options, result.memoryIds ?? [], { ...scanOptions, progressSourceId: sourceId }); appendProcessingFailures(result, failures); return result; } @@ -738,7 +741,7 @@ async function preparePersistentSource(options: CreateAgentSourceServiceOptions, let first = true; let latest: ConversationMessage | null = null; const flushTurn = () => { - if (!currentTurn.length || !isCompleteTurn(currentTurn)) return; + if (!currentTurn.length || (sourceId !== "codex" && !isCompleteTurn(currentTurn))) return; const firstMessage = currentTurn[0]!; const lastMessage = currentTurn[currentTurn.length - 1]!; const turn = { sourceId, conversationId: firstMessage.conversationId, turnIndex, messages: currentTurn }; @@ -781,14 +784,14 @@ async function preparePersistentSource(options: CreateAgentSourceServiceOptions, hash.update("["); first = true; } - if (message.role === "user" && currentTurn.length > 0) { + if (currentTurn.length > 0 && (sourceId === "codex" ? message.rawMeta.sourceTurnId !== currentTurn[0]?.rawMeta.sourceTurnId : message.role === "user")) { flushTurn(); currentTurn = []; } currentTurn.push(message); if (!first) hash.update(","); first = false; - hash.update(JSON.stringify({ messageId: message.messageId, role: message.role, content: message.content, createdAt: message.createdAt, toolName: hashMetaString(message, "toolName") ?? hashMetaString(message, "hermesToolName"), toolCallId: hashMetaString(message, "toolCallId") ?? hashMetaString(message, "hermesToolCallId") })); + hash.update(JSON.stringify({ messageId: message.messageId, role: message.role, content: message.content, createdAt: message.createdAt, toolName: hashMetaString(message, "toolName") ?? hashMetaString(message, "hermesToolName"), toolCallId: hashMetaString(message, "toolCallId") ?? hashMetaString(message, "hermesToolCallId"), ...(sourceId === "codex" ? { sourceTurn: message.rawMeta } : {}) })); latest = message; } const last = page[page.length - 1]!; @@ -859,6 +862,32 @@ async function ingestPersistentSource( if (conversationMeta?.selected === false) continue; const selectedTurn = store.getTurnMeta(sourceId, turn.conversationId, stableTurnIdentity(turn)); if (selectedTurn && !selectedTurn.selected) continue; + if (sourceId === "codex") { + try { + const sourceTurn = sourceTurnFromMessages(turn.messages); + if (!sourceTurn) throw new Error(sourceTurnFailureReason(turn.messages)); + const result = await options.memoryClient.completeSourceTurn(buildSourceTurnRequest(sourceTurn, "agent_source_scan")); + if (result.status === "pending" || result.status === "conflict") throw new Error(result.reason ?? result.status); + const ids = result.result?.l1MemoryIds ?? []; + if (result.status === "stored") { + memoryIdCount += ids.length; + memoryIds.push(...ids.slice(0, Math.max(0, 1000 - memoryIds.length))); + } else { + deduped += turn.messages.length; + } + // The Memory transaction already registered normal capture jobs. Do not enqueue import summaries. + if (ids.length === 0) store.saveResult({ sourceId, conversationId: turn.conversationId }); + for (const memoryId of ids) store.saveResult({ sourceId, conversationId: turn.conversationId, memoryId }); + } catch (error) { + activeConversationFailed = true; + const reason = error instanceof Error ? error.message : "native turn ingestion failed"; + errorCount += 1; + if (errors.length < 1000) errors.push({ conversationId: turn.conversationId, reason }); + store.saveResult({ sourceId, conversationId: turn.conversationId, error: reason }); + } + emitProgress(scanOptions, { sourceId, phase: "add", current: memoryIdCount + deduped, total: store.count(sourceId), message: "Capturing conversation turns" }); + continue; + } let turnSucceeded = true; // One turn is one memory. Splitting an agentic turn fans a single exchange // out into hundreds of near-empty tool-call fragments, so an oversized turn @@ -1294,7 +1323,8 @@ function conversationContentHash(messages: readonly ConversationMessage[]): stri content: message.content, createdAt: message.createdAt, toolName: conversationMetaString(message, "toolName") ?? conversationMetaString(message, "hermesToolName"), - toolCallId: conversationMetaString(message, "toolCallId") ?? conversationMetaString(message, "hermesToolCallId") + toolCallId: conversationMetaString(message, "toolCallId") ?? conversationMetaString(message, "hermesToolCallId"), + ...(message.sourceId === "codex" ? { sourceTurn: message.rawMeta } : {}) })); return createHash("sha256").update(JSON.stringify(content)).digest("hex"); } @@ -1381,6 +1411,14 @@ function buildConversationMemoryUnits(sourceId: string, messages: readonly Conve let current: ConversationMessage[] = []; for (const message of messages) { + if (sourceId === "codex") { + if (current.length > 0 && current[0]?.rawMeta.sourceTurnId !== message.rawMeta.sourceTurnId) { + pushCompleteMemoryUnit(sourceId, current, units); + current = []; + } + current.push(message); + continue; + } if (message.role === "user") { pushCompleteMemoryUnit(sourceId, current, units); current = [message]; @@ -1401,7 +1439,7 @@ function pushCompleteMemoryUnit( messages: readonly ConversationMessage[], units: SourceMemoryUnit[] ): void { - if (!isCompleteMemoryTurn(messages)) { + if (messages.length === 0 || (sourceId !== "codex" && !isCompleteMemoryTurn(messages))) { return; } const userMessage = messages[0]!; diff --git a/App/backend/src/services/ingestion-service.ts b/App/backend/src/services/ingestion-service.ts index bc27f43fb..3355cb343 100644 --- a/App/backend/src/services/ingestion-service.ts +++ b/App/backend/src/services/ingestion-service.ts @@ -1,5 +1,6 @@ /** Ingestion service module. */ import { createHash } from "node:crypto"; +import { orderedTurns, sourceTurnFromMessages, sourceTurnFailureReason, buildSourceTurnRequest } from "@memmy/agent-source-core"; import { setImmediate as yieldToEventLoop } from "node:timers/promises"; import type { ConversationMessage } from "../adapters/outbound/agent-source/types.js"; import type { MemoryClient } from "../adapters/outbound/memory-client/index.js"; @@ -64,7 +65,7 @@ export interface IngestionStats { /** Contract for create ingestion service options. */ export interface CreateIngestionServiceOptions { - memoryClient: Pick; + memoryClient: Pick; agentSourceRepository: Pick; memoryAddAnalytics?: Pick< MemoryDesktopAddAnalytics, @@ -163,6 +164,10 @@ async function processConversation( ctx: IngestionContext, stats: IngestionStats ): Promise { + if (ctx.sourceId === "codex") { + await processNativeConversation(options, messages, ctx, stats); + return; + } let processedTurns = 0; let incomplete = false; let failed = false; @@ -274,6 +279,52 @@ async function processConversation( else stats.completedConversationIds.push(conversationId); } +async function processNativeConversation( + options: CreateIngestionServiceOptions, + messages: readonly ConversationMessage[], + ctx: IngestionContext, + stats: IngestionStats +): Promise { + let failed = false; + const values = (async function* () { yield* messages; })(); + for await (const turn of orderedTurns(values)) { + ctx.signal?.throwIfAborted(); + try { + const sourceTurn = sourceTurnFromMessages(turn.messages); + if (!sourceTurn) { + throw new Error(sourceTurnFailureReason(turn.messages)); + } + const result = await options.memoryClient.completeSourceTurn(buildSourceTurnRequest(sourceTurn, "agent_source_scan")); + if (result.status === "pending" || result.status === "conflict") { + throw new Error(result.reason ?? result.status); + } + if (result.status === "stored") { + const ids = result.result?.l1MemoryIds ?? []; + stats.written += turn.messages.length; + stats.writtenMemories += ids.length; + stats.memoryIds.push(...ids); + } else { + stats.deduped += turn.messages.length; + if (result.status === "existing") stats.dedupedMemories += 1; + } + for (const message of turn.messages) { + options.agentSourceRepository.markSeen(createDedupKey(ctx.sourceId, message.messageId), ctx.sourceId); + } + } catch (error) { + failed = true; + stats.failed += turn.messages.length; + stats.failedMemories += 1; + stats.errors.push({ conversationId: turn.conversationId, reason: error instanceof Error ? error.message : "native turn ingestion failed" }); + } + emitIngestionProgress(ctx, stats); + } + const conversationId = messages[0]?.conversationId; + if (conversationId) { + if (failed) stats.failedConversationIds.push(conversationId); + else stats.completedConversationIds.push(conversationId); + } +} + function emitIngestionProgress(ctx: IngestionContext, stats: IngestionStats): void { ctx.onProgress?.({ sourceId: ctx.sourceId, diff --git a/App/backend/src/services/tests/ingestion-service.test.ts b/App/backend/src/services/tests/ingestion-service.test.ts index 3a6d9b62f..7d43ec94f 100644 --- a/App/backend/src/services/tests/ingestion-service.test.ts +++ b/App/backend/src/services/tests/ingestion-service.test.ts @@ -12,6 +12,74 @@ import { import type { AgentSourceRepository } from "../../infrastructure/agent-source-store/index.js"; import type { ConversationMessage } from "../../adapters/outbound/agent-source/types.js"; +describe("native Codex ingestion", () => { + function nativeMessages(complete = true): ConversationMessage[] { + const sourceTurn = { + source: "codex", conversationId: "native-conversation", turnId: "native-turn", sequence: 1, + startedAt: "2026-09-09T10:00:00.000Z", completedAt: "2026-09-09T10:01:00.000Z", + completionEvidence: "task_complete:native-turn", query: "Run tests", answer: "Tests passed", status: "succeeded", + toolCalls: [{ id: "test-call", name: "test", input: "npm test", output: "passed", success: true }], + toolResults: [{ id: "test-call", output: "passed", success: true }] + }; + return ["user", "assistant"].map((role, index) => ({ + messageId: `native-${index}`, sourceId: "codex", conversationId: "native-conversation", + role: role as "user" | "assistant", content: index === 0 ? sourceTurn.query : sourceTurn.answer, + createdAt: index === 0 ? sourceTurn.startedAt : sourceTurn.completedAt, + workspacePath: null, gitRoot: null, + rawMeta: { sourceTurnId: "native-turn", sourceTurnState: complete ? "complete" : "turn_incomplete", + sourceTurnReason: complete ? undefined : "turn_incomplete", ...(complete && index === 1 ? { sourceTurn } : {}) } + })); + } + + it("submits the structured native turn once and never creates an import-summary memory", async () => { + const addMemory = vi.fn(); + const completeSourceTurn = vi.fn().mockResolvedValue({ status: "stored", result: { l1MemoryIds: ["l1-native"] } }); + const markSeen = vi.fn(); + const stats = await createService({ addMemory, completeSourceTurn }, { markSeen }).ingest(toAsyncIterable(nativeMessages()), { sourceId: "codex" }); + expect(addMemory).not.toHaveBeenCalled(); + expect(completeSourceTurn).toHaveBeenCalledOnce(); + expect(completeSourceTurn).toHaveBeenCalledWith(expect.objectContaining({ + channel: "agent_source_scan", sourceTurn: expect.objectContaining({ conversationId: "native-conversation", turnId: "native-turn" }), + toolCalls: [expect.objectContaining({ id: "test-call", input: "npm test", output: "passed" })] + })); + expect(stats.memoryIds).toEqual(["l1-native"]); + expect(markSeen).toHaveBeenCalledTimes(2); + }); + + it.each(["pending", "conflict"])("retains %s turns for retry without marking seen or completing the conversation", async status => { + const markSeen = vi.fn(); + const addMemory = vi.fn(); + const completeSourceTurn = vi.fn().mockResolvedValue({ status, reason: "episode_unresolved" }); + const stats = await createService({ addMemory, completeSourceTurn }, { markSeen }).ingest(toAsyncIterable(nativeMessages()), { sourceId: "codex" }); + expect(markSeen).not.toHaveBeenCalled(); expect(addMemory).not.toHaveBeenCalled(); + expect(stats.completedConversationIds).toEqual([]); + expect(stats.failedConversationIds).toEqual(["native-conversation"]); + expect(stats.errors[0]?.reason).toContain("episode_unresolved"); + }); + + it("does not submit incomplete evidence, and retries after completion arrives on the same message IDs", async () => { + const completeSourceTurn = vi.fn().mockResolvedValue({ status: "existing", result: { l1MemoryIds: ["same-l1"] } }); + const addMemory = vi.fn(); const markSeen = vi.fn(); + const service = createService({ addMemory, completeSourceTurn }, { markSeen }); + const pending = await service.ingest(toAsyncIterable(nativeMessages(false)), { sourceId: "codex" }); + expect(completeSourceTurn).not.toHaveBeenCalled(); expect(markSeen).not.toHaveBeenCalled(); + expect(pending.errors[0]?.reason).toContain("turn_incomplete"); + const recovered = await service.ingest(toAsyncIterable(nativeMessages()), { sourceId: "codex" }); + expect(completeSourceTurn).toHaveBeenCalledOnce(); expect(markSeen).toHaveBeenCalledTimes(2); + expect(recovered.dedupedMemories).toBe(1); expect(recovered.writtenMemories).toBe(0); + expect(addMemory).not.toHaveBeenCalled(); + }); + + it("acknowledges rejected old turns without reporting a newly written memory", async () => { + const markSeen = vi.fn(); + const completeSourceTurn = vi.fn().mockResolvedValue({ status: "rejected", reason: "legacy_before_activation" }); + const stats = await createService({ completeSourceTurn }, { markSeen }).ingest(toAsyncIterable(nativeMessages()), { sourceId: "codex" }); + expect(stats.writtenMemories).toBe(0); expect(stats.memoryIds).toEqual([]); + expect(stats.completedConversationIds).toEqual(["native-conversation"]); + expect(markSeen).toHaveBeenCalledTimes(2); + }); +}); + describe("ingestion service", () => { it("imports each contiguous conversation as turn memories through memory add", async () => { const added: Array> = []; @@ -480,7 +548,7 @@ describe("ingestion service", () => { const stats = await service.ingest( toAsyncIterable([createMessage("conv-a", 1), createMessage("conv-a", 2)]), - { sourceId: "codex" } + { sourceId: "cursor" } ); expect(markSeen).toHaveBeenCalledTimes(2); diff --git a/App/backend/src/services/tests/native-persistent-scan.test.ts b/App/backend/src/services/tests/native-persistent-scan.test.ts new file mode 100644 index 000000000..7e5140694 --- /dev/null +++ b/App/backend/src/services/tests/native-persistent-scan.test.ts @@ -0,0 +1,101 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { readCodexRollout } from "@memmy/agent-source-core"; +import { createAgentSourceService } from "../agent-source-service.js"; +import { createIngestionService } from "../ingestion-service.js"; +import { createSourceRegistry } from "../../adapters/outbound/agent-source/source-registry.js"; +import { createCodexSourceAdapter } from "../../adapters/outbound/agent-source/codex/index.js"; +import { createAppStateStore, type AppStateStore } from "../../infrastructure/app-state-store/index.js"; +import { createMockMemoryClient } from "../../tests/support/mock-memory-client.js"; + +const roots: string[] = []; const stores: AppStateStore[] = []; +afterEach(() => { stores.splice(0).forEach(store => store.close()); roots.splice(0).forEach(root => rmSync(root, { recursive: true, force: true })); }); + +describe("persistent Codex scan", () => { + it("does not checkpoint a lost response; retries the same source turn without import splitting", async () => { + const root = mkdtempSync(join(tmpdir(), "native-backend-scan-")); roots.push(root); + const store = createAppStateStore({ databasePath: join(root, "app.sqlite") }); stores.push(store); + const repository = store.repositories.agentSources; + const file = join(root, "rollout.jsonl"); + const at = "2099-09-09T10:00:00.000Z"; + const event = (type: string, payload: Record) => ({ type, timestamp: at, payload }); + writeFileSync(file, [event("session_meta", { id: "source-session" }), event("event_msg", { type: "task_started", turn_id: "turn-1" }), + event("response_item", { type: "message", role: "user", content: [{ text: "Run tests" }] }), + event("response_item", { type: "custom_tool_call", call_id: "call-1", name: "test", input: "npm test" }), + event("response_item", { type: "custom_tool_call_output", call_id: "call-1", output: "passed ".repeat(4000) }), + event("response_item", { type: "message", role: "assistant", content: [{ text: "Tests passed" }] }), + event("event_msg", { type: "task_complete", turn_id: "turn-1" })].map(value => JSON.stringify(value)).join("\n") + "\n"); + const client = createMockMemoryClient(); + const addMemory = vi.spyOn(client, "addMemory"); const enqueue = vi.spyOn(client, "enqueueImportSummaries"); + const complete = vi.spyOn(client, "completeSourceTurn").mockRejectedValueOnce(new Error("response lost")).mockResolvedValue({ status: "existing", result: { + turnId: "turn-1", sessionId: "session", episodeId: "episode", rawTurnId: "raw", l1MemoryId: "same-l1", l1MemoryIds: ["same-l1"], closedEpisodeIds: [], scheduledEvolution: false, jobs: [], serverTime: at + } }); + const service = createAgentSourceService({ sourceRegistry: createSourceRegistry([{ descriptor: { sourceId: "codex", displayName: "Codex", builtin: true, dataPath: root }, + detect: async () => true, async *scan() { for await (const message of readCodexRollout(file)) yield { ...message, sourceId: "codex", workspacePath: null, gitRoot: null }; } }]), + memoryClient: client, agentSourceRepository: repository, ingestionService: createIngestionService({ memoryClient: client, agentSourceRepository: repository }), + skillDistributionService: { install: async () => undefined, uninstall: async () => undefined, installPlugin: async () => undefined, uninstallPlugin: async () => undefined }, + scanStoreDirectory: join(root, "scans") }); + const failed = await service.scanOne("codex", { scanJobId: "same-job", mode: "incremental" }); + expect(failed.errors[0]?.reason).toBe("response lost"); + expect(repository.getConversationCheckpoint("codex", "source-session")).toBeNull(); + expect(repository.getScanWatermark("codex")).toBeNull(); + const retried = await service.scanOne("codex", { scanJobId: "same-job", mode: "incremental" }); + expect(retried.errors).toEqual([]); expect(retried.memoryIdCount).toBe(0); + expect(complete).toHaveBeenCalledTimes(2); + expect(complete.mock.calls[0]?.[0]).toEqual(complete.mock.calls[1]?.[0]); + expect(complete.mock.calls[0]?.[0].toolCalls).toEqual([expect.objectContaining({ id: "call-1", input: "npm test", output: "passed ".repeat(4000) })]); + expect(repository.getConversationCheckpoint("codex", "source-session")).not.toBeNull(); + expect(addMemory).not.toHaveBeenCalled(); expect(enqueue).not.toHaveBeenCalled(); + }); +}); + + +describe("nonpersistent Codex scan window", () => { + it.each([undefined, 2])("keeps a complete new turn across since without reselecting old conversations (maxMessages=%s)", async maxMessages => { + const root = mkdtempSync(join(tmpdir(), "native-backend-window-")); roots.push(root); + const store = createAppStateStore({ databasePath: join(root, "app.sqlite") }); stores.push(store); + const repository = store.repositories.agentSources; + const writeTurn = (file: string, conversationId: string, start: string, end: string) => { + const event = (timestamp: string, type: string, payload: Record) => ({ timestamp, type, payload }); + writeFileSync(join(root, file), [ + event(start, "session_meta", { id: conversationId }), + event(start, "event_msg", { type: "task_started", turn_id: `${conversationId}-turn` }), + event(start, "response_item", { type: "message", role: "user", content: [{ text: "Inspect the complete source before changing it." }] }), + event(start, "response_item", { type: "function_call", call_id: "read-call", name: "read", arguments: { path: "source.ts" } }), + event(end, "response_item", { type: "function_call_output", call_id: "read-call", output: "Complete source contents" }), + event(end, "response_item", { type: "message", role: "assistant", content: [{ text: "I inspected the complete source and fixed it." }] }), + event(end, "event_msg", { type: "task_complete", turn_id: `${conversationId}-turn` }) + ].map(record => JSON.stringify(record)).join("\n") + "\n"); + }; + writeTurn("rollout-a-old.jsonl", "old-conversation", "2099-09-09T09:00:00.000Z", "2099-09-09T09:01:00.000Z"); + writeTurn("rollout-b-new.jsonl", "new-conversation", "2099-09-09T10:01:00.000Z", "2099-09-09T10:03:00.000Z"); + const client = createMockMemoryClient(); + const complete = vi.spyOn(client, "completeSourceTurn"); + const add = vi.spyOn(client, "addMemory"); + const service = createAgentSourceService({ + sourceRegistry: createSourceRegistry([createCodexSourceAdapter({ sessionsRoot: root })]), + memoryClient: client, agentSourceRepository: repository, + ingestionService: createIngestionService({ memoryClient: client, agentSourceRepository: repository }), + skillDistributionService: { install: async () => undefined, uninstall: async () => undefined, installPlugin: async () => undefined, uninstallPlugin: async () => undefined } + }); + const options = { mode: "incremental" as const, since: "2099-09-09T10:02:00.000Z", maxMessages }; + const collected = await service.collectOne("codex", options); + expect(collected.conversationIds).toEqual(["new-conversation"]); + expect(collected.messages).toHaveLength(5); + expect(collected.messages[0]?.content).toBe("Inspect the complete source before changing it."); + expect(collected.messages.at(-1)).toMatchObject({ role: "system", content: "Codex task_complete" }); + const result = await service.scanOne("codex", options); + expect(result.errors).toEqual([]); + expect(complete).toHaveBeenCalledOnce(); + expect(complete).toHaveBeenCalledWith(expect.objectContaining({ + sourceTurn: expect.objectContaining({ conversationId: "new-conversation", turnId: "new-conversation-turn", startedAt: "2099-09-09T10:01:00.000Z", completedAt: "2099-09-09T10:03:00.000Z" }), + query: "Inspect the complete source before changing it.", answer: "I inspected the complete source and fixed it.", + toolCalls: [{ id: "read-call", name: "read", input: { path: "source.ts" }, output: "Complete source contents" }], + toolResults: [{ id: "read-call", output: "Complete source contents" }] + })); + expect(add).not.toHaveBeenCalled(); + expect(repository.getConversationCheckpoint("codex", "old-conversation")).toBeNull(); + }); +}); diff --git a/App/backend/src/tests/support/mock-memory-client.ts b/App/backend/src/tests/support/mock-memory-client.ts index b65f84c68..870bc36a7 100644 --- a/App/backend/src/tests/support/mock-memory-client.ts +++ b/App/backend/src/tests/support/mock-memory-client.ts @@ -128,6 +128,27 @@ export function createMockMemoryClient(options: CreateMockMemoryClientOptions = }; }, + async completeSourceTurn(input) { + failIfNeeded(); + const l1MemoryId = randomUUID(); + return { + status: "stored", + result: { + turnId: input.sourceTurn.turnId, + sessionId: input.sessionId ?? randomUUID(), + episodeId: randomUUID(), + rawTurnId: randomUUID(), + l1MemoryId, + l1MemoryIds: [l1MemoryId], + closedEpisodeIds: [], + scheduledEvolution: false, + jobs: [], + ...nextChange(), + serverTime: now() + } + }; + }, + async search(input) { failIfNeeded(); const injectedContext = { diff --git a/Memory/src/agent-source/adapters/codex/adapter.ts b/Memory/src/agent-source/adapters/codex/adapter.ts index 62b5dcab9..fd11dfd00 100644 --- a/Memory/src/agent-source/adapters/codex/adapter.ts +++ b/Memory/src/agent-source/adapters/codex/adapter.ts @@ -102,7 +102,8 @@ function toConversationMessage( createdAt: rawMessage.createdAt, workspacePath, gitRoot, - rawMeta: Object.freeze({}) + ordinal: rawMessage.ordinal, + rawMeta: rawMessage.rawMeta }; } diff --git a/Memory/src/agent-source/adapters/codex/rollout-reader.ts b/Memory/src/agent-source/adapters/codex/rollout-reader.ts index 6dd193392..9b2de0524 100644 --- a/Memory/src/agent-source/adapters/codex/rollout-reader.ts +++ b/Memory/src/agent-source/adapters/codex/rollout-reader.ts @@ -1,208 +1 @@ -/** Rollout reader module. */ -import { basename } from "node:path"; -import { readJsonlObjects, type JsonObject } from "../jsonl-lines.js"; - -const MAX_TOOL_NAME_ENTRIES = 4096; - -export interface RawCodexMessage { - /** Message id. */ - messageId: string; - conversationId: string; - role: "user" | "assistant" | "tool" | "system"; - content: string; - createdAt: string; -} - -/** Rollout reader module. */ -export async function* readCodexRollout(filePath: string, signal?: AbortSignal): AsyncIterable { - const rolloutId = parseRolloutId(filePath); - const toolNamesByCallId = new Map(); - let lineNumber = 0; - - for await (const record of readJsonlObjects(filePath, signal)) { - lineNumber += 1; - const message = toRawCodexMessage(record, rolloutId, lineNumber, toolNamesByCallId); - if (message) { - yield message; - } - } -} - -/** Handles to raw codex message. */ -function toRawCodexMessage( - record: JsonObject, - rolloutId: string, - lineNumber: number, - toolNamesByCallId: Map -): RawCodexMessage | null { - if (record.type !== "response_item" || !isRecord(record.payload)) { - return null; - } - - if (record.payload.type !== "message") { - return toToolMessage(record.payload, rolloutId, lineNumber, normalizeTimestamp(record.timestamp), toolNamesByCallId); - } - - const role = record.payload.role; - if (role !== "user" && role !== "assistant" && role !== "developer" && role !== "system") { - return null; - } - - const content = getContentText(record.payload.content); - if (!content) { - return null; - } - - return { - messageId: `${rolloutId}:${lineNumber}`, - conversationId: rolloutId, - role: role === "developer" ? "system" : role, - content, - createdAt: normalizeTimestamp(record.timestamp) - }; -} - -function toToolMessage( - payload: Record, - rolloutId: string, - lineNumber: number, - createdAt: string, - toolNamesByCallId: Map -): RawCodexMessage | null { - const type = payload.type; - if (type === "function_call" || type === "custom_tool_call") { - const callId = getString(payload.call_id) ?? getString(payload.id); - const name = getString(payload.name) ?? "tool"; - if (callId) { - toolNamesByCallId.set(callId, name); - if (toolNamesByCallId.size > MAX_TOOL_NAME_ENTRIES) { - const oldest = toolNamesByCallId.keys().next().value; - if (typeof oldest === "string") toolNamesByCallId.delete(oldest); - } - } - return { - messageId: `${rolloutId}:${lineNumber}`, - conversationId: rolloutId, - role: "tool", - content: renderToolMessage({ - name, - callId, - status: getString(payload.status), - input: firstDefined(payload.arguments, payload.input) - }), - createdAt - }; - } - - if (type === "function_call_output" || type === "custom_tool_call_output") { - const callId = getString(payload.call_id) ?? getString(payload.id); - return { - messageId: `${rolloutId}:${lineNumber}`, - conversationId: rolloutId, - role: "tool", - content: renderToolMessage({ - name: callId ? toolNamesByCallId.get(callId) ?? "tool" : "tool", - callId, - status: getString(payload.status), - output: firstDefined(payload.output, payload.result) - }), - createdAt - }; - } - - if (type === "web_search_call") { - return { - messageId: `${rolloutId}:${lineNumber}`, - conversationId: rolloutId, - role: "tool", - content: renderToolMessage({ - name: "web_search", - callId: getString(payload.call_id) ?? getString(payload.id), - status: getString(payload.status), - input: payload.action - }), - createdAt - }; - } - - return null; -} - -/** - * Parses the rollout uuid from the file name. - * - * @param filePath Rollout path. - * @returns The uuid, falling back to the file name. - */ -function parseRolloutId(filePath: string): string { - const name = basename(filePath).replace(/\.jsonl$/, ""); - const uuid = name.match(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i)?.[0]; - return uuid ?? name; -} - -/** - * Extracts the text from Codex content. - * - * @param content Raw payload.content value. - * @returns The merged text, or null. - */ -function getContentText(content: unknown): string | null { - if (!Array.isArray(content)) { - return null; - } - - const text = content - .filter(isRecord) - .map((item) => (typeof item.text === "string" ? item.text : null)) - .filter((item): item is string => Boolean(item)) - .join("\n"); - return text.length > 0 ? text : null; -} - -function renderToolMessage(input: { - name: string; - callId?: string; - status?: string; - input?: unknown; - output?: unknown; -}): string { - return [ - `Tool: ${input.name}`, - input.callId ? `Call ID: ${input.callId}` : undefined, - input.status ? `Status: ${input.status}` : undefined, - input.input !== undefined ? `Input:\n${formatToolPayload(input.input)}` : undefined, - input.output !== undefined ? `Output:\n${formatToolPayload(input.output)}` : undefined - ].filter(Boolean).join("\n\n"); -} - -function formatToolPayload(value: unknown): string { - if (typeof value === "string") { - return value.trim(); - } - try { - return JSON.stringify(value, null, 2); - } catch { - return String(value); - } -} - -function firstDefined(...values: unknown[]): unknown { - return values.find((value) => value !== undefined && value !== null); -} - -function getString(value: unknown): string | undefined { - return typeof value === "string" && value.length > 0 ? value : undefined; -} - -function normalizeTimestamp(value: unknown): string { - if (typeof value === "string") { - const date = new Date(value); - return Number.isNaN(date.getTime()) ? new Date(0).toISOString() : date.toISOString(); - } - - return new Date(0).toISOString(); -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} +export { readCodexRollout, type RawCodexMessage } from "@memmy/agent-source-core"; diff --git a/Memory/src/agent-source/adapters/jsonl-lines.ts b/Memory/src/agent-source/adapters/jsonl-lines.ts index f6a9cea41..a01d98e59 100644 --- a/Memory/src/agent-source/adapters/jsonl-lines.ts +++ b/Memory/src/agent-source/adapters/jsonl-lines.ts @@ -1,95 +1 @@ -/** Jsonl lines module. */ -import { createReadStream } from "node:fs"; - -export type JsonPrimitive = string | number | boolean | null; -export type JsonValue = JsonPrimitive | JsonObject | JsonValue[]; -export type JsonObject = { readonly [key: string]: JsonValue }; - -/** - * Streams valid object rows from a JSONL file. - * Malformed and non-object rows are skipped without interrupting the stream. - * - * @param filePath JSONL file path. - * @param signal Optional abort signal. - * @returns The JSON objects parsed line by line. - */ -export async function* readJsonlObjects(filePath: string, signal?: AbortSignal): AsyncIterable { - const stream = createReadStream(filePath); - const maxRecordBytes = 64 * 1024 * 1024; - let segments: Buffer[] = []; - let recordBytes = 0; - let overLimit = false; - const append = (segment: Buffer): void => { - if (overLimit || segment.length === 0) return; - recordBytes += segment.length; - if (recordBytes > maxRecordBytes) { - segments = []; - overLimit = true; - return; - } - segments.push(segment); - }; - const reset = (): void => { - segments = []; - recordBytes = 0; - overLimit = false; - }; - const parseSegments = (): JsonObject | null => { - if (overLimit) return null; - const line = segments.length === 1 ? segments[0]! : Buffer.concat(segments, recordBytes); - const text = line.toString("utf8").trim(); - if (!text) return null; - try { - const parsed = JSON.parse(text) as unknown; - return isJsonObject(parsed) ? parsed : null; - } catch { - return null; - } - }; - - try { - for await (const chunk of stream) { - throwIfAborted(signal, filePath); - const buffer = chunk as Buffer; - let start = 0; - while (start <= buffer.length) { - const newline = buffer.indexOf(0x0a, start); - if (newline < 0) { - append(buffer.subarray(start)); - break; - } - append(buffer.subarray(start, newline)); - const parsed = parseSegments(); - reset(); - if (parsed) yield parsed; - start = newline + 1; - } - } - const parsed = parseSegments(); - if (parsed) yield parsed; - } finally { - stream.destroy(); - } -} - -/** - * Abort-signal check. - * - * @param signal Optional abort signal. - * @param filePath Current file path. - */ -function throwIfAborted(signal: AbortSignal | undefined, filePath: string): void { - if (signal?.aborted) { - throw new DOMException(`JSONL read aborted: ${filePath}`, "AbortError"); - } -} - -/** - * JSON object type guard. - * - * @param value Unknown value. - * @returns Whether it is a non-array object. - */ -function isJsonObject(value: unknown): value is JsonObject { - return typeof value === "object" && value !== null && !Array.isArray(value); -} +export { readJsonlObjects, type JsonObject, type JsonValue, type JsonPrimitive } from "@memmy/agent-source-core"; diff --git a/Memory/src/agent-source/adapters/secret-redactor.ts b/Memory/src/agent-source/adapters/secret-redactor.ts index 49a9a4ef5..7038a55b1 100644 --- a/Memory/src/agent-source/adapters/secret-redactor.ts +++ b/Memory/src/agent-source/adapters/secret-redactor.ts @@ -1,120 +1 @@ -/** Type definition for redaction rule. */ -type RedactionRule = { - /** Pattern. */ - pattern: RegExp; - /** Token. */ - token: string; - /** Replace. */ - replace?: (match: string, ...groups: string[]) => string; -}; - -const REDACTION_RULES: readonly RedactionRule[] = [ - { - pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, - token: "[REDACTED:ssh_private_key]" - }, - { - pattern: /\b(Authorization\s*:\s*Bearer\s+)[A-Za-z0-9._~+/=-]+/gi, - token: "[REDACTED:authorization_bearer]", - replace: (_match, prefix: string) => `${prefix}[REDACTED:authorization_bearer]` - }, - { - pattern: /\bsk-ant-api\d{2}-[A-Za-z0-9_-]{40,}\b/g, - token: "[REDACTED:anthropic_api_key]" - }, - { - pattern: /\bsk-(?:proj-)?[A-Za-z0-9_-]{40,}\b/g, - token: "[REDACTED:openai_api_key]" - }, - { - pattern: /\bAIza[A-Za-z0-9_-]{32,}\b/g, - token: "[REDACTED:google_api_key]" - }, - { - pattern: /\b([A-Za-z0-9_]*password[A-Za-z0-9_]*\s*[:=]\s*)(?:"[^"\n]+"|'[^'\n]+'|[^\s#&]+)/gi, - token: "[REDACTED:password]", - replace: (_match, prefix: string) => `${prefix}[REDACTED:password]` - } -]; - -const BASE64_SECRET_TOKEN = "[REDACTED:base64_secret]"; -const BASE64_SECRET_MIN_LENGTH = 32; -const LARGE_BASE64_PAYLOAD_MIN_LENGTH = 4096; - -/** - * Redacts common secrets from text. - * - * @param input Raw message text from an external Agent. - * @returns The plain text with secrets replaced, or the original text when no rule matches. - */ -export function redactSecrets(input: string): string { - const withoutLargeBinaryPayloads = redactBase64Runs(input, LARGE_BASE64_PAYLOAD_MIN_LENGTH); - const redacted = REDACTION_RULES.reduce((current, rule) => { - if (rule.replace) { - return current.replace(rule.pattern, rule.replace); - } - - return current.replace(rule.pattern, rule.token); - }, withoutLargeBinaryPayloads); - - return redactBase64Runs(redacted, BASE64_SECRET_MIN_LENGTH); -} - -function redactBase64Runs(input: string, minLength: number): string { - let output = ""; - let cursor = 0; - let index = 0; - - while (index < input.length) { - if (!isBase64CoreChar(input.charCodeAt(index))) { - index += 1; - continue; - } - - const start = index; - while (index < input.length && isBase64CoreChar(input.charCodeAt(index))) { - index += 1; - } - const coreEnd = index; - let padding = 0; - while (padding < 2 && input.charCodeAt(index) === 61) { - index += 1; - padding += 1; - } - - if (coreEnd - start >= minLength && hasBase64Boundary(input, start, index)) { - output += input.slice(cursor, start); - output += BASE64_SECRET_TOKEN; - cursor = index; - } - } - - if (cursor === 0) { - return input; - } - - return output + input.slice(cursor); -} - -function hasBase64Boundary(input: string, start: number, end: number): boolean { - return !isAsciiWord(input.charCodeAt(start - 1)) && !isAsciiWord(input.charCodeAt(end)); -} - -function isBase64CoreChar(code: number): boolean { - return ( - (code >= 65 && code <= 90) || - (code >= 97 && code <= 122) || - (code >= 48 && code <= 57) || - code === 43 || - code === 47 - ); -} - -function isAsciiWord(code: number): boolean { - return ( - (code >= 65 && code <= 90) || - (code >= 97 && code <= 122) || - (code >= 48 && code <= 57) || - code === 95 - ); -} +export { redactSecrets } from "@memmy/agent-source-core"; diff --git a/Memory/src/agent-source/integration/templates/memmy-resume-hook.ts b/Memory/src/agent-source/integration/templates/memmy-resume-hook.ts index 716677ef6..a3d38ee8a 100644 --- a/Memory/src/agent-source/integration/templates/memmy-resume-hook.ts +++ b/Memory/src/agent-source/integration/templates/memmy-resume-hook.ts @@ -16,6 +16,8 @@ import { join } from "node:path"; import { closeRuntimeSession, completeRuntimeTurn, + completeSourceTurn, + readCodexSourceTurn, loadRuntimeL3, notifyRuntimeBoundary, openRuntimeSession, @@ -57,7 +59,8 @@ async function main() { if (isStopEvent(payload)) { try { await captureCompletedTurn(payload); - } catch { + } catch (error) { + reportCaptureFailure("request_failed", error, payload); // Memory capture must not interrupt host turn completion. } writeStopOutput(); @@ -88,7 +91,8 @@ async function main() { try { const started = await startCapturedTurn(payload, prompt); writeTurnStartOutput(started); - } catch { + } catch (error) { + reportCaptureFailure("start_failed", error, payload); writeAllowOutput(); } return; @@ -223,6 +227,10 @@ function isAgentResponseEvent(payload) { } async function captureCompletedTurn(payload) { + if (MODE === "codex") { + await captureCodexSourceTurn(payload); + return; + } const pending = await readTurnState(payload); const status = completedTurnStatus(payload); if (status === "cancelled") { @@ -264,6 +272,59 @@ async function captureCompletedTurn(payload) { await clearTurnState(payload); } +async function captureCodexSourceTurn(payload) { + const status = completedTurnStatus(payload); + if (status === "cancelled") { + await clearTurnState(payload); + return; + } + const transcriptPath = normalizeText(payload.transcript_path || payload.transcriptPath); + if (!transcriptPath) { + reportCaptureFailure("transcript_unavailable", undefined, payload); + return; + } + const pending = await readTurnState(payload); + const expectedTurnId = platformTurnId(payload); + const expectedConversationId = normalizeText(payload.session_id || payload.sessionId || payload.conversation_id || payload.conversationId || payload.thread_id || payload.threadId); + const parsed = await readCodexSourceTurn(transcriptPath, { + turnId: expectedTurnId || undefined, + conversationId: expectedConversationId || undefined, + stop: status === "succeeded" + }); + if (!parsed.turn) { + reportCaptureFailure(parsed.reason || "identity_unresolved", undefined, payload); + return; + } + if (status === "failed" && parsed.turn.status !== "failed") { + reportCaptureFailure("turn_status_unresolved", undefined, payload); + return; + } + if (isResumeCommand(parsed.turn.query)) { + await clearTurnState(payload); + return; + } + const result = await completeSourceTurn({ + configUrl: CONFIG_URL, + turn: parsed.turn, + sessionId: normalizeText(pending && pending.sessionId) || undefined, + sourceMemoryIds: Array.isArray(pending && pending.sourceMemoryIds) ? pending.sourceMemoryIds : undefined + }); + if (result.status === "stored" || result.status === "existing" || result.status === "rejected") { + await clearTurnState(payload); + return; + } + reportCaptureFailure(normalizeText(result.reason) || normalizeText(result.status) || "unexpected_response", undefined, payload); +} + +function reportCaptureFailure(reason, error, payload = {}) { + process.stderr.write(JSON.stringify({ + event: "memmy.hook.capture_failed", source: SOURCE, reason, + sourceSessionId: sessionStateKey(payload), sourceTurnId: platformTurnId(payload) || undefined, + transcriptPath: normalizeText(payload.transcript_path || payload.transcriptPath) || undefined, + error: error ? formatError(error) : undefined + }) + "\n"); +} + async function startCapturedTurn(payload, prompt) { const query = sanitizeCaptureText(prompt); if (!query) { diff --git a/Memory/src/agent-source/integration/workspace-bridge/runtime.test.ts b/Memory/src/agent-source/integration/workspace-bridge/runtime.test.ts index b8bc695bd..604c72e99 100644 --- a/Memory/src/agent-source/integration/workspace-bridge/runtime.test.ts +++ b/Memory/src/agent-source/integration/workspace-bridge/runtime.test.ts @@ -5,7 +5,9 @@ import { join } from "node:path"; import { pathToFileURL } from "node:url"; import { afterEach, describe, expect, it } from "vitest"; import { loadMemmyWorkspaceBridgeRuntimeAsset } from "./runtime-loader.js"; +import type { SourceTurn } from "@memmy/agent-source-core"; import { + completeSourceTurn, notifyRuntimeBoundary, openRuntimeSession, readRuntimeConfig, @@ -48,6 +50,47 @@ describe("Memory lifecycle runtime", () => { }); }); + it.each([undefined, "pending-session"])("submits the pinned owner with pending Session %s across account changes", async (sessionId) => { + const fixture = createFixture(); + const requests: Array<{ path: string; body: Record }> = []; + const server = createServer(async (request, response) => { + requests.push({ path: request.url ?? "", body: await requestBody(request) }); + return json(response, 200, { status: "stored" }); + }); + const endpoint = await listen(server); + const configUrl = runtimeConfig(fixture, endpoint); + const turn: SourceTurn = { + source: "codex", conversationId: "native-conversation", turnId: "native-turn", + startedAt: "2026-09-09T10:00:00.000Z", completedAt: "2026-09-09T10:00:01.000Z", + sequence: 1, completionEvidence: "final_answer:native-turn", query: "Fix parser", answer: "Fixed", + status: "succeeded", toolCalls: [], toolResults: [], workspacePath: fixture, + }; + try { + await completeSourceTurn({ configUrl, turn, sessionId, profileId: "work", sourceMemoryIds: ["recalled-1"] }); + writeFileSync(join(fixture, "missing.yaml"), [ + "app:", " userId: switched-app-owner", "memmyMemory:", " userId: switched-memory-owner", "", + ].join("\n")); + await completeSourceTurn({ configUrl, turn, sessionId, profileId: "work", sourceMemoryIds: ["recalled-1"] }); + + expect(requests).toHaveLength(2); + expect(requests[1]).toEqual(requests[0]); + for (const request of requests) { + expect(request.path).toBe("/api/v1/source-turns/complete"); + expect(request.body.namespace).toEqual({ + source: "codex", profileId: "work", userId: "installed-owner", sessionKey: "native-conversation", + }); + expect(request.body).toMatchObject({ + sourceTurn: { source: "codex", profileId: "work", conversationId: "native-conversation" }, + sourceMemoryIds: ["recalled-1"], workspacePath: fixture, + }); + if (sessionId) expect(request.body.sessionId).toBe(sessionId); + else expect(request.body).not.toHaveProperty("sessionId"); + } + } finally { + await close(server); + } + }); + it("opens a v2 project Session with only canonical workspace identity", async () => { const fixture = createFixture(); const requests: Array<{ path: string; body: Record }> = []; diff --git a/Memory/src/agent-source/integration/workspace-bridge/runtime.ts b/Memory/src/agent-source/integration/workspace-bridge/runtime.ts index b690c13e2..378150550 100644 --- a/Memory/src/agent-source/integration/workspace-bridge/runtime.ts +++ b/Memory/src/agent-source/integration/workspace-bridge/runtime.ts @@ -1,3 +1,5 @@ +import { buildSourceTurnRequest, type SourceTurn } from "@memmy/agent-source-core"; +export { readCodexSourceTurn } from "@memmy/agent-source-core"; import { createHash, randomUUID } from "node:crypto"; import { lstat, readFile, realpath, stat } from "node:fs/promises"; import { homedir } from "node:os"; @@ -232,6 +234,31 @@ export async function completeRuntimeTurn( await client.post(`/api/v1/turns/${encodeURIComponent(input.turnId)}/complete`, compact(body)); } +/** Submit a completed native turn without opening a new runtime Session before deduplication. */ +export async function completeSourceTurn(input: { + configUrl: URL; + turn: SourceTurn; + sessionId?: string; + sourceMemoryIds?: string[]; + profileId?: string; +}): Promise> { + const config = await readRuntimeConfig(input.configUrl, true); + const client = new RuntimeHttpClient(config); + const profileId = input.profileId || "default"; + return objectValue(await client.post("/api/v1/source-turns/complete", compact({ + ...buildSourceTurnRequest(input.turn, "hook", profileId), + namespace: { + source: input.turn.source, + profileId, + userId: config.userId, + sessionKey: input.turn.conversationId, + }, + sessionId: input.sessionId, + sourceMemoryIds: input.sourceMemoryIds, + adapterId: "memmy-codex-hook", + }))); +} + class RuntimeHttpClient { constructor(private readonly config: RuntimeConfig) {} diff --git a/Memory/src/agent-source/runtime.ts b/Memory/src/agent-source/runtime.ts index 47816672a..5e0787c01 100644 --- a/Memory/src/agent-source/runtime.ts +++ b/Memory/src/agent-source/runtime.ts @@ -34,6 +34,9 @@ import type { ConversationMessage, ScanProgress, SourceAdapter } from "./adapter import { isCompleteTurn, orderedTurns, + sourceTurnFromMessages, + sourceTurnFailureReason, + buildSourceTurnRequest, renderTurnClipped, stableTurnIdentity, legacyTurnId, @@ -784,6 +787,29 @@ async function ingestStagedMessages( if (conversationMeta?.selected === false) continue; const selectedTurn = store.getTurnMeta(sourceId, turn.conversationId, stableTurnIdentity(turn)); if (selectedTurn && !selectedTurn.selected) continue; + if (sourceId === "codex") { + try { + const sourceTurn = sourceTurnFromMessages(turn.messages); + if (!sourceTurn) throw new Error(sourceTurnFailureReason(turn.messages)); + const result = service.completeSourceTurn(buildSourceTurnRequest(sourceTurn, "agent_source_scan")); + if (result.status === "pending" || result.status === "conflict") throw new Error(result.reason ?? result.status); + const ids = result.result?.l1MemoryIds ?? []; + if (result.status === "stored") written += ids.length; + if (ids.length === 0) store.saveResult({ sourceId, conversationId: turn.conversationId }); + for (const memoryId of ids) store.saveResult({ sourceId, conversationId: turn.conversationId, memoryId }); + messageCount += turn.messages.length; + if (result.status === "stored") scheduleWorker?.(); + } catch (error) { + activeConversationFailed = true; + const reason = error instanceof Error ? error.message : "native turn ingestion failed"; + errorCount += 1; + if (errors.length < 1000) errors.push(`${turn.conversationId}: ${reason}`); + store.saveResult({ sourceId, conversationId: turn.conversationId, error: reason }); + } + processed += turn.messages.length; + onProgress({ sourceId, phase: "add", current: processed, total: store.count(sourceId), message: "Capturing conversation turns" }); + continue; + } let succeeded = true; // One turn is one memory. Splitting an agentic turn fans a single exchange // out into hundreds of near-empty tool-call fragments, so an oversized turn @@ -834,7 +860,7 @@ async function prepareStandaloneSource( sourceHash.update("["); let firstSourceMessage = true; const flushTurn = () => { - if (!currentTurn.length || !isCompleteTurn(currentTurn)) return; + if (!currentTurn.length || (sourceId !== "codex" && !isCompleteTurn(currentTurn))) return; const firstMessage = currentTurn[0]!; const lastMessage = currentTurn[currentTurn.length - 1]!; const turn = { sourceId, conversationId: firstMessage.conversationId, turnIndex: 0, messages: currentTurn }; @@ -878,7 +904,7 @@ async function prepareStandaloneSource( hash.update("["); first = true; } - if (message.role === "user" && currentTurn.length > 0) { + if (currentTurn.length > 0 && (sourceId === "codex" ? message.rawMeta.sourceTurnId !== currentTurn[0]?.rawMeta.sourceTurnId : message.role === "user")) { flushTurn(); currentTurn = []; } @@ -891,7 +917,8 @@ async function prepareStandaloneSource( content: message.content, createdAt: message.createdAt, toolName: hashMeta(message, "toolName") ?? hashMeta(message, "hermesToolName"), - toolCallId: hashMeta(message, "toolCallId") ?? hashMeta(message, "hermesToolCallId") + toolCallId: hashMeta(message, "toolCallId") ?? hashMeta(message, "hermesToolCallId"), + ...(sourceId === "codex" ? { sourceTurn: message.rawMeta } : {}) }; const serialized = JSON.stringify(hashable); if (!firstSourceMessage) sourceHash.update(","); diff --git a/Memory/src/agent-source/scan-store.ts b/Memory/src/agent-source/scan-store.ts index dc3f06255..37d1e3013 100644 --- a/Memory/src/agent-source/scan-store.ts +++ b/Memory/src/agent-source/scan-store.ts @@ -33,9 +33,23 @@ export async function openMemoryAgentSourceScanStore(path: string, job: MemorySc if (!db.prepare("SELECT 1 FROM scan_meta WHERE id=1").get()) db.prepare("INSERT INTO scan_meta(id,job_id,source_id,mode,phase,created_at,updated_at,error) VALUES(1,@jobId,@sourceId,@mode,@phase,@createdAt,@updatedAt,@error)").run({ ...job, error: job.error ?? null }); let ordinal = Number((db.prepare("SELECT COALESCE(MAX(ordinal),-1) AS value FROM staged_messages WHERE job_id=?").get(job.jobId) as { value: number }).value) + 1; const insert = db.prepare("INSERT OR IGNORE INTO staged_messages(job_id,source_id,conversation_id,message_id,role,content,created_at,workspace_path,git_root,raw_meta_json,ordinal) VALUES(?,?,?,?,?,?,?,?,?,?,?)"); + const refreshCodex = db.prepare(`UPDATE staged_messages + SET conversation_id=?,role=?,content=?,created_at=?,workspace_path=?,git_root=?,raw_meta_json=? + WHERE job_id=? AND source_id=? AND message_id=?`); const store: MemoryAgentSourceScanStore = { path, - stage(message) { const bytes = Buffer.byteLength(JSON.stringify(message)); if (bytes > MAX_RECORD_BYTES) throw new Error(`scan record exceeds 64 MiB limit (${bytes} bytes)`); return Number(insert.run(job.jobId,message.sourceId,message.conversationId,message.messageId,message.role,message.content,message.createdAt,message.workspacePath,message.gitRoot,JSON.stringify(message.rawMeta),ordinal++).changes)>0; }, + stage(message) { + const bytes = Buffer.byteLength(JSON.stringify(message)); + if (bytes > MAX_RECORD_BYTES) throw new Error(`scan record exceeds 64 MiB limit (${bytes} bytes)`); + const rawMetaJson = JSON.stringify(message.rawMeta); + const inserted = Number(insert.run(job.jobId, message.sourceId, message.conversationId, message.messageId, message.role, message.content, message.createdAt, message.workspacePath, message.gitRoot, rawMetaJson, ordinal++).changes) > 0; + if (!inserted && message.sourceId === "codex" && typeof message.rawMeta.sourceTurnState === "string") { + // Retrying a staged turn can add native identity or completion evidence + // to an existing message. Preserve its ordinal and the insertion count. + refreshCodex.run(message.conversationId, message.role, message.content, message.createdAt, message.workspacePath, message.gitRoot, rawMetaJson, job.jobId, message.sourceId, message.messageId); + } + return inserted; + }, stageBatch(messages) { const tx = db.transaction(() => { let count = 0; for (const message of messages) if (store.stage(message)) count += 1; return count; }); return tx(); }, messages(sourceId, cursor, limit=500) { limit=Math.min(500,Math.max(1,limit)); const params: unknown[]=[job.jobId,sourceId]; let where="job_id=? AND source_id=?"; if(cursor){where += " AND ((conversation_id > ?) OR (conversation_id = ? AND (created_at > ? OR (created_at = ? AND (message_id > ? OR (message_id = ? AND ordinal > ?))))))"; params.push(cursor.conversationId,cursor.conversationId,cursor.createdAt,cursor.createdAt,cursor.messageId,cursor.messageId,cursor.ordinal);} const rows = db.prepare(`SELECT source_id AS sourceId,conversation_id AS conversationId,message_id AS messageId,role,content,created_at AS createdAt,workspace_path AS workspacePath,git_root AS gitRoot,raw_meta_json AS rawMetaJson,ordinal FROM staged_messages WHERE ${where} ORDER BY conversation_id,created_at,message_id,ordinal LIMIT ?`).iterate(...params,limit) as Iterable>; return (function*(){let bytes=0; let count=0; for(const row of rows){const message=rowToMessage(row); yield message; count+=1; bytes+=Buffer.byteLength(JSON.stringify(message)); if(count>=500||bytes>=MAX_PAGE_BYTES) break;}})(); }, saveScanCursor(s,c){db.prepare("INSERT INTO scan_cursors(source_id,conversation_id,created_at,message_id,ordinal) VALUES(?,?,?,?,?) ON CONFLICT(source_id) DO UPDATE SET conversation_id=excluded.conversation_id,created_at=excluded.created_at,message_id=excluded.message_id,ordinal=excluded.ordinal").run(s,c.conversationId,c.createdAt,c.messageId,c.ordinal);}, diff --git a/Memory/src/algorithm/plugin-algorithms.ts b/Memory/src/algorithm/plugin-algorithms.ts index b98a07268..f8e5d5805 100644 --- a/Memory/src/algorithm/plugin-algorithms.ts +++ b/Memory/src/algorithm/plugin-algorithms.ts @@ -10,6 +10,7 @@ import type { LlmClient } from "../model/types.js"; import { MEMORY_SUMMARY_MAX_TOKENS } from "../config/index.js"; import { memoryVector } from "../storage/memory-vector-state.js"; import { stableHash } from "../utils/id.js"; +import { matchToolResultIndices } from "../utils/tool-call-pairing.js"; import { formatZonedTime } from "../utils/time.js"; import { renderL3WorldModelFields, @@ -4468,14 +4469,37 @@ export function tracePolicySimilarity( } function normalizeToolCalls(toolCalls: ToolCallPayload[], toolResults: unknown[]): ToolCallPayload[] { + const resultIndices = matchToolResultIndices(toolCalls, toolResults); return toolCalls.map((call, index) => { - const result = toolResults[index]; - const output = call.output ?? result; + const resultIndex = resultIndices[index]; + const pairedResult = resultIndex === undefined ? undefined : toolResults[resultIndex]; + const result = pairedResult && typeof pairedResult === "object" && !Array.isArray(pairedResult) + ? pairedResult as Record + : {}; + const output = call.output ?? result.output ?? result.result ?? result.content ?? pairedResult; + const resultError = typeof result.error === "string" ? result.error : errorMessageFromUnknown(result.error); + const error = call.error ?? resultError ?? + (result.success === false ? errorMessageFromUnknown(result) : undefined); + let success: boolean | undefined; + if (typeof call.success === "boolean") { + success = call.success; + } else if (typeof result.success === "boolean") { + success = result.success; + } else if (error) { + success = false; + } else if (output !== undefined) { + success = true; + } else { + success = undefined; + } + const resultErrorCode = result.errorCode ?? result.error_code; return { ...call, output, - error: call.error, - success: call.success ?? !call.error + status: call.status ?? (typeof result.status === "string" ? result.status : undefined), + error, + errorCode: call.errorCode ?? (typeof resultErrorCode === "string" ? resultErrorCode : undefined), + success }; }); } diff --git a/Memory/src/server/http.ts b/Memory/src/server/http.ts index 4d0979e09..2857a1a95 100644 --- a/Memory/src/server/http.ts +++ b/Memory/src/server/http.ts @@ -22,6 +22,7 @@ import type { RuntimeNamespace, SessionOpenRequest, TurnCompleteRequest, + SourceTurnCompleteRequest, TurnStartRequest } from "../types.js"; import { DEFAULT_NAMESPACE_SOURCE } from "../types.js"; @@ -64,6 +65,7 @@ export const API_ROUTES = [ "GET /api/v1/l3-world-model/sessions/:sessionId/context", "POST /api/v1/turns/start", "POST /api/v1/turns/:turnId/complete", + "POST /api/v1/source-turns/complete", "POST /api/v1/memory/search", "GET /api/v1/memory/recalls/:queryId", "POST /api/v1/memory/add", @@ -629,6 +631,39 @@ async function routeRequest( return publicStartTurnResponse(result); } + if (method === "POST" && path === "/api/v1/source-turns/complete") { + requireMemoryWrite(principal); + const input = asObject(body, "source-turn.complete"); + const sourceIdentity = isRecord(input.sourceTurn) ? input.sourceTurn : {}; + const requestedScope = isRecord(input.namespace) ? input.namespace : {}; + const namespace = { + source: sourceIdentity.source, + profileId: sourceIdentity.profileId, + sessionKey: sourceIdentity.conversationId, + ...requestedScope + }; + // Local headers may carry the generic default source; it is not a source restriction. + let scopedPrincipal = principal; + if ((principal.kind === "local" || principal.kind === "anonymous") && + principal.namespace?.source === DEFAULT_NAMESPACE_SOURCE && typeof namespace.source === "string") { + scopedPrincipal = { ...principal, namespace: { ...principal.namespace, source: namespace.source } }; + } + const request = strictEnvelopeWithPrincipal({ ...input, namespace }, scopedPrincipal) as unknown as SourceTurnCompleteRequest; + requireStringField(request, "query", "source-turn.complete"); + requireStringField(request, "answer", "source-turn.complete"); + const result = service.completeSourceTurn({ + namespace: request.namespace, timeZone: request.timeZone, source: request.source, + sourceTurn: request.sourceTurn, channel: request.channel, workspacePath: request.workspacePath, + sessionId: request.sessionId, episodeId: request.episodeId, + query: request.query, answer: request.answer, reasoningSummary: request.reasoningSummary, + toolCalls: request.toolCalls, toolResults: request.toolResults, artifacts: request.artifacts, + sourceMemoryIds: request.sourceMemoryIds, usage: request.usage, status: request.status, + tags: request.tags, userMemoryCorrection: request.userMemoryCorrection + }); + if (result.result) scheduleAutoWorkerForEvolution(result.result, autoWorker); + return result; + } + const turnComplete = match(path, /^\/api\/v1\/turns\/([^/]+)\/complete$/); if (method === "POST" && turnComplete) { requireMemoryWrite(principal); diff --git a/Memory/src/service/memory-service.ts b/Memory/src/service/memory-service.ts index 45c5c17ae..d7434c817 100644 --- a/Memory/src/service/memory-service.ts +++ b/Memory/src/service/memory-service.ts @@ -86,6 +86,9 @@ import type { ToolCallPayload, ToolObserveRequest, TurnCompleteRequest, + SourceTurnCompleteRequest, + SourceTurnCompleteResponse, + TurnCompletionResult, TurnStartRequest } from "../types.js"; import { MemoryServiceError } from "../utils/error.js"; @@ -198,24 +201,8 @@ export interface MemoryServiceOptions { viewerEndpoint?: string; } -export interface CompleteTurnResponse { - turnId: string; - sessionId: string; - episodeId: string; - rawTurnId: string; - userMemoryId: string; - userMemoryIds: string[]; - l1MemoryId: string; - l1MemoryIds: string[]; - closedEpisodeIds: string[]; - scheduledEvolution: boolean; - jobs: JobRef[]; - changeSeq: number; - syncCursor: string; - etag: string; - serverTime: string; - duplicate?: boolean; -} +export type CompleteTurnResponse = TurnCompletionResult; + type TraceMeta = NonNullable>; interface DecisionRepairSummary { @@ -1065,6 +1052,35 @@ export class MemoryService { return this.withModelTaskContext(() => this.sessionTurns.startTurn(this.withTimeZone(request))); } + completeSourceTurn(request: SourceTurnCompleteRequest): SourceTurnCompleteResponse { + // Native scans have no Hook envelope. Use the configured owner only when + // the request (including authenticated scope) did not provide one. + const response = this.sessionTurns.completeSourceTurn(this.withTimeZone({ + ...request, + namespace: { + source: request.sourceTurn?.source, + profileId: request.sourceTurn?.profileId, + sessionKey: request.sourceTurn?.conversationId, + ...request.namespace, + userId: request.namespace?.userId ?? this.config.userId + } + })); + serviceLogger.info("source_turn.complete", { + source: request.sourceTurn?.source, + profileId: request.sourceTurn?.profileId, + conversationId: request.sourceTurn?.conversationId, + turnId: request.sourceTurn?.turnId, + channel: request.channel, + status: response.status, + reason: response.reason, + sessionId: response.result?.sessionId, + episodeId: response.result?.episodeId, + rawTurnId: response.result?.rawTurnId, + l1MemoryIds: response.result?.l1MemoryIds + }); + return response; + } + completeTurn(turnId: string, request: TurnCompleteRequest & Record): CompleteTurnResponse { return this.sessionTurns.completeTurn(turnId, this.withTimeZone(request)); } @@ -2685,9 +2701,12 @@ function sanitizeTraceToolCalls(toolCalls: ToolCallPayload[]): ToolCallPayload[] return toolCalls.map((call) => ({ id: call.id, name: call.name, + input: call.input, + output: call.output, + status: call.status, success: call.success, errorCode: call.errorCode, - error: call.error ?? errorMessageFromUnknown(call.output), + error: call.error, startedAt: call.startedAt, endedAt: call.endedAt, thinkingBefore: call.thinkingBefore, diff --git a/Memory/src/service/session/session-turn-service.ts b/Memory/src/service/session/session-turn-service.ts index d070497e0..cc1750b07 100644 --- a/Memory/src/service/session/session-turn-service.ts +++ b/Memory/src/service/session/session-turn-service.ts @@ -23,7 +23,8 @@ import { type EpisodeRecord, type EvolutionJobRecord, type RawTurnRecord, - type SessionRecord + type SessionRecord, + type SourceTurnCaptureScope } from "../../storage/repositories.js"; import type { FeedbackRequest, @@ -43,6 +44,10 @@ import type { ToolCallPayload, ToolObserveRequest, TurnCompleteRequest, + SourceTurnCompleteRequest, + SourceTurnCompleteResponse, + SourceTurnIdentity, + TurnCompletionResult, TurnStartRequest } from "../../types.js"; import { MemoryServiceError } from "../../utils/error.js"; @@ -104,7 +109,7 @@ type SessionTurnDependencies = { readonly skillLlm: LlmClient; synthesizeDecisionRepairDraft: SynthesizeDecisionRepairDraft; } & Record; -interface CompleteTurnResponse { turnId: string; sessionId: string; episodeId: string; rawTurnId: string; userMemoryId: string; userMemoryIds: string[]; l1MemoryId: string; l1MemoryIds: string[]; closedEpisodeIds: string[]; scheduledEvolution: boolean; jobs: JobRef[]; changeSeq: number; syncCursor: string; etag: string; serverTime: string; duplicate?: boolean; } +type CompleteTurnResponse = TurnCompletionResult; type EndTopicDecision = TurnRelationDecision & { relation: "end_topic" }; interface EpisodeTurnRoute { episode: EpisodeRecord; endTopicDecision?: EndTopicDecision; } type TurnRouteAction = "create_first" | "append" | "split" | "end_topic"; @@ -471,7 +476,7 @@ export class SessionTurnService { constructor(private readonly deps: SessionTurnDependencies) {} - openSession(request: SessionOpenRequest): { + openSession(request: SessionOpenRequest, options: { createNew?: boolean; at?: string } = {}): { sessionId: string; userId: string; source: string; @@ -504,7 +509,7 @@ export class SessionTurnService { } } const namespace = normalizeNamespace(request.namespace); - const at = nowIso(); + const at = options.at ?? nowIso(); if (request.l3WorldModelProtocolVersion === undefined && ( request.l3WorldModelTransition !== undefined || request.workspaceUri !== undefined || @@ -519,7 +524,7 @@ export class SessionTurnService { } return body; } - if (request.sessionId) { + if (request.sessionId && !options.createNew) { const existingSession = this.deps.repos.runtime.getSession(request.sessionId); if (existingSession) { this.deps.assertSessionInScope(existingSession, request.namespace); @@ -551,7 +556,7 @@ export class SessionTurnService { } } const hostSessionKey = namespace.sessionKey; - if (hostSessionKey) { + if (hostSessionKey && !options.createNew) { const existingSession = this.deps.repos.runtime.findOpenSessionByHostKey({ userId: namespace.userId, source: request.source ?? namespace.source, @@ -1204,7 +1209,227 @@ export class SessionTurnService { }; } - completeTurn(turnId: string, request: TurnCompleteRequest & Record): CompleteTurnResponse { + completeSourceTurn(request: SourceTurnCompleteRequest): SourceTurnCompleteResponse { + const identity = request.sourceTurn; + if (!identity || ![identity.source, identity.profileId, identity.conversationId, identity.turnId, + identity.startedAt, identity.completedAt, identity.completionEvidence].every((value) => typeof value === "string" && value.trim())) { + return { status: "pending", reason: "identity_unresolved" }; + } + const startedAtMs = Date.parse(identity.startedAt); + const completedAtMs = Date.parse(identity.completedAt); + if (!Number.isFinite(startedAtMs) || !Number.isFinite(completedAtMs) || completedAtMs < startedAtMs || + (identity.sequence !== undefined && (!Number.isSafeInteger(identity.sequence) || identity.sequence < 0))) { + return { status: "pending", reason: "source_turn_boundary_unresolved" }; + } + if (request.channel !== "hook" && request.channel !== "agent_source_scan") { + throw new MemoryServiceError("invalid_argument", "invalid source capture channel"); + } + const namespace = normalizeNamespace(request.namespace ?? { + source: identity.source, profileId: identity.profileId, sessionKey: identity.conversationId + }); + if (namespace.source !== identity.source || namespace.profileId !== identity.profileId || + (request.source && request.source !== identity.source) || + (namespace.sessionKey && namespace.sessionKey !== identity.conversationId)) { + throw new MemoryServiceError("forbidden", "source_turn_namespace_conflict"); + } + const sourceTurn: SourceTurnIdentity = { + ...identity, + startedAt: new Date(startedAtMs).toISOString(), + completedAt: new Date(completedAtMs).toISOString() + }; + const scope: SourceTurnCaptureScope = { + userId: namespace.userId, + source: identity.source, + profileId: identity.profileId, + namespaceKey: stableHash({ tenantId: namespace.tenantId ?? null, projectId: namespace.projectId ?? null, + workspaceId: namespace.workspaceId ?? null }), + conversationId: identity.conversationId + }; + const normalized = sanitizeTurnCompleteRequest({ ...request, sessionId: request.sessionId ?? "" }); + const contentHash = stableHash({ + query: normalized.query, answer: normalized.answer, status: normalized.status ?? "succeeded", + reasoningSummary: normalized.reasoningSummary ?? null, + toolCalls: normalizeCompleteTurnToolCalls(normalized), toolResults: normalizeCompleteTurnToolResults(normalized) + }); + return this.deps.repos.transaction(() => { + // A durable source identity is checked before touching an open Session. Deleted memories + // and closed Sessions therefore cannot turn a retry into a second capture. + const existing = this.deps.repos.runtime.getSourceTurnCapture(scope, identity.turnId); + if (existing) { + if (existing.contentHash !== contentHash) { + return { status: "conflict", reason: "source_turn_content_conflict" }; + } + const result = existing.response.result; + if (!result) return existing.response; + const responseResult = { ...result, duplicate: true, scheduledEvolution: false, jobs: [] }; + const memories = result.l1MemoryIds.map((id) => this.deps.repos.memories.getIncludingDeleted(id)); + if (memories.some((memory) => isRecord(memory?.properties.internal_info.capture_decision) && + memory.properties.internal_info.capture_decision.status === "rejected")) { + return { status: "rejected", reason: "capture_policy", result: responseResult }; + } + if (memories.length > 0 && memories.every((memory) => !memory || memory.status === "deleted" || memory.deletedAt)) { + return { status: "rejected", reason: "capture_deleted", result: responseResult }; + } + if (existing.response.status === "rejected") { + return { ...existing.response, result: responseResult }; + } + return { status: "existing", result: responseResult }; + } + const activation = this.deps.repos.runtime.getKv("source_turn_capture_activated_at")?.value; + if (typeof activation !== "string" || !Number.isFinite(Date.parse(activation))) { + return { status: "pending", reason: "source_capture_activation_unresolved" }; + } + if (completedAtMs <= Date.parse(activation)) { + return { status: "rejected", reason: "legacy_before_activation" }; + } + if (!normalized.query.trim() || !normalized.answer.trim() || normalized.status === "cancelled") { + return { status: "pending", reason: "source_turn_incomplete" }; + } + if (!this.deps.memoryAddEnabled()) { + return { status: "pending", reason: "memory_add_disabled" }; + } + const previous = this.deps.repos.runtime.latestSourceTurnCapture(scope); + let gapEpisode: EpisodeRecord | undefined; + let targetSessionId = previous?.sessionId; + if (previous && (startedAtMs < Date.parse(previous.startedAt) || completedAtMs < Date.parse(previous.completedAt))) { + const neighbors = this.deps.repos.runtime.sourceTurnCaptureNeighbors(scope, sourceTurn.startedAt); + const before = neighbors.before; + const after = neighbors.after; + if (!before?.episodeId || before.episodeId !== after?.episodeId || before.sessionId !== after.sessionId || + Date.parse(before.completedAt) > startedAtMs || completedAtMs > Date.parse(after.startedAt)) { + return { status: "pending", reason: "source_turn_out_of_order" }; + } + const candidate = this.deps.repos.runtime.getEpisode(before.episodeId); + const beforeRaw = before.rawTurnId ? this.deps.repos.runtime.getRawTurn(before.rawTurnId) : undefined; + if (!candidate || candidate.status !== "open" || !beforeRaw) { + return { status: "pending", reason: "source_episode_closed" }; + } + const relation = classifyTurnRelation({ + prevUserText: beforeRaw.userText ?? "", prevAssistantText: beforeRaw.assistantText ?? "", + newUserText: normalized.query, gapMs: startedAtMs - Date.parse(before.completedAt), prevTags: [] + }); + if (relation.relation === "new_task" || relation.relation === "end_topic" || explicitEndTopicDecision(normalized.query)) { + return { status: "pending", reason: "source_turn_out_of_order" }; + } + gapEpisode = candidate; + targetSessionId = before.sessionId; + } + const resolved = this.resolveSourceSession({ ...request, namespace }, sourceTurn, scope, targetSessionId); + if ("status" in resolved && resolved.status === "pending") return resolved; + const session = resolved as SessionRecord; + const observed = this.deps.repos.runtime.getRawTurnBySessionTurn(session.id, identity.turnId); + if (observed && this.deps.repos.runtime.getEpisode(observed.episodeId)?.status !== "open") { + return { status: "pending", reason: "source_episode_closed" }; + } + const requestedEpisodeId = gapEpisode?.id ?? request.episodeId; + const episode = requestedEpisodeId + ? this.deps.repos.runtime.getEpisode(requestedEpisodeId) + : this.deps.repos.runtime.latestEpisodeForSession(session.id); + if (request.episodeId && (!episode || episode.sessionId !== session.id || episode.userId !== session.userId)) { + throw new MemoryServiceError("forbidden", "source_episode_scope_conflict"); + } + if (episode && episode.status !== "open") { + const proposal = this.proposeEpisodeRoute(session, normalized.query, undefined, sourceTurn.startedAt); + // Only an unambiguously new task after closure may create a fresh Episode. Never + // reopen an evaluated Episode merely because an offline turn arrived late. + if (request.episodeId || proposal.relationDecision.relation !== "new_task" || + startedAtMs < Date.parse(episode.closedAt ?? episode.updatedAt)) { + return { status: "pending", reason: "source_episode_closed" }; + } + } + if (episode && !gapEpisode) { + const laterRaw = episode.rawTurnIds.map((id) => this.deps.repos.runtime.getRawTurn(id)) + .some((raw) => raw && raw.turnId !== identity.turnId && Date.parse(raw.createdAt) > startedAtMs); + if (laterRaw) return { status: "pending", reason: "source_turn_out_of_order" }; + } + if (observed && isRecord(observed.messagePayload?.turn_complete)) { + // An old writer has already completed this turn without the source ledger. Do not + // adopt it into the new lifecycle or rewrite its contents during this release. + return { status: "pending", reason: "legacy_source_turn_already_completed" }; + } + // Bind only this verified live Session immediately before capture. Keep its original + // host key, workspace and protocol so Hook recall and SessionEnd still reach it. + if (!session.conversationId && !this.deps.repos.runtime.bindSessionSourceConversation(session.id, identity.conversationId)) { + throw new MemoryServiceError("conflict", "source_session_scope_conflict"); + } + if (observed && !this.deps.repos.runtime.bindRawTurnSourceConversation({ + id: observed.id, sessionId: session.id, userId: session.userId, turnId: identity.turnId + }, identity.conversationId)) { + throw new MemoryServiceError("forbidden", "source_raw_turn_scope_conflict"); + } + const result = this.completeTurn(identity.turnId, { ...normalized, sessionId: session.id, + ...(gapEpisode ? { episodeId: gapEpisode.id } : {}) }, sourceTurn); + if (gapEpisode) this.deps.repos.runtime.orderEpisodeTurnsBySourceTime(gapEpisode.id); + const response: SourceTurnCompleteResponse = result.l1MemoryIds.length > 0 + ? { status: "stored", result } + : { status: "rejected", reason: "capture_policy", result }; + this.deps.repos.runtime.insertSourceTurnCapture({ + ...scope, turnId: identity.turnId, contentHash, + sessionId: result.sessionId, episodeId: result.episodeId, rawTurnId: result.rawTurnId, + startedAt: sourceTurn.startedAt, completedAt: sourceTurn.completedAt, sequence: identity.sequence, + response, createdAt: nowIso() + }); + return response; + }); + } + + private resolveSourceSession( + request: SourceTurnCompleteRequest, + identity: SourceTurnIdentity, + scope: SourceTurnCaptureScope, + mappedSessionId?: string + ): SessionRecord | { status: "pending"; reason: string } { + const namespace = normalizeNamespace(request.namespace); + const inScope = (candidate: SessionRecord): boolean => + candidate.userId === namespace.userId && candidate.source === identity.source && candidate.profileId === identity.profileId && + (!candidate.conversationId || candidate.conversationId === identity.conversationId) && + (candidate.hostSessionKey === identity.conversationId || candidate.conversationId === identity.conversationId || + (identity.source === "codex" && candidate.hostSessionKey === `codex-memory-${identity.conversationId}`)) && + (!namespace.projectId || candidate.projectId === namespace.projectId) && + (!namespace.workspaceId || candidate.workspaceId === namespace.workspaceId) && + (candidate.meta.source_namespace_key === undefined || candidate.meta.source_namespace_key === scope.namespaceKey) && + (!namespace.tenantId || candidate.meta.source_namespace_key === scope.namespaceKey); + const canStartAfter = (candidate: SessionRecord): boolean => candidate.status === "closed" && + Date.parse(identity.startedAt) >= Date.parse(candidate.closedAt ?? candidate.updatedAt); + if (mappedSessionId) { + const mapped = this.deps.repos.runtime.getSession(mappedSessionId); + if (!mapped) return { status: "pending", reason: "source_session_missing" }; + if (!inScope(mapped)) throw new MemoryServiceError("forbidden", "source_session_scope_conflict"); + if (mapped.status === "open") return mapped; + if (!canStartAfter(mapped)) return { status: "pending", reason: "source_session_closed" }; + } + if (request.sessionId) { + const supplied = this.deps.repos.runtime.getSession(request.sessionId); + if (!supplied) return { status: "pending", reason: "source_session_missing" }; + if (!inScope(supplied)) throw new MemoryServiceError("forbidden", "source_session_scope_conflict"); + if (supplied.status === "open") return supplied; + if (!canStartAfter(supplied)) return { status: "pending", reason: "source_session_closed" }; + } + const candidates = this.deps.repos.runtime.sourceConversationSessions(scope).filter(inScope); + const open = candidates.filter((candidate) => candidate.status === "open"); + if (open.length > 1) return { status: "pending", reason: "source_session_ambiguous" }; + if (candidates.some((candidate) => candidate.status !== "open" && !canStartAfter(candidate))) { + return { status: "pending", reason: "source_session_closed" }; + } + if (open[0]) return open[0]; + // The source lookup has already made the strict reuse decision. The legacy host-key + // lookup does not include every namespace dimension and must not run a second time. + const opened = this.openSession({ + namespace: { ...namespace, sessionKey: identity.conversationId }, + source: identity.source, + profileId: identity.profileId, + workspacePath: request.workspacePath, + meta: { conversationId: identity.conversationId, source_namespace_key: scope.namespaceKey }, + timeZone: request.timeZone + }, { createNew: true, at: identity.startedAt }); + return this.deps.repos.runtime.getSession(opened.sessionId)!; + } + + completeTurn( + turnId: string, + request: TurnCompleteRequest & Record, + sourceTurn?: SourceTurnIdentity + ): CompleteTurnResponse { request = sanitizeTurnCompleteRequest(request); if (request.status === "cancelled") { throw new MemoryServiceError("invalid_argument", "cancelled turns are not persisted"); @@ -1219,7 +1444,7 @@ export class SessionTurnService { return this.deps.completeTurnNoWrite(turnId, request); } const startedAt = Date.now(); - const idempotencyKey = request.adapterId && request.requestId + const idempotencyKey = !sourceTurn && request.adapterId && request.requestId ? `turn.complete:${request.adapterId}:${request.requestId}` : undefined; const requestHash = stableHash({ @@ -1253,7 +1478,7 @@ export class SessionTurnService { if (existingRawTurn && isRecord(existingRawTurn.messagePayload?.turn_complete)) { const at = nowIso(); const episode = this.deps.requireEpisode(existingRawTurn.episodeId); - const existingCaptureClaim = existingRawTurn.userText && existingRawTurn.assistantText + const existingCaptureClaim = !sourceTurn && existingRawTurn.userText && existingRawTurn.assistantText ? this.deps.repos.captureClaims.get( session.userId, normalizeMemoryCaptureSource(session.source), @@ -1315,7 +1540,7 @@ export class SessionTurnService { const endTopicDecision = explicitEndTopicDecision(request.query) ?? (existingRawTurn ? endTopicDecisionFromRawTurn(existingRawTurn) : undefined); - const at = nowIso(); + const at = sourceTurn?.completedAt ?? nowIso(); const recalledProposal = turnRouteProposalFromRecallRequest(turnStartRecall?.request); let route: CommittedTurnRoute; if (request.episodeId) { @@ -1352,6 +1577,7 @@ export class SessionTurnService { !this.episodeRelationContext(latest).prevUserText ); const proposalIsCurrent = Boolean(recalledProposal) && + !(sourceTurn && latest && latest.status !== "open") && (recalledProposal?.baseEpisodeId === latest?.id || proposalUsesObservedUnboundEpisode) && !(recalledProposal?.action === "append" && latest?.status === "closed" && @@ -1382,14 +1608,15 @@ export class SessionTurnService { } else { const proposal = proposalIsCurrent ? recalledProposal! - : this.proposeEpisodeRoute(session, request.query, endTopicDecision); + : this.proposeEpisodeRoute(session, request.query, endTopicDecision, sourceTurn?.startedAt); route = this.commitTurnRouteProposal( session, proposal, request.query, "turn.complete", at, - !proposalIsCurrent + !proposalIsCurrent, + sourceTurn?.startedAt ); } } @@ -1405,7 +1632,12 @@ export class SessionTurnService { at ); } - this.deps.repos.runtime.touchSession(session.id, at); + let activityAt = at; + if (sourceTurn) { + activityAt = new Date(Math.max(Date.parse(at), Date.parse(session.lastSeenAt ?? session.updatedAt), + Date.parse(episode.updatedAt))).toISOString(); + } + this.deps.repos.runtime.touchSession(session.id, activityAt); const rawTurnId = rawTurnIdForSessionTurn(session.id, turnId); const requestToolCalls = normalizeCompleteTurnToolCalls(completionRequest); const requestToolResults = normalizeCompleteTurnToolResults(completionRequest); @@ -1453,6 +1685,7 @@ export class SessionTurnService { sourceMemoryIds, usage: isRecord(request.usage) ? request.usage : {}, messagePayload: { + ...(sourceTurn ? { source_turn: sourceTurn } : {}), turn_start: turnStartPayload, turn_complete: { completed_at: at, @@ -1461,7 +1694,7 @@ export class SessionTurnService { } }, status: request.status ?? "succeeded", - createdAt: at + createdAt: sourceTurn?.startedAt ?? at }); const rawTurnCreated = !existingRawTurn; const rawTurnFirstCompleted = rawTurnCreated @@ -1469,6 +1702,7 @@ export class SessionTurnService { const completedObservedRawTurn = existingRawTurn ? { ...completeObservedRawTurn(existingRawTurn, completionRequest, at), + ...(sourceTurn ? { createdAt: sourceTurn.startedAt } : {}), episodeId: episode.id } : undefined; @@ -1477,6 +1711,7 @@ export class SessionTurnService { ...completedObservedRawTurn, messagePayload: { ...completedObservedRawTurn.messagePayload, + ...(sourceTurn ? { source_turn: sourceTurn } : {}), turn_start: turnStartPayload } }) @@ -1514,11 +1749,11 @@ export class SessionTurnService { intentDecision }); } - this.deps.repos.runtime.appendEpisodeRawTurn(episode.id, rawTurn.id, at); + this.deps.repos.runtime.appendEpisodeRawTurn(episode.id, rawTurn.id, activityAt); const userMemoryCapture = this.captureUserMemory(rawTurn, request, at); const requestTags = this.deps.normalizeRequestTags(request.tags); - const capturedSteps = this.captureEpisodeIncrementalSteps(episode, rawTurn, at) + const capturedSteps = this.captureEpisodeIncrementalSteps(episode, rawTurn, at, Boolean(sourceTurn)) .map((step) => { const stepRawTurnId = step.rawTurnId ?? rawTurn.id; return stepRawTurnId === rawTurn.id && requestTags.length > 0 @@ -1624,7 +1859,7 @@ export class SessionTurnService { createdAt: at }); - let captureClaimed = captureClaimByRawTurnId.get(stepRawTurnId); + let captureClaimed = sourceTurn ? true : captureClaimByRawTurnId.get(stepRawTurnId); if (captureClaimed === undefined) { const qaQuery = sourceRawTurn.userText ?? ""; const qaAnswer = sourceRawTurn.assistantText ?? ""; @@ -1655,7 +1890,7 @@ export class SessionTurnService { createdAt: at }); } - this.deps.repos.runtime.appendEpisodeTurn(episode.id, stepRawTurnId, existing.id, at); + this.deps.repos.runtime.appendEpisodeTurn(episode.id, stepRawTurnId, existing.id, activityAt); } } } else { @@ -1690,7 +1925,7 @@ export class SessionTurnService { source: "turn.complete.capture.v7", createdAt: at }); - this.deps.repos.runtime.appendEpisodeTurn(episode.id, stepRawTurnId, upsert.memory.id, at); + this.deps.repos.runtime.appendEpisodeTurn(episode.id, stepRawTurnId, upsert.memory.id, activityAt); const existingProcessing = this.deps.repos.processing.get(upsert.memory.id); const contentChanged = Boolean( !upsert.created && upsert.previous?.contentHash !== upsert.memory.contentHash @@ -2550,7 +2785,8 @@ export class SessionTurnService { private captureEpisodeIncrementalSteps( episode: EpisodeRecord, currentRawTurn: RawTurnRecord, - at: string + at: string, + currentTurnOnly = false ): ReturnType { const seenRawTurnIds = new Set( episode.l1MemoryIds @@ -2559,7 +2795,7 @@ export class SessionTurnService { .map((memory) => this.deps.rawTurnIdFromMemory(memory)) .filter((id): id is string => Boolean(id)) ); - const rawTurns = uniq([...episode.rawTurnIds, currentRawTurn.id]) + const rawTurns = uniq(currentTurnOnly ? [currentRawTurn.id] : [...episode.rawTurnIds, currentRawTurn.id]) .map((id) => id === currentRawTurn.id ? currentRawTurn : this.deps.repos.runtime.getRawTurn(id)) .filter((rawTurn): rawTurn is RawTurnRecord => Boolean(rawTurn && (rawTurn.id === currentRawTurn.id || !seenRawTurnIds.has(rawTurn.id))) @@ -2742,7 +2978,8 @@ export class SessionTurnService { private proposeEpisodeRoute( session: SessionRecord, userText: string, - forcedDecision?: TurnRelationDecision + forcedDecision?: TurnRelationDecision, + at = nowIso() ): TurnRouteProposal { const latest = this.deps.repos.runtime.latestEpisodeForSession(session.id); const relationContext = latest ? this.episodeRelationContext(latest) : undefined; @@ -2751,11 +2988,11 @@ export class SessionTurnService { prevAssistantText: relationContext?.prevAssistantText ?? "", newUserText: userText, gapMs: relationContext?.lastTurnAtMs - ? Math.max(0, Date.now() - relationContext.lastTurnAtMs) + ? Math.max(0, Date.parse(at) - relationContext.lastTurnAtMs) : undefined, prevTags: relationContext?.tags ?? [] }); - return this.buildTurnRouteProposal(latest, decision, relationContext?.lastTurnAtMs); + return this.buildTurnRouteProposal(latest, decision, relationContext?.lastTurnAtMs, at); } private async proposeEpisodeRouteWithLlm( @@ -2806,14 +3043,15 @@ export class SessionTurnService { userText: string, source: string, at: string, - proposalStale: boolean + proposalStale: boolean, + sourceStartedAt?: string ): CommittedTurnRoute { const decision = proposal.relationDecision; const closedEpisodeIds: string[] = []; const jobs: EvolutionJobRecord[] = []; if (proposal.action === "create_first") { return { - episode: this.ensureEpisode(session), + episode: this.ensureEpisode(session, undefined, sourceStartedAt), closedEpisodeIds, jobs, proposal, @@ -2946,7 +3184,7 @@ export class SessionTurnService { }); } } - const next = this.ensureEpisode(session); + const next = this.ensureEpisode(session, undefined, sourceStartedAt); const episode = this.deps.repos.runtime.updateEpisodeMeta(next.id, { relation: decision.relation, relationDecision: decision, @@ -3194,7 +3432,7 @@ export class SessionTurnService { }); } - ensureEpisode(session: SessionRecord, episodeId?: string): EpisodeRecord { + ensureEpisode(session: SessionRecord, episodeId?: string, at = nowIso()): EpisodeRecord { if (episodeId) { const existing = this.deps.repos.runtime.getEpisode(episodeId); if (existing) { @@ -3207,7 +3445,6 @@ export class SessionTurnService { return latest; } - const at = nowIso(); const episode = this.deps.repos.runtime.createEpisode({ id: episodeId ?? newId("episode"), sessionId: session.id, diff --git a/Memory/src/service/turn/turn-normalization.ts b/Memory/src/service/turn/turn-normalization.ts index 69e483554..0489ef38b 100644 --- a/Memory/src/service/turn/turn-normalization.ts +++ b/Memory/src/service/turn/turn-normalization.ts @@ -2,6 +2,7 @@ import type { MemoryAddRequest, RepairSuggestionRequest, ToolCallPayload, TurnCo import type { RawTurnRecord } from "../../storage/repositories.js"; import { buildPluginRetrievalQuery, isStandaloneMathFinalAnswerTask, STANDALONE_MATH_FINAL_ANSWER_TASK_KIND } from "../../algorithm/plugin-algorithms.js"; import { stableHash } from "../../utils/id.js"; +import { matchToolResultIndices, toolInvocationId } from "../../utils/tool-call-pairing.js"; import { isMemmyRecallToolName, memmyRecallToolPlaceholder, sanitizeMemmyProtocolText, sanitizeMemmyProtocolValue } from "../../utils/memmy-context-tags.js"; export interface NormalizedCompleteTurnArtifact { kind: string; uri?: string; payload: Record; } @@ -11,19 +12,76 @@ export function turnStartContextHints(request: TurnStartRequest): Record(); for (const item of value) { const keyword = String(item ?? "").trim(); const normalized = keyword.toLowerCase(); if (!keyword || seen.has(normalized)) continue; seen.add(normalized); out.push(keyword); if (out.length >= 5) break; } return out; } export function buildRepairSuggestionQuery(request: RepairSuggestionRequest): string { const error = errorMessageFromUnknown(request.error); const pluginQuery = request.toolName ? buildPluginRetrievalQuery({ reason: "decision_repair", failingTool: request.toolName, failureCount: 1, lastErrorCode: error }).text : ""; return [pluginQuery, request.issue, error, request.context].filter(Boolean).join("\n"); } export function sanitizeTurnStartRequest>(request: T): T { return { ...request, query: sanitizeMemmyProtocolText(String(request.query ?? "")) }; } -export function sanitizeTurnCompleteRequest>(request: T): T { const toolCalls = Array.isArray(request.toolCalls) ? request.toolCalls : []; return { ...request, query: sanitizeMemmyProtocolText(String(request.query ?? "")), answer: sanitizeMemmyProtocolText(String(request.answer ?? "")), toolCalls: Array.isArray(request.toolCalls) ? request.toolCalls.map(sanitizeMemmyProtocolValue) : request.toolCalls, toolResults: Array.isArray(request.toolResults) ? request.toolResults.map((result, index) => sanitizeCompleteTurnToolResult(result, toolNameFromToolCall(toolCalls[index]))) : request.toolResults }; } +export function sanitizeTurnCompleteRequest>(request: T): T { + const toolCalls = Array.isArray(request.toolCalls) ? request.toolCalls : []; + const toolResults = Array.isArray(request.toolResults) ? request.toolResults : []; + const pairedToolNames = new Map(); + matchToolResultIndices(toolCalls, toolResults).forEach((resultIndex, callIndex) => { + if (resultIndex !== undefined) pairedToolNames.set(resultIndex, toolNameFromToolCall(toolCalls[callIndex])); + }); + return { + ...request, + query: sanitizeMemmyProtocolText(String(request.query ?? "")), + answer: sanitizeMemmyProtocolText(String(request.answer ?? "")), + toolCalls: Array.isArray(request.toolCalls) ? toolCalls.map(sanitizeMemmyProtocolValue) : request.toolCalls, + toolResults: Array.isArray(request.toolResults) + ? toolResults.map((result, index) => sanitizeCompleteTurnToolResult(result, pairedToolNames.get(index))) + : request.toolResults + }; +} export function sanitizeMemoryAddRequest(request: T): T { return { ...request, content: sanitizeMemmyProtocolText(request.content ?? ""), title: typeof request.title === "string" ? sanitizeMemmyProtocolText(request.title) : request.title }; } export function completeObservedRawTurn(existing: RawTurnRecord, request: TurnCompleteRequest & Record, completedAt: string): RawTurnRecord { const toolCalls = normalizeCompleteTurnToolCalls(request); const toolResults = normalizeCompleteTurnToolResults(request); const previousComplete = isRecord(existing.messagePayload?.turn_complete) ? existing.messagePayload.turn_complete : {}; return { ...existing, userText: request.query ?? existing.userText, assistantText: request.answer, reasoningSummary: stringFromMaybeRecord(request, "reasoningSummary") ?? existing.reasoningSummary, toolCalls: toolCalls.length ? toolCalls : existing.toolCalls, toolResults: toolResults.length ? toolResults : existing.toolResults, sourceMemoryIds: normalizeCompleteTurnSourceMemoryIds(request, existing.sourceMemoryIds), usage: isRecord(request.usage) ? request.usage : existing.usage, messagePayload: { ...(existing.messagePayload ?? {}), turn_complete: { completed_at: completedAt, source_memory_ids: normalizeCompleteTurnSourceMemoryIds(request, existing.sourceMemoryIds), time_zone: request.timeZone ?? stringFromMaybeRecord(previousComplete, "time_zone") } }, status: request.status ?? "succeeded" }; } export function normalizeCompleteTurnSourceMemoryIds(request: TurnCompleteRequest & Record, fallback: string[] = []): string[] { return Array.isArray(request.sourceMemoryIds) ? request.sourceMemoryIds.filter((value): value is string => typeof value === "string" && value.trim().length > 0) : fallback; } export function normalizeCompleteTurnArtifacts(request: TurnCompleteRequest): NormalizedCompleteTurnArtifact[] { if (!Array.isArray(request.artifacts)) return []; return request.artifacts.map((artifact) => { if (!isRecord(artifact)) return null; const normalized: NormalizedCompleteTurnArtifact = { kind: stringFromRecord(artifact, "kind") ?? "artifact", payload: artifact }; const uri = stringFromRecord(artifact, "uri"); if (uri) normalized.uri = uri; return normalized; }).filter((artifact): artifact is NormalizedCompleteTurnArtifact => Boolean(artifact)); } -export function normalizeCompleteTurnToolCalls(request: TurnCompleteRequest): ToolCallPayload[] { const results = normalizeCompleteTurnToolResults(request); return (Array.isArray(request.toolCalls) ? request.toolCalls : []).map((call, index) => normalizeCompleteTurnToolCall(call, results[index])).filter((call): call is ToolCallPayload => Boolean(call)); } +export function normalizeCompleteTurnToolCalls(request: TurnCompleteRequest): ToolCallPayload[] { + const calls = Array.isArray(request.toolCalls) ? request.toolCalls : []; + const results = normalizeCompleteTurnToolResults(request); + const resultIndices = matchToolResultIndices(calls, results); + return calls.map((call, index) => { + const resultIndex = resultIndices[index]; + return normalizeCompleteTurnToolCall(call, resultIndex === undefined ? undefined : results[resultIndex]); + }).filter((call): call is ToolCallPayload => Boolean(call)); +} export function normalizeCompleteTurnToolResults(request: TurnCompleteRequest): unknown[] { return Array.isArray(request.toolResults) ? request.toolResults : []; } export function rawTurnIdForSessionTurn(sessionId: string, turnId: string): string { return `raw_${stableHash(`${sessionId}:${turnId}`).slice(0, 20)}`; } -function sanitizeCompleteTurnToolResult(value: unknown, pairedToolName: string | undefined): unknown { const toolName = toolNameFromToolResult(value) ?? pairedToolName; if (!isMemmyRecallToolName(toolName)) return sanitizeMemmyProtocolValue(value); const output: Record = { name: toolName, output: memmyRecallToolPlaceholder(toolName) }; if (isRecord(value)) { const toolCallId = stringFromRecord(value, "toolCallId") ?? stringFromRecord(value, "tool_call_id") ?? stringFromRecord(value, "id"); if (toolCallId) output.toolCallId = toolCallId; } return output; } +function sanitizeCompleteTurnToolResult(value: unknown, pairedToolName: string | undefined): unknown { const toolName = toolNameFromToolResult(value) ?? pairedToolName; if (!isMemmyRecallToolName(toolName)) return sanitizeMemmyProtocolValue(value); const output: Record = { name: toolName, output: memmyRecallToolPlaceholder(toolName) }; if (isRecord(value)) { const toolCallId = toolInvocationId(value); if (toolCallId) output.toolCallId = toolCallId; } return output; } function toolNameFromToolCall(value: unknown): string | undefined { if (!isRecord(value)) return undefined; const fn = isRecord(value.function) ? value.function : {}; return stringFromRecord(value, "name") ?? stringFromRecord(value, "toolName") ?? stringFromRecord(fn, "name"); } function toolNameFromToolResult(value: unknown): string | undefined { return isRecord(value) ? stringFromRecord(value, "name") ?? stringFromRecord(value, "toolName") ?? stringFromRecord(value, "tool_name") : undefined; } -function normalizeCompleteTurnToolCall(value: unknown, pairedResult: unknown): ToolCallPayload | null { if (!isRecord(value)) return null; const fn = isRecord(value.function) ? value.function : {}; const name = stringFromRecord(value, "name") ?? stringFromRecord(fn, "name"); if (!name) return null; const result = isRecord(pairedResult) ? pairedResult : {}; const error = errorMessageFromUnknown(value.error) ?? errorMessageFromUnknown(result.error) ?? errorMessageFromUnknown(result.message); return { id: stringFromRecord(value, "id") ?? stringFromRecord(value, "call_id") ?? stringFromRecord(value, "tool_call_id"), name, input: firstDefined(value.input, value.args, value.arguments, fn.arguments), output: firstDefined(value.output, value.result, result.output, result.result, result.content, pairedResult), error, success: typeof value.success === "boolean" ? value.success : error ? false : undefined, startedAt: timeFromRecord(value, "startedAt") ?? timeFromRecord(value, "started_at"), endedAt: timeFromRecord(value, "endedAt") ?? timeFromRecord(value, "ended_at"), thinkingBefore: stringFromRecord(value, "thinkingBefore") ?? stringFromRecord(value, "thinking_before"), assistantTextBefore: stringFromRecord(value, "assistantTextBefore") ?? stringFromRecord(value, "assistant_text_before") }; } +function normalizeCompleteTurnToolCall(value: unknown, pairedResult: unknown): ToolCallPayload | null { + if (!isRecord(value)) return null; + const fn = isRecord(value.function) ? value.function : {}; + const name = toolNameFromToolCall(value); + if (!name) return null; + const result = isRecord(pairedResult) ? pairedResult : {}; + const error = errorMessageFromUnknown(value.error) ?? errorMessageFromUnknown(result.error) ?? + (result.success === false ? errorMessageFromUnknown(result.message) : undefined); + let success: boolean | undefined; + if (typeof value.success === "boolean") { + success = value.success; + } else if (typeof result.success === "boolean") { + success = result.success; + } else if (error) { + success = false; + } else { + success = undefined; + } + return { + id: toolInvocationId(value), + name, + input: firstDefined(value.input, value.args, value.arguments, fn.arguments), + output: firstDefined(value.output, value.result, result.output, result.result, result.content, pairedResult), + status: stringFromRecord(value, "status") ?? stringFromRecord(result, "status"), + error, + errorCode: stringFromRecord(value, "errorCode") ?? stringFromRecord(value, "error_code") ?? + stringFromRecord(result, "errorCode") ?? stringFromRecord(result, "error_code"), + success, + startedAt: timeFromRecord(value, "startedAt") ?? timeFromRecord(value, "started_at"), + endedAt: timeFromRecord(value, "endedAt") ?? timeFromRecord(value, "ended_at"), + thinkingBefore: stringFromRecord(value, "thinkingBefore") ?? stringFromRecord(value, "thinking_before"), + assistantTextBefore: stringFromRecord(value, "assistantTextBefore") ?? stringFromRecord(value, "assistant_text_before") + }; +} function firstDefined(...values: unknown[]): unknown { return values.find((value) => value !== undefined && value !== null); } function timeFromRecord(record: Record, key: string): string | undefined { const value = record[key]; return typeof value === "string" && value.trim() ? value : undefined; } function stringFromRecord(record: Record, key: string): string | undefined { const value = record[key]; return typeof value === "string" ? value : undefined; } diff --git a/Memory/src/storage/polardb.ts b/Memory/src/storage/polardb.ts index 76ea593b4..8df3278bb 100644 --- a/Memory/src/storage/polardb.ts +++ b/Memory/src/storage/polardb.ts @@ -1,5 +1,5 @@ -export const POLARDB_SCHEMA_VERSION = "runtime-v3"; -export const POLARDB_MIGRATION_ID = "003_memory_capture_claims"; +export const POLARDB_SCHEMA_VERSION = "runtime-v4"; +export const POLARDB_MIGRATION_ID = "004_source_turn_captures"; export function polardbMigrationSql(): string[] { return [ @@ -286,6 +286,35 @@ export function polardbMigrationSql(): string[] { created_at TIMESTAMPTZ NOT NULL, expires_at TIMESTAMPTZ )`, + `CREATE TABLE IF NOT EXISTS source_turn_captures ( + user_id TEXT NOT NULL, + source TEXT NOT NULL, + profile_id TEXT NOT NULL, + namespace_key TEXT NOT NULL, + conversation_id TEXT NOT NULL, + turn_id TEXT NOT NULL, + content_hash TEXT NOT NULL, + session_id TEXT, + episode_id TEXT, + raw_turn_id TEXT, + response JSONB NOT NULL, + started_at TIMESTAMPTZ NOT NULL, + completed_at TIMESTAMPTZ NOT NULL, + source_sequence INTEGER, + created_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (user_id, source, profile_id, namespace_key, conversation_id, turn_id) + )`, + `CREATE INDEX IF NOT EXISTS idx_source_turn_captures_conversation + ON source_turn_captures (user_id, source, profile_id, namespace_key, conversation_id, completed_at DESC)`, + + `CREATE TABLE IF NOT EXISTS runtime_kv ( + key TEXT PRIMARY KEY, + value JSONB NOT NULL, + updated_at TIMESTAMPTZ NOT NULL + )`, + `INSERT INTO runtime_kv (key, value, updated_at) + VALUES ('source_turn_capture_activated_at', to_jsonb(CURRENT_TIMESTAMP::text), CURRENT_TIMESTAMP) + ON CONFLICT(key) DO NOTHING`, `CREATE TABLE IF NOT EXISTS memory_capture_claims ( user_id TEXT NOT NULL, source TEXT NOT NULL, diff --git a/Memory/src/storage/repositories.ts b/Memory/src/storage/repositories.ts index 66afb5d60..649a72db9 100644 --- a/Memory/src/storage/repositories.ts +++ b/Memory/src/storage/repositories.ts @@ -29,6 +29,7 @@ import type { MemoryStatsRow, MemoryStatus, RecallHit, + SourceTurnCompleteResponse, UserMemoryRecord, UserMemoryStatus, UserMemoryType @@ -55,6 +56,7 @@ type SqlValue = string | number | Buffer | null; const BUNDLE_TABLES = [ "memories", "memory_capture_claims", + "source_turn_captures", "l3_world_model_scopes", "user_memories", "sessions", @@ -81,7 +83,7 @@ const BUNDLE_TABLES = [ "audit_logs" ] as const; const CLEAR_MEMORY_TABLES = [ - ...BUNDLE_TABLES, + ...BUNDLE_TABLES.filter((table) => table !== "source_turn_captures"), "memories_fts", "user_memories_fts", "memory_vector_entries", @@ -1678,6 +1680,41 @@ export class MemoryProcessingRepository { } } +export interface SourceTurnCaptureScope { + userId: string; + source: string; + profileId: string; + namespaceKey: string; + conversationId: string; +} + +export interface SourceTurnCaptureRecord extends SourceTurnCaptureScope { + turnId: string; + contentHash: string; + sessionId?: string; + episodeId?: string; + rawTurnId?: string; + response: SourceTurnCompleteResponse; + startedAt: string; + completedAt: string; + sequence?: number; + createdAt: string; +} + +function sourceTurnCaptureFromSql(row: Record): SourceTurnCaptureRecord { + return { + userId: String(row.user_id), source: String(row.source), profileId: String(row.profile_id), + namespaceKey: String(row.namespace_key), conversationId: String(row.conversation_id), turnId: String(row.turn_id), + contentHash: String(row.content_hash), sessionId: typeof row.session_id === "string" ? row.session_id : undefined, + episodeId: typeof row.episode_id === "string" ? row.episode_id : undefined, + rawTurnId: typeof row.raw_turn_id === "string" ? row.raw_turn_id : undefined, + response: parseJson(String(row.response_json), { status: "pending", reason: "source_capture_response_missing" }), + startedAt: String(row.started_at), completedAt: String(row.completed_at), + sequence: typeof row.source_sequence === "number" ? row.source_sequence : undefined, + createdAt: String(row.created_at) + }; +} + export class RuntimeRepository { private readonly scheduledLogPrunes = new Set(); @@ -1719,6 +1756,90 @@ export class RuntimeRepository { })); } + getSourceTurnCapture(scope: SourceTurnCaptureScope, turnId: string): SourceTurnCaptureRecord | undefined { + const row = this.db.prepare(`SELECT * FROM source_turn_captures + WHERE user_id = @userId AND source = @source AND profile_id = @profileId + AND namespace_key = @namespaceKey AND conversation_id = @conversationId AND turn_id = @turnId` + ).get({ ...scope, turnId }) as Record | undefined; + return row ? sourceTurnCaptureFromSql(row) : undefined; + } + + latestSourceTurnCapture(scope: SourceTurnCaptureScope): SourceTurnCaptureRecord | undefined { + const row = this.db.prepare(`SELECT * FROM source_turn_captures + WHERE user_id = @userId AND source = @source AND profile_id = @profileId + AND namespace_key = @namespaceKey AND conversation_id = @conversationId AND session_id IS NOT NULL + ORDER BY completed_at DESC, source_sequence DESC LIMIT 1` + ).get(scope) as Record | undefined; + return row ? sourceTurnCaptureFromSql(row) : undefined; + } + + sourceTurnCaptureNeighbors(scope: SourceTurnCaptureScope, startedAt: string): { + before?: SourceTurnCaptureRecord; + after?: SourceTurnCaptureRecord; + } { + const prefix = `SELECT * FROM source_turn_captures + WHERE user_id = @userId AND source = @source AND profile_id = @profileId + AND namespace_key = @namespaceKey AND conversation_id = @conversationId AND session_id IS NOT NULL`; + const before = this.db.prepare(`${prefix} AND started_at < @startedAt ORDER BY started_at DESC LIMIT 1`) + .get({ ...scope, startedAt }) as Record | undefined; + const after = this.db.prepare(`${prefix} AND started_at > @startedAt ORDER BY started_at ASC LIMIT 1`) + .get({ ...scope, startedAt }) as Record | undefined; + return { before: before ? sourceTurnCaptureFromSql(before) : undefined, + after: after ? sourceTurnCaptureFromSql(after) : undefined }; + } + + insertSourceTurnCapture(capture: SourceTurnCaptureRecord): void { + this.db.prepare(`INSERT INTO source_turn_captures ( + user_id, source, profile_id, namespace_key, conversation_id, turn_id, content_hash, + session_id, episode_id, raw_turn_id, response_json, started_at, completed_at, source_sequence, created_at + ) VALUES (@userId, @source, @profileId, @namespaceKey, @conversationId, @turnId, @contentHash, + @sessionId, @episodeId, @rawTurnId, @responseJson, @startedAt, @completedAt, @sequence, @createdAt)` + ).run({ ...capture, sessionId: capture.sessionId ?? null, episodeId: capture.episodeId ?? null, + rawTurnId: capture.rawTurnId ?? null, responseJson: toJson(capture.response), sequence: capture.sequence ?? null }); + } + + orderEpisodeTurnsBySourceTime(episodeId: string): void { + const episode = this.getEpisode(episodeId); + if (!episode) throw new Error(`episode not found: ${episodeId}`); + const rawTimes = new Map(episode.rawTurnIds.map((id) => [id, this.getRawTurn(id)?.createdAt ?? ""])); + const rawTurnIds = [...episode.rawTurnIds].sort((a, b) => rawTimes.get(a)!.localeCompare(rawTimes.get(b)!)); + const memoryTime = this.db.prepare(`SELECT created_at, + COALESCE(json_extract(info_json, '$.raw_turn_id'), json_extract(properties_json, '$.internal_info.raw_turn_id')) AS raw_turn_id + FROM memories WHERE id = ?`); + const memoryTimes = new Map(episode.l1MemoryIds.map((id) => { + const row = memoryTime.get(id) as { created_at: string; raw_turn_id: string | null } | undefined; + return [id, rawTimes.get(row?.raw_turn_id ?? "") ?? row?.created_at ?? ""]; + })); + const l1MemoryIds = [...episode.l1MemoryIds].sort((a, b) => memoryTimes.get(a)!.localeCompare(memoryTimes.get(b)!)); + this.db.prepare(`UPDATE episodes SET raw_turn_ids_json = ?, l1_memory_ids_json = ? WHERE id = ?`) + .run(toJson(rawTurnIds), toJson(l1MemoryIds), episodeId); + } + + sourceConversationSessions(input: { userId: string; source: string; profileId: string; conversationId: string }): SessionRecord[] { + return (this.db.prepare(`SELECT * FROM sessions WHERE user_id = @userId AND source = @source + AND profile_id = @profileId AND (host_session_key = @conversationId OR conversation_id = @conversationId + OR (@source = 'codex' AND host_session_key = 'codex-memory-' || @conversationId)) + ORDER BY opened_at DESC`).all(input) as SqlSessionRow[]).map(sessionFromSql); + } + + bindSessionSourceConversation(id: string, conversationId: string): boolean { + const result = this.db.prepare(`UPDATE sessions SET conversation_id = @conversationId + WHERE id = @id AND status = 'open' AND (conversation_id IS NULL OR conversation_id = '')`) + .run({ id, conversationId }); + return result.changes === 1; + } + + bindRawTurnSourceConversation( + scope: Pick, + conversationId: string + ): boolean { + const result = this.db.prepare(`UPDATE raw_turns SET conversation_id = @conversationId + WHERE id = @id AND session_id = @sessionId AND user_id = @userId AND turn_id = @turnId + AND (conversation_id IS NULL OR conversation_id = '' OR conversation_id = @conversationId)`) + .run({ ...scope, conversationId }); + return result.changes === 1; + } + createSession(session: SessionRecord): SessionRecord { this.db .prepare( @@ -4046,6 +4167,10 @@ export class RuntimeRepository { }> }; this.db.transaction(() => { + const hadRuntimeData = Boolean(this.db.prepare(`SELECT EXISTS( + SELECT 1 FROM memories UNION ALL SELECT 1 FROM sessions UNION ALL + SELECT 1 FROM raw_turns UNION ALL SELECT 1 FROM source_turn_captures + )`).pluck().get()); for (const table of BUNDLE_TABLES) { const rows = Array.isArray(tables[table]) ? tables[table] as Array> : []; for (const row of rows) { @@ -4055,6 +4180,12 @@ export class RuntimeRepository { recordMigrationMap(result.migrationMap, table, identity.sourceId, identity.sourceId); } const existed = identity !== undefined && this.rowExists(table, identity.columns, identity.values); + if (table === "runtime_kv" && normalized.key === "source_turn_capture_activated_at") { + const outcome = this.mergeSourceTurnActivation(normalized, hadRuntimeData); + const counts = result[outcome]; + counts[table] = (counts[table] ?? 0) + 1; + continue; + } if (existed && conflictStrategy === "skip") { result.conflicts.push({ table, @@ -4106,6 +4237,30 @@ export class RuntimeRepository { return result; } + private mergeSourceTurnActivation( + row: Record, + hadRuntimeData: boolean + ): "inserted" | "replaced" | "skipped" { + const imported = typeof row.value_json === "string" ? parseJson(row.value_json, undefined) : undefined; + if (typeof imported !== "string" || !Number.isFinite(Date.parse(imported))) { + throw new Error("invalid source turn activation boundary in bundle"); + } + const key = "source_turn_capture_activated_at"; + const existing = this.getKv(key); + if (existing && (typeof existing.value !== "string" || !Number.isFinite(Date.parse(existing.value)))) { + throw new Error("invalid source turn activation boundary in target database"); + } + // A new database contains only the automatically initialized boundary. A full restore + // inherits the backup's original start. Merging into existing data must never widen + // the historical capture window of either database; known identities remain reusable. + if (existing && (existing.value === imported || + (hadRuntimeData && Date.parse(existing.value as string) >= Date.parse(imported)))) { + return "skipped"; + } + this.setKv(key, imported, typeof row.updated_at === "string" ? row.updated_at : nowIso()); + return existing ? "replaced" : "inserted"; + } + private rowExists(table: BundleTableName, columns: string[], values: Array): boolean { const tableColumns = this.tableColumns(table); if (columns.length === 0 || columns.some((column) => !tableColumns.includes(column))) { @@ -5168,8 +5323,14 @@ export class Repositories { return this.db.transaction(() => { const cleared: Record = {}; for (const table of tables) { - cleared[table] = this.db.prepare(`DELETE FROM "${table}"`).run().changes; + if (table === "runtime_kv") { + cleared[table] = this.db.prepare(`DELETE FROM runtime_kv WHERE key != 'source_turn_capture_activated_at'`).run().changes; + } else { + cleared[table] = this.db.prepare(`DELETE FROM "${table}"`).run().changes; + } } + // Keep source tombstones, but a deleted Session must not block future turns. + this.db.prepare(`UPDATE source_turn_captures SET session_id = NULL, episode_id = NULL, raw_turn_id = NULL`).run(); if (existing.has("sqlite_sequence")) { const sequenceTables = tables.filter((table) => !table.startsWith("memory_vec_")); if (sequenceTables.length) { @@ -6890,6 +7051,7 @@ function bundleIdentity( ): BundleIdentity | undefined { const newTableIdentityColumns: Partial> = { memory_capture_claims: ["user_id", "source", "qa_hash"], + source_turn_captures: ["user_id", "source", "profile_id", "namespace_key", "conversation_id", "turn_id"], l3_world_model_scopes: ["scope_key"], l3_world_model_session_cursors: ["session_id"], l3_world_model_input_traces: ["session_id", "trace_seq"], diff --git a/Memory/src/storage/schema.ts b/Memory/src/storage/schema.ts index c583ef910..65b07f5d1 100644 --- a/Memory/src/storage/schema.ts +++ b/Memory/src/storage/schema.ts @@ -1,8 +1,8 @@ import type Database from "better-sqlite3"; import { memoryCaptureQaHash, normalizeMemoryCaptureSource } from "../utils/memory-capture-claim.js"; -export const SCHEMA_VERSION = 7; -export const SCHEMA_MIGRATION_ID = "007_memory_capture_claims"; +export const SCHEMA_VERSION = 8; +export const SCHEMA_MIGRATION_ID = "008_source_turn_captures"; const API_LOG_SOURCE_AGENT_MIGRATION_FROM_VERSION = 2; const PROCESSING_TAGS = new Set([ "摘要排队中", @@ -469,6 +469,27 @@ const statements = [ expires_at TEXT )`, + `CREATE TABLE IF NOT EXISTS source_turn_captures ( + user_id TEXT NOT NULL, + source TEXT NOT NULL, + profile_id TEXT NOT NULL, + namespace_key TEXT NOT NULL, + conversation_id TEXT NOT NULL, + turn_id TEXT NOT NULL, + content_hash TEXT NOT NULL, + session_id TEXT, + episode_id TEXT, + raw_turn_id TEXT, + response_json TEXT NOT NULL CHECK (json_valid(response_json)), + started_at TEXT NOT NULL, + completed_at TEXT NOT NULL, + source_sequence INTEGER, + created_at TEXT NOT NULL, + PRIMARY KEY (user_id, source, profile_id, namespace_key, conversation_id, turn_id) + )`, + `CREATE INDEX IF NOT EXISTS idx_source_turn_captures_conversation + ON source_turn_captures (user_id, source, profile_id, namespace_key, conversation_id, completed_at DESC)`, + `CREATE TABLE IF NOT EXISTS memory_capture_claims ( user_id TEXT NOT NULL, source TEXT NOT NULL, @@ -620,7 +641,7 @@ export function migrate(db: Database.Database): void { const hasMemories = tableExists(db, "memories"); const version = currentSchemaVersion(db); - if (hasMemories && version !== SCHEMA_VERSION && version !== 2 && version !== 3 && version !== 4 && version !== 5 && version !== 6) { + if (hasMemories && version !== SCHEMA_VERSION && version !== 2 && version !== 3 && version !== 4 && version !== 5 && version !== 6 && version !== 7) { throw new Error( `Unsupported memory database schema version ${version}; the database was left unchanged` ); @@ -658,6 +679,11 @@ export function migrate(db: Database.Database): void { WHERE dedupe_key IS NOT NULL AND status IN ('queued', 'leased', 'failed')` ).run(); + // This boundary belongs to the Memory database and is shared by Hook and scanners. + db.prepare(`INSERT INTO runtime_kv (key, value_json, updated_at) + VALUES ('source_turn_capture_activated_at', ?, ?) + ON CONFLICT(key) DO NOTHING`).run(JSON.stringify(now), now); + if (hasMemories && version > 0 && version < 5) { backfillMemoryProcessingState(db, now); removeLegacyProcessingMetadata(db); diff --git a/Memory/src/types.ts b/Memory/src/types.ts index 1fc0daf10..511ec32ac 100644 --- a/Memory/src/types.ts +++ b/Memory/src/types.ts @@ -302,6 +302,7 @@ export interface ToolCallPayload { name: string; input?: unknown; output?: unknown; + status?: string; error?: string; errorCode?: string; success?: boolean; @@ -362,6 +363,50 @@ export interface TurnCompleteRequest extends RequestEnvelope { }; } +/** A completed native source turn; channel is deliberately excluded from its identity. */ +export interface SourceTurnIdentity { + source: string; + profileId: string; + conversationId: string; + turnId: string; + startedAt: IsoTime; + completedAt: IsoTime; + sequence?: number; + completionEvidence: string; +} + +export interface SourceTurnCompleteRequest extends Omit { + sessionId?: string; + sourceTurn: SourceTurnIdentity; + channel: "hook" | "agent_source_scan"; + workspacePath?: string; +} + +export interface TurnCompletionResult { + turnId: string; + sessionId: string; + episodeId: string; + rawTurnId: string; + userMemoryId: string; + userMemoryIds: string[]; + l1MemoryId: string; + l1MemoryIds: string[]; + closedEpisodeIds: string[]; + scheduledEvolution: boolean; + jobs: JobRef[]; + changeSeq: number; + syncCursor: string; + etag: string; + serverTime: string; + duplicate?: boolean; +} + +export interface SourceTurnCompleteResponse { + status: "stored" | "existing" | "rejected" | "pending" | "conflict"; + reason?: string; + result?: TurnCompletionResult; +} + export type UserMemoryType = "User Fact" | "User Preference" | "User Directive"; export type UserMemoryStatus = "active" | "archived" | "deleted"; diff --git a/Memory/src/utils/tool-call-pairing.ts b/Memory/src/utils/tool-call-pairing.ts new file mode 100644 index 000000000..7bf12687c --- /dev/null +++ b/Memory/src/utils/tool-call-pairing.ts @@ -0,0 +1,39 @@ +/** Source-native invocation IDs take precedence over generic message/record IDs. */ +export function toolInvocationId(value: unknown): string | undefined { + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + const record = value as Record; + for (const key of ["call_id", "tool_call_id", "toolCallId", "id"]) { + const id = record[key]; + if (typeof id === "string" && id.trim()) return id; + } + return undefined; +} + +/** Match records from one turn only; callers are responsible for the turn boundary. */ +export function matchToolResultIndices(toolCalls: unknown[], toolResults: unknown[]): Array { + const callIds = toolCalls.map(toolInvocationId); + const resultIds = toolResults.map(toolInvocationId); + // Older adapters supplied positional arrays without any invocation IDs. Keep + // that form only when neither array has IDs and both arrays retain equal slot counts. + if (toolCalls.length === toolResults.length && callIds.every((id) => !id) && resultIds.every((id) => !id)) { + return toolCalls.map((_, index) => index); + } + + const callCounts = new Map(); + for (const id of callIds) { + if (id) callCounts.set(id, (callCounts.get(id) ?? 0) + 1); + } + const resultsById = new Map(); + resultIds.forEach((id, index) => { + if (!id) return; + const indices = resultsById.get(id) ?? []; + indices.push(index); + resultsById.set(id, indices); + }); + return callIds.map((id) => { + if (!id || callCounts.get(id) !== 1) return undefined; + const indices = resultsById.get(id); + // Duplicate IDs are ambiguous; never guess by position or consume another ID. + return indices?.length === 1 ? indices[0] : undefined; + }); +} diff --git a/Memory/tests/agent-source-scan-store.test.ts b/Memory/tests/agent-source-scan-store.test.ts new file mode 100644 index 000000000..da8e91814 --- /dev/null +++ b/Memory/tests/agent-source-scan-store.test.ts @@ -0,0 +1,50 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { openMemoryAgentSourceScanStore } from "../src/agent-source/scan-store.js"; + +let directory: string | undefined; +afterEach(() => { if (directory) rmSync(directory, { recursive: true, force: true }); directory = undefined; }); + +describe("standalone durable scan store retry", () => { + it("refreshes unresolved Codex rows in the same job without counting them as new messages", async () => { + directory = mkdtempSync(join(tmpdir(), "memmy-codex-scan-retry-")); + const path = join(directory, "job.sqlite"); + const job = { jobId: "job", sourceId: "codex", mode: "incremental", phase: "failed", createdAt: "2026-09-09", updatedAt: "2026-09-09" }; + let store = await openMemoryAgentSourceScanStore(path, job); + const pending = { messageId: "rollout:000000000009", sourceId: "codex", conversationId: "fallback-conversation", role: "assistant" as const, content: "Draft", createdAt: "2026-09-09T00:00:00Z", workspacePath: null, gitRoot: null, rawMeta: { sourceTurnState: "identity_unresolved", sourceTurnReason: "identity_unresolved" } }; + expect(store.stage(pending)).toBe(true); + const stagedOrdinal = [...store.messages("codex")][0]!.ordinal; + store.saveResult({ sourceId: "codex", conversationId: pending.conversationId, error: "identity_unresolved" }); + store.close(); + store = await openMemoryAgentSourceScanStore(path, job); + const completed = { ...pending, conversationId: "native-conversation", content: "Final answer", workspacePath: "/tmp/project", rawMeta: { sourceTurnState: "complete", sourceTurnId: "turn-native", sourceTurn: { turnId: "turn-native", completionEvidence: "task_complete:turn-native" } } }; + const next = { ...completed, messageId: "rollout:000000000010", content: "Next answer" }; + expect(store.stageBatch([completed, next])).toBe(1); + expect(store.stage(completed)).toBe(false); + expect(store.count("codex")).toBe(2); + const rows = [...store.messages("codex")]; + expect(rows[0]).toMatchObject({ ...completed, ordinal: stagedOrdinal }); + expect(rows[0]!.rawMeta).not.toHaveProperty("sourceTurnReason"); + expect([...store.results("codex")]).toEqual([{ sourceId: "codex", conversationId: pending.conversationId, error: "identity_unresolved" }]); + store.close(); + store = await openMemoryAgentSourceScanStore(path, job); + expect([...store.messages("codex")][0]).toMatchObject(completed); + store.close(); + }); + + it("keeps existing non-Codex staged rows and unrelated rows unchanged", async () => { + directory = mkdtempSync(join(tmpdir(), "memmy-scan-legacy-dedup-")); + const store = await openMemoryAgentSourceScanStore(join(directory, "job.sqlite"), { jobId: "job", sourceId: "all", mode: "full", phase: "stage", createdAt: "2026-09-09", updatedAt: "2026-09-09" }); + const message = { messageId: "shared-id", sourceId: "fixture", conversationId: "conversation", role: "user" as const, content: "Original", createdAt: "2026-09-09T00:00:00Z", workspacePath: null, gitRoot: null, rawMeta: {} }; + const codex = { ...message, sourceId: "codex", rawMeta: { sourceTurnState: "turn_incomplete" } }; + expect(store.stageBatch([message, codex])).toBe(2); + expect(store.stage({ ...message, content: "Revised" })).toBe(false); + expect(store.stage({ ...codex, content: "Complete", rawMeta: { sourceTurnState: "complete" } })).toBe(false); + expect([...store.messages("fixture")][0]!.content).toBe("Original"); + expect([...store.messages("codex")][0]!.content).toBe("Complete"); + expect(store.count()).toBe(2); + store.close(); + }); +}); diff --git a/Memory/tests/codex-native-hook.test.ts b/Memory/tests/codex-native-hook.test.ts new file mode 100644 index 000000000..c1c599045 --- /dev/null +++ b/Memory/tests/codex-native-hook.test.ts @@ -0,0 +1,131 @@ +import { spawn } from "node:child_process"; +import { mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildSourceTurnRequest, readCodexSourceTurn } from "@memmy/agent-source-core"; +import { renderMemmyResumeHookScript } from "../src/agent-source/integration/templates/memmy-resume-hook.js"; +import { loadMemmyWorkspaceBridgeRuntimeAsset } from "../src/agent-source/integration/workspace-bridge/runtime-loader.js"; + +const directories: string[] = []; +afterEach(() => { for (const dir of directories.splice(0)) rmSync(dir, { recursive: true, force: true }); }); +const createdAt = "2026-09-09T10:00:00.000Z"; +const expectedNamespace = { source: "codex", profileId: "default", userId: "fixture-owner", sessionKey: "native-session" }; +function record(type: string, payload: unknown) { return JSON.stringify({ type, timestamp: createdAt, payload }); } +function transcript(complete = true) { + return [record("session_meta", { id: "native-session", cwd: "/project" }), record("event_msg", { type: "task_started", turn_id: "native-turn" }), + record("response_item", { type: "message", role: "user", content: [{ text: "Fix the parser" }] }), + record("response_item", { type: "custom_tool_call", call_id: "a", name: "read", input: { path: "parser.ts" } }), + record("response_item", { type: "custom_tool_call_output", call_id: "a", output: "source contents" }), + record("response_item", { type: "message", role: "assistant", phase: "final_answer", content: [{ text: "Fixed the parser" }] }), + ...(complete ? [record("event_msg", { type: "task_complete", turn_id: "native-turn" })] : [])].join("\n") + "\n"; +} +async function fixture(responseStatus = "stored") { + const dir = mkdtempSync(join(tmpdir(), "memmy-native-hook-")); directories.push(dir); + const requests: { path: string; body: Record }[] = []; + const server = createServer(async (req, res) => { + let data = ""; for await (const chunk of req) data += chunk; + requests.push({ path: req.url ?? "", body: data ? JSON.parse(data) : {} }); + res.setHeader("content-type", "application/json"); + if (req.url === "/api/v1/health") res.end(JSON.stringify({ features: {} })); + else if (req.url === "/api/v1/sessions/open") res.end(JSON.stringify({ sessionId: "runtime-session" })); + else if (req.url === "/api/v1/turns/start") res.end(JSON.stringify({ turnId: "native-turn", episodeId: "episode-1" })); + else res.end(JSON.stringify({ status: responseStatus, reason: responseStatus === "pending" ? "episode_closed" : undefined, result: { rawTurnId: "raw-1" } })); + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + const endpoint = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + writeFileSync(join(dir, "memmy-resume-hook.mjs"), renderMemmyResumeHookScript({ source: "codex", mode: "codex" })); + writeFileSync(join(dir, "memmy-workspace-bridge.mjs"), await loadMemmyWorkspaceBridgeRuntimeAsset()); + writeFileSync(join(dir, "memmy-memory-config.json"), JSON.stringify({ endpoint, userId: "fixture-owner", memmy_config_path: join(dir, "missing.yaml") })); + const path = join(dir, "transcript.jsonl"); writeFileSync(path, transcript()); + return { dir, path, requests, close: () => new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())) }; +} +async function run(dir: string, path: string, extra: Record = {}) { + return new Promise<{ status: number | null; stdout: string; stderr: string }>((resolve, reject) => { + const child = spawn(process.execPath, [join(dir, "memmy-resume-hook.mjs")]); + let stdout = ""; let stderr = ""; + child.stdout.on("data", chunk => { stdout += chunk; }); child.stderr.on("data", chunk => { stderr += chunk; }); + child.on("error", reject); child.on("close", status => resolve({ status, stdout, stderr })); + child.stdin.end(JSON.stringify({ hook_event_name: "Stop", session_id: "native-session", turn_id: "native-turn", transcript_path: path, ...extra })); + }); +} + +describe("installed Codex Hook native capture", () => { + it("submits exactly the scanner canonical content without pending state or opening a runtime Session", async () => { + const f = await fixture(); + try { + const result = await run(f.dir, f.path); + expect(result.status).toBe(0); expect(result.stderr).toBe(""); + expect(f.requests).toHaveLength(1); + const turn = (await readCodexSourceTurn(f.path)).turn!; + expect(f.requests[0]).toEqual({ path: "/api/v1/source-turns/complete", body: { ...buildSourceTurnRequest(turn, "hook"), namespace: expectedNamespace, adapterId: "memmy-codex-hook" } }); + expect(f.requests[0]?.body.toolCalls).toEqual([{ id: "a", name: "read", input: { path: "parser.ts" }, output: "source contents" }]); + } finally { await f.close(); } + }); + + it("does not post incomplete source content and logs a reason that scanning can later recover", async () => { + const f = await fixture(); + try { + writeFileSync(f.path, transcript(false).replace('"final_answer"', '"commentary"')); + expect((await run(f.dir, f.path)).stderr).toContain("turn_incomplete"); + expect(f.requests).toHaveLength(0); + writeFileSync(f.path, transcript()); + expect((await run(f.dir, f.path)).stderr).toBe(""); + expect(f.requests).toHaveLength(1); + } finally { await f.close(); } + }); + + it("captures Stop before task_complete using the same final-answer fields as the subsequent scan", async () => { + const f = await fixture(); + try { + writeFileSync(f.path, transcript(false)); + const hookResult = await run(f.dir, f.path); + expect(hookResult.stderr).toBe(""); + expect(f.requests).toHaveLength(1); + expect((await readCodexSourceTurn(f.path)).turn).toBeNull(); + const completedRecords = transcript().trim().split("\n").map(line => JSON.parse(line)); + completedRecords[completedRecords.length - 1].timestamp = "2026-09-09T10:00:02.000Z"; + writeFileSync(f.path, completedRecords.map(record => JSON.stringify(record)).join("\n") + "\n"); + const scanned = (await readCodexSourceTurn(f.path)).turn!; + expect(scanned.completionEvidence).toBe("final_answer:native-turn"); + expect(scanned.completedAt).toBe(createdAt); + expect(f.requests[0]?.body).toEqual({ ...buildSourceTurnRequest(scanned, "hook"), namespace: expectedNamespace, adapterId: "memmy-codex-hook" }); + } finally { await f.close(); } + }); + + it("requires the native turn ID when Stop arrives before task_complete", async () => { + const f = await fixture(); + try { + writeFileSync(f.path, transcript(false)); + expect((await run(f.dir, f.path, { turn_id: undefined })).stderr).toContain("turn_incomplete"); + expect(f.requests).toHaveLength(0); + } finally { await f.close(); } + }); + + it("preserves start state on pending completion and leaves a recoverable capture reason", async () => { + const f = await fixture("pending"); + try { + await run(f.dir, f.path, { hook_event_name: "UserPromptSubmit", prompt: "Fix the parser" }); + const stateFiles = readdirSync(f.dir).filter(name => name.startsWith("memmy-turn-state-")); + expect(stateFiles).toHaveLength(1); + writeFileSync(join(f.dir, "missing.yaml"), [ + "app:", " userId: switched-app-owner", "memmyMemory:", " userId: switched-memory-owner", "", + ].join("\n")); + const result = await run(f.dir, f.path); + expect(result.stderr).toContain("episode_closed"); + expect(readdirSync(f.dir)).toContain(stateFiles[0]); + expect(f.requests.at(-1)?.body.sessionId).toBe("runtime-session"); + expect(f.requests.at(-1)?.body.namespace).toEqual(expectedNamespace); + } finally { await f.close(); } + }); + it.each(["cancelled", "failed"])("does not turn an explicit %s Stop into a succeeded L1", async (status) => { + const f = await fixture(); + try { + writeFileSync(f.path, transcript(false)); + await run(f.dir, f.path, { status }); + expect(f.requests).toHaveLength(0); + } finally { await f.close(); } + }); + +}); diff --git a/Memory/tests/contract/source-turn-http.test.ts b/Memory/tests/contract/source-turn-http.test.ts new file mode 100644 index 000000000..34f070f86 --- /dev/null +++ b/Memory/tests/contract/source-turn-http.test.ts @@ -0,0 +1,81 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { buildSourceTurnRequest } from "@memmy/agent-source-core"; +import { DEFAULT_MEMMY_CONFIG, createMemoryHttpServer } from "../../src/index.js"; +import { Repositories } from "../../src/storage/repositories.js"; +import { createMemoryServiceFixture } from "../fixtures/memory-service-fixture.js"; + +const { cleanup, createTestService } = createMemoryServiceFixture(); +afterEach(cleanup); + +describe("source turn HTTP contract", () => { + it("accepts the real shared builder and principal namespace for both channels", async () => { + const { service, db } = createTestService({ config: { ...DEFAULT_MEMMY_CONFIG, userId: "configured-owner" } }); + const server = createMemoryHttpServer({ service, startAgentSourceAutomation: false, + auth: { mode: "dev", scopedApiKeys: { "scope-test-token": { + namespace: { source: "codex", profileId: "default", userId: "http-user" }, scopes: ["memory:read", "memory:write"] + } } } }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("expected TCP address"); + const payload = buildSourceTurnRequest({ source: "codex", conversationId: "http-conversation", turnId: "http-turn", + startedAt: "2099-01-01T10:00:00.000Z", completedAt: "2099-01-01T10:01:00.000Z", sequence: 1, + completionEvidence: "final_answer:http-turn", query: "Implement the source-turn endpoint transaction.", + answer: "The source-turn endpoint transaction is implemented.", status: "succeeded", toolCalls: [], toolResults: [] + }, "hook"); + const submit = (body: unknown) => fetch(`http://127.0.0.1:${address.port}/api/v1/source-turns/complete`, { + method: "POST", headers: { "Content-Type": "application/json", Authorization: "Bearer scope-test-token" }, body: JSON.stringify(body) + }); + try { + const responses = await Promise.all([submit(payload), submit({ ...payload, channel: "agent_source_scan" })]); + expect(responses.map((response) => response.status)).toEqual([200, 200]); + const bodies = await Promise.all(responses.map((response) => response.json())) as Array<{ status: string; result: { sessionId: string; rawTurnId: string; l1MemoryId: string } }>; + expect(bodies.map((body) => body.status).sort()).toEqual(["existing", "stored"]); + expect(bodies[0]!.result.rawTurnId).toBe(bodies[1]!.result.rawTurnId); + expect(bodies[0]!.result.l1MemoryId).toBe(bodies[1]!.result.l1MemoryId); + expect(new Repositories(db.db).runtime.getSession(bodies[0]!.result.sessionId)).toMatchObject({ userId: "http-user", source: "codex", profileId: "default" }); + const forbidden = await submit({ ...payload, sourceTurn: { ...payload.sourceTurn, profileId: "foreign-profile" } }); + expect(forbidden.status).toBe(403); + const foreignUser = await submit({ ...payload, namespace: { source: "codex", profileId: "default", userId: "foreign-user" } }); + expect(foreignUser.status).toBe(403); + } finally { await new Promise((resolve) => server.close(() => resolve())); } + }); + + it.each(["hook", "agent_source_scan"] as const)("uses the configured local owner and existing Hook Session when %s arrives first", async (channel) => { + const { service, db } = createTestService({ config: { ...DEFAULT_MEMMY_CONFIG, userId: "configured-owner" } }); + const opened = service.openSession({ + source: "codex", namespace: { source: "codex", profileId: "default", userId: "configured-owner", sessionKey: "codex-memory-local-conversation" }, + l3WorldModelProtocolVersion: 2, l3WorldModelTransition: "allow_legacy_rollover", + workspaceUri: "file:///fixture-project", workspaceHostId: "a".repeat(64) + }); + const server = createMemoryHttpServer({ service, startAgentSourceAutomation: false, auth: { mode: "local", allowAnonymous: true } }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("expected TCP address"); + const payload = buildSourceTurnRequest({ source: "codex", conversationId: "local-conversation", turnId: "local-turn", + startedAt: "2099-01-01T10:00:00.000Z", completedAt: "2099-01-01T10:01:00.000Z", sequence: 1, + completionEvidence: "final_answer:local-turn", query: "Fix the source capture user and Session identity.", + answer: "The source capture reuses the existing user and Session identity.", status: "succeeded", toolCalls: [], toolResults: [] + }, channel); + const submit = async (source: "hook" | "agent_source_scan") => { + const response = await fetch(`http://127.0.0.1:${address.port}/api/v1/source-turns/complete`, { + method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ ...payload, channel: source, + ...(source === "hook" ? { sessionId: opened.sessionId, namespace: { userId: "configured-owner" } } : {}) }) + }); + expect(response.status).toBe(200); + return response.json() as Promise<{ status: string; result: { sessionId: string; l1MemoryId: string } }>; + }; + try { + const first = await submit(channel); + const second = await submit(channel === "hook" ? "agent_source_scan" : "hook"); + expect(first.status).toBe("stored"); + expect(second.status).toBe("existing"); + expect(first.result.sessionId).toBe(opened.sessionId); + expect(second.result.l1MemoryId).toBe(first.result.l1MemoryId); + const repos = new Repositories(db.db); + expect(repos.memories.get(first.result.l1MemoryId)).toMatchObject({ userId: "configured-owner", sessionId: opened.sessionId, + info: { project_id: repos.runtime.getSession(opened.sessionId)!.projectId } }); + expect(db.db.prepare("SELECT COUNT(*) AS n FROM sessions").get()).toEqual({ n: 1 }); + } finally { await new Promise(resolve => server.close(() => resolve())); } + }); + +}); diff --git a/Memory/tests/native-scanner-lifecycle.test.ts b/Memory/tests/native-scanner-lifecycle.test.ts new file mode 100644 index 000000000..7eb89f055 --- /dev/null +++ b/Memory/tests/native-scanner-lifecycle.test.ts @@ -0,0 +1,79 @@ +import { writeFileSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { readCodexRollout, sourceTurnFromMessages, buildSourceTurnRequest, type RawCodexMessage } from "@memmy/agent-source-core"; +import { createAgentSourceExecutor } from "../src/agent-source/runtime.js"; +import { createSourceRegistry } from "../src/agent-source/adapters/source-registry.js"; +import { createMemoryServiceFixture } from "./fixtures/memory-service-fixture.js"; +import { DEFAULT_MEMMY_CONFIG } from "../src/index.js"; +import { Repositories } from "../src/storage/repositories.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(() => { vi.restoreAllMocks(); fixture.cleanup(); }); +const at = "2099-09-09T10:00:00.000Z"; +const event = (type: string, payload: Record) => ({ type, timestamp: at, payload }); +function records(complete: boolean) { + return [event("session_meta", { id: "native-session" }), event("event_msg", { type: "task_started", turn_id: "native-turn" }), + event("response_item", { type: "message", role: "user", content: [{ text: "Inspect the issue and run tests." }] }), + event("response_item", { type: "function_call", call_id: "read", name: "read_file", arguments: { path: "src/main.ts" } }), + event("response_item", { type: "function_call_output", call_id: "read", output: "source contents ".repeat(3000) }), + event("response_item", { type: "message", role: "assistant", content: [{ text: "The issue is fixed and all tests passed." }] }), + ...(complete ? [event("event_msg", { type: "task_complete", turn_id: "native-turn" })] : [])]; +} +async function wait(executor: ReturnType) { + await vi.waitFor(() => expect(executor.scanStatus().running).toBe(false)); +} + +describe("Codex scan and Hook share the actual Memory lifecycle", () => { + it.each(["hook", "scan"])("keeps one RawTurn, L1 and capture job when %s arrives first", async first => { + const { service, db, root } = fixture.createTestService({ config: { ...DEFAULT_MEMMY_CONFIG, userId: "scan-owner" } }); + const path = join(root, "rollout.jsonl"); + writeFileSync(path, records(true).map(record => JSON.stringify(record)).join("\n") + "\n"); + const messages: RawCodexMessage[] = []; for await (const value of readCodexRollout(path)) messages.push(value); + const turn = sourceTurnFromMessages(messages)!; + const hook = () => service.completeSourceTurn({ ...buildSourceTurnRequest(turn, "hook"), namespace: { source: "codex", profileId: "default", userId: "scan-owner" } }); + const add = vi.spyOn(service, "addMemory"); + const executor = createAgentSourceExecutor({ service, configPath: join(root, "config.yaml"), + resolveAgentSkillRoot: () => null, + sourceRegistry: createSourceRegistry([{ descriptor: { sourceId: "codex", displayName: "Codex", builtin: true, dataPath: root }, + detect: async () => true, async *scan() { for (const message of messages) yield { ...message, sourceId: "codex", workspacePath: null, gitRoot: null }; } }]) }); + try { + if (first === "hook") expect(hook().status).toBe("stored"); + await executor.startScan({ sourceId: "codex", mode: "full" }); await wait(executor); + expect(executor.scanStatus().error).toBeNull(); + expect(hook().status).toBe("existing"); + expect(add).not.toHaveBeenCalled(); + for (const table of ["raw_turns", "source_turn_captures"]) expect(db.db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get()).toEqual({ n: 1 }); + expect(db.db.prepare("SELECT COUNT(*) AS n FROM memories WHERE memory_layer = 'L1'").get()).toEqual({ n: 1 }); + const captured = hook().result!; + const repos = new Repositories(db.db); + expect(repos.runtime.getSession(captured.sessionId)?.userId).toBe("scan-owner"); + expect(repos.memories.get(captured.l1MemoryId)?.userId).toBe("scan-owner"); + expect(repos.runtime.getEpisode(captured.episodeId)?.l1MemoryIds).toEqual([captured.l1MemoryId]); + expect(repos.runtime.getRawTurn(captured.rawTurnId)?.toolCalls[0]).toMatchObject({ id: "read", input: { path: "src/main.ts" }, output: expect.stringContaining("source contents") }); + expect(db.db.prepare("SELECT COUNT(*) AS n FROM evolution_jobs WHERE job_type = 'import_summary'").get()).toEqual({ n: 0 }); + } finally { await executor.dispose(); } + }); + + it("keeps an incomplete scan retryable, then captures after task_complete is appended", async () => { + const { service, db, root } = fixture.createTestService(); + const path = join(root, "rollout.jsonl"); const statePath = join(root, "scan-state.json"); + const write = (complete: boolean) => writeFileSync(path, records(complete).map(record => JSON.stringify(record)).join("\n") + "\n"); + write(false); + const executor = createAgentSourceExecutor({ service, configPath: join(root, "config.yaml"), statePath, + resolveAgentSkillRoot: () => null, + sourceRegistry: createSourceRegistry([{ descriptor: { sourceId: "codex", displayName: "Codex", builtin: true, dataPath: root }, + detect: async () => true, async *scan() { for await (const message of readCodexRollout(path)) yield { ...message, sourceId: "codex", workspacePath: null, gitRoot: null }; } }]) }); + try { + await executor.startScan({ sourceId: "codex", mode: "incremental" }); await wait(executor); + expect(executor.scanStatus().error).toContain("turn_incomplete"); + expect(JSON.parse(readFileSync(statePath, "utf8")).sources.codex.latestSeenAt).toBeNull(); + expect(db.db.prepare("SELECT COUNT(*) AS n FROM raw_turns").get()).toEqual({ n: 0 }); + write(true); + await executor.startScan({ sourceId: "codex", mode: "incremental" }); await wait(executor); + expect(executor.scanStatus().error).toBeNull(); + expect(db.db.prepare("SELECT COUNT(*) AS n FROM raw_turns").get()).toEqual({ n: 1 }); + expect(JSON.parse(readFileSync(statePath, "utf8")).sources.codex.latestSeenAt).toBe(at); + } finally { await executor.dispose(); } + }); +}); diff --git a/Memory/tests/repository/polardb-schema.test.ts b/Memory/tests/repository/polardb-schema.test.ts index 059d88aa8..aa29aeb00 100644 --- a/Memory/tests/repository/polardb-schema.test.ts +++ b/Memory/tests/repository/polardb-schema.test.ts @@ -8,8 +8,8 @@ import { describe("repository PolarDB schema contract", () => { it("publishes migration SQL for the memories table and runtime support tables", () => { const sql = polardbMigrationSql().join("\n"); - expect(POLARDB_MIGRATION_ID).toBe("003_memory_capture_claims"); - expect(POLARDB_SCHEMA_VERSION).toBe("runtime-v3"); + expect(POLARDB_MIGRATION_ID).toBe("004_source_turn_captures"); + expect(POLARDB_SCHEMA_VERSION).toBe("runtime-v4"); expect(sql).toContain("CREATE EXTENSION IF NOT EXISTS vector"); expect(sql).toContain("CREATE TABLE IF NOT EXISTS memories"); expect(sql).toContain("properties JSONB"); @@ -41,6 +41,8 @@ describe("repository PolarDB schema contract", () => { expect(sql).toContain("CREATE TABLE IF NOT EXISTS memory_change_log"); expect(sql).toContain("CREATE TABLE IF NOT EXISTS idempotency_keys"); expect(sql).toContain("CREATE TABLE IF NOT EXISTS memory_capture_claims"); + expect(sql).toContain("CREATE TABLE IF NOT EXISTS source_turn_captures"); + expect(sql).toContain("PRIMARY KEY (user_id, source, profile_id, namespace_key, conversation_id, turn_id)"); expect(sql).toContain("CREATE TABLE IF NOT EXISTS evolution_jobs"); expect(sql).toContain("CREATE TABLE IF NOT EXISTS embedding_retry_queue"); expect(sql).toContain("idx_embedding_retry_due"); diff --git a/Memory/tests/repository/source-turn-schema.test.ts b/Memory/tests/repository/source-turn-schema.test.ts new file mode 100644 index 000000000..febdf1edd --- /dev/null +++ b/Memory/tests/repository/source-turn-schema.test.ts @@ -0,0 +1,51 @@ +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { MemoryDb, SCHEMA_VERSION } from "../../src/index.js"; +import { Repositories } from "../../src/storage/repositories.js"; +import { createMemoryServiceFixture } from "../fixtures/memory-service-fixture.js"; + +const { cleanup, createTestService } = createMemoryServiceFixture(); +afterEach(cleanup); + +describe("source turn schema upgrade", () => { + it("upgrades v7 without changing old memories and keeps its activation boundary after reopen", () => { + const { db, service, root } = createTestService(); + const session = service.openSession({}); + service.completeTurn("old-turn", { sessionId: session.sessionId, query: "Implement a schema migration safely.", answer: "Existing data is preserved by the migration." }); + const before = db.db.prepare("SELECT * FROM memories ORDER BY id").all(); + db.db.exec(`DROP TABLE source_turn_captures; + DELETE FROM runtime_kv WHERE key = 'source_turn_capture_activated_at'; + DELETE FROM schema_migrations; + INSERT INTO schema_migrations VALUES ('007_memory_capture_claims', 7, '2026-01-01', 'old-v7');`); + const path = join(root, "memory.sqlite"); + db.close(); + const upgraded = new MemoryDb({ path }); + const activation = new Repositories(upgraded.db).runtime.getKv("source_turn_capture_activated_at"); + expect(activation?.value).toEqual(expect.any(String)); + expect(upgraded.schemaVersion().version).toBe(SCHEMA_VERSION); + expect(upgraded.db.prepare("SELECT * FROM memories ORDER BY id").all()).toEqual(before); + expect(upgraded.db.prepare("SELECT COUNT(*) AS count FROM source_turn_captures").get()).toEqual({ count: 0 }); + expect(existsSync(`${path}.pre-v${SCHEMA_VERSION}.bak`)).toBe(true); + upgraded.close(); + const reopened = new MemoryDb({ path }); + try { expect(new Repositories(reopened.db).runtime.getKv("source_turn_capture_activated_at")).toEqual(activation); } + finally { reopened.close(); } + }); + + it("enforces source uniqueness across separate database connections", () => { + const { db, root } = createTestService(); + const second = new MemoryDb({ path: join(root, "memory.sqlite") }); + const capture = { + userId: "user-a", source: "codex", profileId: "default", namespaceKey: "scope-a", + conversationId: "conversation-a", turnId: "turn-a", contentHash: "same-content", + startedAt: "2099-01-01T10:00:00.000Z", completedAt: "2099-01-01T10:01:00.000Z", createdAt: "2099-01-01T10:01:00.000Z", + response: { status: "rejected" as const, reason: "capture_policy" } + }; + try { + new Repositories(db.db).runtime.insertSourceTurnCapture(capture); + expect(() => new Repositories(second.db).runtime.insertSourceTurnCapture(capture)).toThrow(/UNIQUE constraint/); + expect(new Repositories(second.db).runtime.getSourceTurnCapture(capture, capture.turnId)?.response).toEqual(capture.response); + } finally { second.close(); } + }); +}); diff --git a/Memory/tests/service/evolution/l3-world-model.test.ts b/Memory/tests/service/evolution/l3-world-model.test.ts index 8a928df4d..2af9e3968 100644 --- a/Memory/tests/service/evolution/l3-world-model.test.ts +++ b/Memory/tests/service/evolution/l3-world-model.test.ts @@ -50,7 +50,7 @@ describe("L3 World Model trace field pipeline", () => { thinkingBefore: "PRIVATE_REASONING_TOOL", assistantTextBefore: "VISIBLE_ASSISTANT_PROGRESS" }], - toolResults: [{ name: "exec", output: "dynamic linker error", exitCode: 1 }] + toolResults: [{ id: "call-1", name: "exec", output: "dynamic linker error", exitCode: 1 }] }); service.closeSession(opened.sessionId); diff --git a/Memory/tests/service/evolution/reflection.test.ts b/Memory/tests/service/evolution/reflection.test.ts index 72557cb0f..6122f3d43 100644 --- a/Memory/tests/service/evolution/reflection.test.ts +++ b/Memory/tests/service/evolution/reflection.test.ts @@ -633,10 +633,10 @@ describe("MemoryService / evolution / reflection", () => { expect(payload.steps).toHaveLength(1); expect(payload.steps[0]!.tool_calls).toMatchObject([{ name: "shell", - input: "", - output: "" + input: "npm run migrate", + output: "error: missing sqlite migration 003" }]); - expect(JSON.stringify(payload)).not.toContain("npm run migrate"); + expect(JSON.stringify(payload)).toContain("npm run migrate"); expect(payload.steps[0]!.action).toContain("inspected the migration failure"); expect(calls.some((call) => call.options.operation === "capture.alpha.reflection.score.v3")).toBe(false); db.close(); @@ -780,9 +780,11 @@ describe("MemoryService / evolution / reflection", () => { task_context?: string | null; }; expect(payload.task_context).toContain("debug migration"); - const toolInputs = payload.steps.flatMap((step) => step.tool_calls.map((call) => call.input)).join("\n"); - expect(toolInputs).not.toContain("npm run migrate"); - expect(toolInputs).not.toContain("npm test -- memory-service"); + const toolCalls = payload.steps.flatMap((step) => step.tool_calls); + expect(toolCalls).toMatchObject([ + { input: "npm run migrate", output: "error: missing sqlite migration 003" }, + { input: "npm test -- memory-service", output: "1 test passed" } + ]); db.close(); }); diff --git a/Memory/tests/service/session/source-turn-capture.test.ts b/Memory/tests/service/session/source-turn-capture.test.ts new file mode 100644 index 000000000..a29badf3c --- /dev/null +++ b/Memory/tests/service/session/source-turn-capture.test.ts @@ -0,0 +1,423 @@ +import { Repositories, RuntimeRepository } from "../../../src/storage/repositories.js"; +import { memoryCaptureQaHash } from "../../../src/utils/memory-capture-claim.js"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { buildSourceTurnRequest } from "@memmy/agent-source-core"; +import { MemoryDb } from "../../../src/index.js"; +import type { SourceTurnCompleteRequest } from "../../../src/types.js"; +import { createMemoryServiceFixture, createBatchReflectionLlm, runWorkerRounds } from "../../fixtures/memory-service-fixture.js"; + +const { cleanup, createTestService, createTestMemoryService } = createMemoryServiceFixture(); +afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); cleanup(); }); + +function request(overrides: Partial = {}): SourceTurnCompleteRequest { + return { + namespace: { source: "codex", profileId: "default", userId: "source-user" }, + source: "codex", + channel: "hook", + sourceTurn: { + source: "codex", profileId: "default", conversationId: "native-conversation", turnId: "native-turn-1", + startedAt: "2099-01-01T10:00:00.000Z", completedAt: "2099-01-01T10:01:00.000Z", + sequence: 1, completionEvidence: "task_complete:10" + }, + query: "Implement a transaction that preserves the native source turn identity.", + answer: "The transaction now stores the full source turn and schedules capture.", + toolCalls: [{ id: "call-a", name: "read_file", input: { path: "schema.ts" } }], + toolResults: [{ id: "call-a", output: "CREATE TABLE source_turn_captures" }], + ...overrides + }; +} + +function counts(db: MemoryDb) { + return Object.fromEntries(["sessions", "episodes", "raw_turns", "memories", "source_turn_captures", "evolution_jobs"].map((table) => [table, + (db.db.prepare(`SELECT COUNT(*) AS count FROM ${table}`).get() as { count: number }).count])); +} + +describe("native source turn submission", () => { + it("accepts the shared parser builder without inventing a second namespace", () => { + const { service } = createTestService(); + const input = request(); + const built = buildSourceTurnRequest({ + source: "codex", conversationId: input.sourceTurn.conversationId, turnId: input.sourceTurn.turnId, + startedAt: input.sourceTurn.startedAt, completedAt: input.sourceTurn.completedAt, + sequence: 1, completionEvidence: input.sourceTurn.completionEvidence, + query: input.query, answer: input.answer, status: "succeeded", toolCalls: [], toolResults: [] + }, "agent_source_scan"); + expect(service.completeSourceTurn(built).status).toBe("stored"); + }); + + it.each(["hook", "agent_source_scan"] as const)("commits one lifecycle when %s arrives first and reuses it after restart", (channel) => { + const { db, service, root } = createTestService(); + const first = service.completeSourceTurn(request({ channel })); + expect(first.status).toBe("stored"); + expect(first.result?.l1MemoryIds).toHaveLength(1); + const result = first.result!; + const raw = new Repositories(db.db).runtime.getRawTurn(result.rawTurnId)!; + const memory = new Repositories(db.db).memories.get(result.l1MemoryId)!; + expect(raw).toMatchObject({ sessionId: result.sessionId, episodeId: result.episodeId, turnId: "native-turn-1", createdAt: request().sourceTurn.startedAt }); + expect(memory).toMatchObject({ sessionId: result.sessionId }); + expect(new Repositories(db.db).runtime.getEpisode(result.episodeId)?.l1MemoryIds).toContain(result.l1MemoryId); + expect(db.db.prepare("SELECT COUNT(*) AS count FROM recall_events").get()).toEqual({ count: 0 }); + expect(db.db.prepare("SELECT COUNT(*) AS count FROM memory_capture_claims").get()).toEqual({ count: 0 }); + const before = counts(db); + db.close(); + const reopened = new MemoryDb({ path: join(root, "memory.sqlite") }); + try { + const restarted = createTestMemoryService({ db: reopened }); + const second = restarted.completeSourceTurn(request({ channel: channel === "hook" ? "agent_source_scan" : "hook", sessionId: "late-hook-session" })); + expect(second).toMatchObject({ status: "existing", result: { rawTurnId: result.rawTurnId, l1MemoryId: result.l1MemoryId, duplicate: true, jobs: [] } }); + expect(counts(reopened)).toEqual(before); + } finally { reopened.close(); } + }); + + it("reuses a scoped Hook Session without a simulated recall and ignores old QA claims", () => { + const { db, service } = createTestService(); + const input = request(); + const opened = service.openSession({ namespace: { ...input.namespace!, sessionKey: input.sourceTurn.conversationId }, meta: { conversationId: input.sourceTurn.conversationId } }); + new Repositories(db.db).captureClaims.claim({ userId: "source-user", source: "codex", qaHash: memoryCaptureQaHash(input.query, input.answer), primaryMemoryId: "deleted-old-memory", capturedBy: "agent_source_scan", createdAt: "2026-01-01" }); + const stored = service.completeSourceTurn(request({ channel: "agent_source_scan" })); + expect(stored.result?.sessionId).toBe(opened.sessionId); + expect(counts(db).sessions).toBe(1); + expect(stored.result?.l1MemoryIds).toHaveLength(1); + }); + + it.each(["hook", "agent_source_scan"] as const)("reuses the existing prefixed Codex Hook Session when %s captures first", (channel) => { + const { db, service } = createTestService(); + const input = request(); + const opened = service.openSession({ + namespace: { ...input.namespace!, sessionKey: `codex-memory-${input.sourceTurn.conversationId}` }, + l3WorldModelProtocolVersion: 2, l3WorldModelTransition: "allow_legacy_rollover", + workspaceUri: "file:///workspace/project-a", workspaceHostId: "a".repeat(64) + }); + const runtime = new Repositories(db.db).runtime; + const original = runtime.getSession(opened.sessionId)!; + expect(original.conversationId).toBeUndefined(); + const first = service.completeSourceTurn(request({ channel, + ...(channel === "hook" ? { sessionId: opened.sessionId } : {}) })); + expect(first).toMatchObject({ status: "stored", result: { sessionId: opened.sessionId } }); + const second = service.completeSourceTurn(request({ channel: channel === "hook" ? "agent_source_scan" : "hook", + sessionId: opened.sessionId })); + expect(second).toMatchObject({ status: "existing", result: { sessionId: opened.sessionId, l1MemoryId: first.result!.l1MemoryId } }); + expect(counts(db).sessions).toBe(1); + expect(runtime.getSession(opened.sessionId)).toMatchObject({ + userId: "source-user", hostSessionKey: original.hostSessionKey, conversationId: input.sourceTurn.conversationId, + projectId: original.projectId, workspaceId: original.workspaceId, meta: original.meta + }); + expect(runtime.getRawTurn(first.result!.rawTurnId)).toMatchObject({ userId: "source-user", conversationId: input.sourceTurn.conversationId }); + const memory = new Repositories(db.db).memories.get(first.result!.l1MemoryId)!; + expect(memory).toMatchObject({ userId: "source-user", conversationId: input.sourceTurn.conversationId, appId: original.workspaceId }); + expect(memory.info.project_id).toBe(original.projectId); + service.closeSession(opened.sessionId, { namespace: input.namespace }); + expect(runtime.getEpisode(first.result!.episodeId)?.status).toBe("closed"); + expect(runtime.getSession(opened.sessionId)?.status).toBe("closed"); + }); + + it.each([ + { label: "another user with the same native conversation", namespace: { userId: "other-user" } }, + { label: "another project", namespace: { projectId: "project-b" } }, + { label: "another workspace", namespace: { workspaceId: "workspace-b" } }, + { label: "an unproven tenant", namespace: { tenantId: "tenant-b" } }, + { label: "another profile", namespace: { profileId: "profile-b" } } + ])("does not adopt a prefixed Hook Session into $label", ({ namespace }) => { + const { db, service } = createTestService(); + const input = request(); + const opened = service.openSession({ namespace: { ...input.namespace!, sessionKey: `codex-memory-${input.sourceTurn.conversationId}` } }); + const before = counts(db); + const requestedNamespace = { ...input.namespace!, ...namespace }; + expect(() => service.completeSourceTurn(request({ sessionId: opened.sessionId, + namespace: requestedNamespace, + sourceTurn: { ...input.sourceTurn, profileId: requestedNamespace.profileId ?? input.sourceTurn.profileId } + }))).toThrow("source_session_scope_conflict"); + expect(counts(db)).toEqual(before); + expect(new Repositories(db.db).runtime.getSession(opened.sessionId)?.conversationId).toBeUndefined(); + }); + + it("refuses a conflicting saved conversation even when the prefixed host key matches", () => { + const { db, service } = createTestService(); + const input = request(); + const opened = service.openSession({ namespace: { ...input.namespace!, sessionKey: `codex-memory-${input.sourceTurn.conversationId}` }, + meta: { conversationId: "another-native-conversation" } }); + const before = counts(db); + expect(() => service.completeSourceTurn(request({ sessionId: opened.sessionId }))).toThrow("source_session_scope_conflict"); + expect(counts(db)).toEqual(before); + expect(new Repositories(db.db).runtime.getSession(opened.sessionId)?.conversationId).toBe("another-native-conversation"); + }); + + it("keeps an ambiguous native and prefixed Session unresolved", () => { + const { db, service } = createTestService(); + const input = request(); + for (const sessionKey of [input.sourceTurn.conversationId, `codex-memory-${input.sourceTurn.conversationId}`]) { + service.openSession({ namespace: { ...input.namespace!, sessionKey } }); + } + const before = counts(db); + expect(service.completeSourceTurn(request({ channel: "agent_source_scan" }))).toMatchObject({ status: "pending", reason: "source_session_ambiguous" }); + expect(counts(db)).toEqual(before); + }); + + it("rolls back a newly bound Hook conversation when capture persistence fails", () => { + const { db, service } = createTestService(); + const input = request(); + const opened = service.openSession({ namespace: { ...input.namespace!, sessionKey: `codex-memory-${input.sourceTurn.conversationId}` } }); + const before = counts(db); + vi.spyOn(RuntimeRepository.prototype, "insertSourceTurnCapture").mockImplementation(() => { throw new Error("simulated persistence failure"); }); + expect(() => service.completeSourceTurn(request({ sessionId: opened.sessionId }))).toThrow("simulated persistence failure"); + expect(counts(db)).toEqual(before); + expect(new Repositories(db.db).runtime.getSession(opened.sessionId)?.conversationId).toBeUndefined(); + }); + + it("binds only the current observed RawTurn when completing a prefixed Hook Session", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime("2099-01-01T09:58:00.000Z"); + const { db, service } = createTestService(); + const input = request(); + const opened = service.openSession({ namespace: { ...input.namespace!, sessionKey: `codex-memory-${input.sourceTurn.conversationId}` } }); + const runtime = new Repositories(db.db).runtime; + const historical = service.completeTurn("historical-turn", { sessionId: opened.sessionId, query: input.query, answer: input.answer }); + const historicalRaw = runtime.getRawTurn(historical.rawTurnId)!; + expect(historicalRaw.conversationId).toBeUndefined(); + vi.setSystemTime(input.sourceTurn.startedAt); + await service.startTurn({ namespace: input.namespace, sessionId: opened.sessionId, turnId: input.sourceTurn.turnId, query: input.query }); + expect(runtime.getRawTurnBySessionTurn(opened.sessionId, input.sourceTurn.turnId)).toBeUndefined(); + const observed = await service.observeTool({ namespace: input.namespace, sessionId: opened.sessionId, + turnId: input.sourceTurn.turnId, toolCallId: "call-a", toolName: "read_file", args: { path: "schema.ts" } }); + expect(runtime.getRawTurn(observed.rawTurnId!)?.conversationId).toBeUndefined(); + vi.setSystemTime(input.sourceTurn.completedAt); + const completed = service.completeSourceTurn(request({ sessionId: opened.sessionId })); + expect(completed).toMatchObject({ status: "stored", result: { sessionId: opened.sessionId, rawTurnId: observed.rawTurnId } }); + expect(runtime.getRawTurn(observed.rawTurnId!)).toMatchObject({ + conversationId: input.sourceTurn.conversationId, userText: input.query, assistantText: input.answer, status: "succeeded" + }); + expect(runtime.getRawTurn(historical.rawTurnId)).toEqual(historicalRaw); + expect(new Repositories(db.db).memories.get(completed.result!.l1MemoryId)?.conversationId).toBe(input.sourceTurn.conversationId); + }); + + it.each(["user", "conversation"] as const)("refuses an observed RawTurn with a conflicting %s", async (conflict) => { + const { db, service } = createTestService(); + const input = request(); + const opened = service.openSession({ namespace: { ...input.namespace!, sessionKey: `codex-memory-${input.sourceTurn.conversationId}` } }); + const observed = await service.observeTool({ sessionId: opened.sessionId, turnId: input.sourceTurn.turnId, + toolCallId: "call-a", toolName: "read_file", args: { path: "schema.ts" } }); + if (conflict === "user") db.db.prepare("UPDATE raw_turns SET user_id = 'other-user' WHERE id = ?").run(observed.rawTurnId); + else db.db.prepare("UPDATE raw_turns SET conversation_id = 'other-conversation' WHERE id = ?").run(observed.rawTurnId); + const runtime = new Repositories(db.db).runtime; + const original = runtime.getRawTurn(observed.rawTurnId!)!; + const before = counts(db); + expect(() => service.completeSourceTurn(request({ sessionId: opened.sessionId }))).toThrow("source_raw_turn_scope_conflict"); + expect(counts(db)).toEqual(before); + expect(runtime.getRawTurn(observed.rawTurnId!)).toEqual(original); + expect(runtime.getSession(opened.sessionId)?.conversationId).toBeUndefined(); + }); + + it("does not apply the Codex host-key alias to another source", () => { + const { db, service } = createTestService(); + const input = request(); + const namespace = { ...input.namespace!, source: "cursor" }; + const opened = service.openSession({ namespace: { ...namespace, sessionKey: `codex-memory-${input.sourceTurn.conversationId}` } }); + expect(new Repositories(db.db).runtime.sourceConversationSessions({ userId: "source-user", source: "cursor", profileId: "default", + conversationId: input.sourceTurn.conversationId })).toEqual([]); + const before = counts(db); + expect(() => service.completeSourceTurn(request({ namespace, source: "cursor", sessionId: opened.sessionId, + sourceTurn: { ...input.sourceTurn, source: "cursor" } }))).toThrow("source_session_scope_conflict"); + expect(counts(db)).toEqual(before); + }); + + it("does not merge equal QA belonging to different turns or scopes", () => { + const { service } = createTestService(); + const first = service.completeSourceTurn(request()); + const second = service.completeSourceTurn(request({ sourceTurn: { ...request().sourceTurn, turnId: "native-turn-2", sequence: 2, startedAt: "2099-01-01T10:02:00.000Z", completedAt: "2099-01-01T10:03:00.000Z" } })); + const otherUser = service.completeSourceTurn(request({ namespace: { ...request().namespace!, userId: "other-user" } })); + expect(new Set([first.result?.l1MemoryId, second.result?.l1MemoryId, otherUser.result?.l1MemoryId]).size).toBe(3); + }); + + it("reports conflicting content and keeps deleted capture identities", () => { + const { db, service } = createTestService(); + const first = service.completeSourceTurn(request()); + const before = counts(db); + expect(service.completeSourceTurn(request({ answer: "A conflicting result." }))).toMatchObject({ status: "conflict", reason: "source_turn_content_conflict" }); + db.db.prepare("UPDATE memories SET status = 'deleted', deleted_at = ? WHERE id = ?").run("2099-01-02", first.result!.l1MemoryId); + expect(service.completeSourceTurn(request())).toMatchObject({ status: "rejected", reason: "capture_deleted" }); + expect(counts(db)).toEqual(before); + }); + + it("rolls back every lifecycle write when source registration fails", () => { + const { db, service } = createTestService(); + const before = counts(db); + vi.spyOn(RuntimeRepository.prototype, "insertSourceTurnCapture").mockImplementation(() => { throw new Error("simulated persistence failure"); }); + expect(() => service.completeSourceTurn(request())).toThrow("simulated persistence failure"); + expect(counts(db)).toEqual(before); + }); + + it("retains unresolved, out of order, and closed Episode turns for retry", () => { + const { db, service } = createTestService(); + expect(service.completeSourceTurn(request({ sourceTurn: { ...request().sourceTurn, turnId: "" } }))).toMatchObject({ status: "pending", reason: "identity_unresolved" }); + const first = service.completeSourceTurn(request()); + const before = counts(db); + expect(service.completeSourceTurn(request({ sourceTurn: { ...request().sourceTurn, turnId: "late-turn", sequence: 0, startedAt: "2099-01-01T09:00:00.000Z", completedAt: "2099-01-01T09:01:00.000Z" } }))).toMatchObject({ status: "pending", reason: "source_turn_out_of_order" }); + new Repositories(db.db).runtime.closeEpisode(first.result!.episodeId, { closeReason: "idle" }, "2099-01-01T10:01:30.000Z"); + expect(service.completeSourceTurn(request({ sourceTurn: { ...request().sourceTurn, turnId: "after-close", sequence: 2, startedAt: "2099-01-01T10:02:00.000Z", completedAt: "2099-01-01T10:03:00.000Z" } }))).toMatchObject({ status: "pending", reason: "source_episode_closed" }); + expect(counts(db)).toEqual(before); + }); + + it("rejects pre-activation turns while allowing a turn that completes after activation", () => { + const { service } = createTestService(); + expect(service.completeSourceTurn(request({ sourceTurn: { ...request().sourceTurn, startedAt: "2000-01-01T10:00:00.000Z", completedAt: "2000-01-01T10:01:00.000Z" } }))).toMatchObject({ status: "rejected", reason: "legacy_before_activation" }); + expect(service.completeSourceTurn(request({ sourceTurn: { ...request().sourceTurn, startedAt: "2000-01-01T10:00:00.000Z" } })).status).toBe("stored"); + }); + + it("fills a missing turn bounded by captured turns in the same open Episode", () => { + const { db, service } = createTestService(); + const first = service.completeSourceTurn(request()); + const third = service.completeSourceTurn(request({ sourceTurn: { ...request().sourceTurn, turnId: "turn-3", sequence: 3, + startedAt: "2099-01-01T10:04:00.000Z", completedAt: "2099-01-01T10:05:00.000Z" } })); + expect(third.result?.episodeId).toBe(first.result?.episodeId); + const second = service.completeSourceTurn(request({ channel: "agent_source_scan", sourceTurn: { ...request().sourceTurn, turnId: "turn-2", sequence: 2, + startedAt: "2099-01-01T10:02:00.000Z", completedAt: "2099-01-01T10:03:00.000Z" } })); + expect(second).toMatchObject({ status: "stored", result: { sessionId: first.result!.sessionId, episodeId: first.result!.episodeId } }); + const episode = new Repositories(db.db).runtime.getEpisode(first.result!.episodeId)!; + expect(episode.l1MemoryIds).toEqual([first.result!.l1MemoryId, second.result!.l1MemoryId, third.result!.l1MemoryId]); + expect(episode.rawTurnIds).toEqual([first.result!.rawTurnId, second.result!.rawTurnId, third.result!.rawTurnId]); + expect(episode.updatedAt).toBe("2099-01-01T10:05:00.000Z"); + expect(new Repositories(db.db).runtime.getSession(first.result!.sessionId)?.lastSeenAt).toBe("2099-01-01T10:05:00.000Z"); + }); + + it("does not treat a sequence reset in a new source artifact as old content", () => { + const { service } = createTestService(); + service.completeSourceTurn(request({ sourceTurn: { ...request().sourceTurn, sequence: 100 } })); + const next = service.completeSourceTurn(request({ sourceTurn: { ...request().sourceTurn, turnId: "new-artifact-turn", sequence: 1, + startedAt: "2099-01-01T10:02:00.000Z", completedAt: "2099-01-01T10:03:00.000Z" } })); + expect(next.status).toBe("stored"); + }); + + it("uses a new scoped Hook Session after the previous source Session closes", () => { + const { db, service } = createTestService(); + const first = service.completeSourceTurn(request()); + new Repositories(db.db).runtime.closeSession(first.result!.sessionId, "2099-01-01T10:01:30.000Z"); + const opened = service.openSession({ namespace: { ...request().namespace!, sessionKey: request().sourceTurn.conversationId } }); + const next = service.completeSourceTurn(request({ sessionId: opened.sessionId, sourceTurn: { ...request().sourceTurn, turnId: "new-session-turn", sequence: 2, + startedAt: "2099-01-01T10:02:00.000Z", completedAt: "2099-01-01T10:03:00.000Z" } })); + expect(next).toMatchObject({ status: "stored", result: { sessionId: opened.sessionId } }); + expect(next.result?.sessionId).not.toBe(first.result?.sessionId); + }); + + it("does not cross project, profile, or tenant scopes", () => { + const { service } = createTestService(); + const baseline = request(); + const results = [ + service.completeSourceTurn(baseline), + service.completeSourceTurn(request({ namespace: { ...baseline.namespace!, projectId: "project-b" } })), + service.completeSourceTurn(request({ namespace: { ...baseline.namespace!, profileId: "profile-b" }, + sourceTurn: { ...baseline.sourceTurn, profileId: "profile-b" } })), + service.completeSourceTurn(request({ namespace: { ...baseline.namespace!, tenantId: "tenant-b" } })) + ]; + expect(results.map((result) => result.status)).toEqual(["stored", "stored", "stored", "stored"]); + expect(new Set(results.map((result) => result.result?.sessionId)).size).toBe(4); + expect(new Set(results.map((result) => result.result?.l1MemoryId)).size).toBe(4); + }); + + it("keeps the original pending or failed capture job on retry", () => { + const { db, service } = createTestService(); + const first = service.completeSourceTurn(request()); + const before = counts(db); + db.db.prepare("UPDATE evolution_jobs SET status = 'failed' WHERE target_memory_id = ?") + .run(first.result!.l1MemoryId); + expect(service.completeSourceTurn(request())).toMatchObject({ status: "existing", result: { jobs: [] } }); + expect(counts(db)).toEqual(before); + }); + + it("returns a policy rejection without resurrecting its original L1", () => { + const { db, service } = createTestService(); + const first = service.completeSourceTurn(request()); + const before = counts(db); + db.db.prepare(`UPDATE memories SET status = 'deleted', deleted_at = '2099-01-01', + properties_json = json_set(properties_json, '$.internal_info.capture_decision.status', 'rejected') WHERE id = ?`) + .run(first.result!.l1MemoryId); + expect(service.completeSourceTurn(request())).toMatchObject({ status: "rejected", reason: "capture_policy" }); + expect(counts(db)).toEqual(before); + }); + + it("preserves capture identities in bundles and across deleting all memory data", () => { + const first = createTestService(); + const stored = first.service.completeSourceTurn(request()); + const activation = new Repositories(first.db.db).runtime.getKv("source_turn_capture_activated_at"); + const bundle = first.service.exportBundle({ includeRawText: true }); + expect(bundle.tables.source_turn_captures).toHaveLength(1); + expect(JSON.stringify(bundle.tables.source_turn_captures)).not.toContain(request().query); + const restored = createTestService(); + const imported = restored.service.importBundle({ bundle }); + expect(imported.ok).toBe(true); + expect(imported.conflicts).toEqual([]); + expect(new Repositories(restored.db.db).runtime.getKv("source_turn_capture_activated_at")).toEqual(activation); + expect(restored.service.completeSourceTurn(request())).toMatchObject({ status: "existing", result: { l1MemoryId: stored.result!.l1MemoryId } }); + new Repositories(first.db.db).clearAllMemoryData(); + expect(new Repositories(first.db.db).runtime.getKv("source_turn_capture_activated_at")).toEqual(activation); + expect(first.service.completeSourceTurn(request())).toMatchObject({ status: "rejected", reason: "capture_deleted" }); + expect(first.service.completeSourceTurn(request({ sourceTurn: { ...request().sourceTurn, turnId: "after-clear", + startedAt: "2099-01-01T10:02:00.000Z", completedAt: "2099-01-01T10:03:00.000Z" } })).status).toBe("stored"); + }); + + it("does not widen either historical capture window when merging a bundle into existing data", () => { + const target = createTestService(); + target.service.completeSourceTurn(request()); + const runtime = new Repositories(target.db.db).runtime; + runtime.setKv("source_turn_capture_activated_at", "2090-01-01T00:00:00.000Z"); + const mergeBoundary = (value: string) => runtime.importBundleTables({ runtime_kv: [{ + key: "source_turn_capture_activated_at", value_json: JSON.stringify(value), updated_at: "2099-01-01T00:00:00.000Z" + }] }); + expect(mergeBoundary("2080-01-01T00:00:00.000Z").conflicts).toEqual([]); + expect(runtime.getKv("source_turn_capture_activated_at")?.value).toBe("2090-01-01T00:00:00.000Z"); + expect(mergeBoundary("2095-01-01T00:00:00.000Z").conflicts).toEqual([]); + expect(runtime.getKv("source_turn_capture_activated_at")?.value).toBe("2095-01-01T00:00:00.000Z"); + expect(target.service.completeSourceTurn(request())).toMatchObject({ status: "existing" }); + expect(() => mergeBoundary("invalid-date")).toThrow("invalid source turn activation boundary in bundle"); + expect(runtime.getKv("source_turn_capture_activated_at")?.value).toBe("2095-01-01T00:00:00.000Z"); + }); + + it("feeds a newly scanned L1 into the existing reflection and reward jobs after closure", async () => { + const calls: Parameters[0] = []; + const { db, service } = createTestService({ llm: createBatchReflectionLlm(calls) }); + const captured = service.completeSourceTurn(request({ channel: "agent_source_scan" })); + await runWorkerRounds(service, 3); + service.closeSession(captured.result!.sessionId); + await runWorkerRounds(service, 3); + const jobs = db.db.prepare("SELECT job_type, status, episode_id FROM evolution_jobs WHERE episode_id = ?") + .all(captured.result!.episodeId) as Array<{ job_type: string; status: string; episode_id: string }>; + expect(jobs).toEqual(expect.arrayContaining([ + expect.objectContaining({ job_type: "trace_summary", status: "succeeded" }), + expect.objectContaining({ job_type: "reflection", status: "succeeded" }), + expect.objectContaining({ job_type: "reward" }) + ])); + expect(calls.some((call) => call.options.operation === "capture.reflection.batch.v13")).toBe(true); + }); + + it("keeps the ordinary completeTurn Episode creation time at completion", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime("2099-01-01T10:00:00.000Z"); + const { db, service } = createTestService(); + const opened = service.openSession({ namespace: request().namespace }); + await service.startTurn({ sessionId: opened.sessionId, turnId: "ordinary-turn", query: request().query }); + vi.setSystemTime("2099-01-01T10:05:00.000Z"); + const completed = service.completeTurn("ordinary-turn", { sessionId: opened.sessionId, query: request().query, answer: request().answer }); + expect(new Repositories(db.db).runtime.getEpisode(completed.episodeId)?.openedAt).toBe("2099-01-01T10:05:00.000Z"); + }); + + it("does not reopen a closed Episode from an earlier cached source turn proposal", async () => { + const { db, service } = createTestService(); + const runtime = new Repositories(db.db).runtime; + const first = service.completeSourceTurn(request()); + await service.startTurn({ namespace: request().namespace, sessionId: first.result!.sessionId, turnId: "changed-source-turn", + query: "Continue with the same migration." }); + runtime.closeEpisode(first.result!.episodeId, { closeReason: "idle" }, "2099-01-01T10:01:30.000Z"); + const completed = service.completeSourceTurn(request({ query: "New task: implement a weather dashboard.", + sourceTurn: { ...request().sourceTurn, turnId: "changed-source-turn", startedAt: "2099-01-01T10:02:00.000Z", completedAt: "2099-01-01T10:03:00.000Z" } })); + expect(completed.status).toBe("stored"); + expect(completed.result?.episodeId).not.toBe(first.result?.episodeId); + expect(runtime.getEpisode(first.result!.episodeId)?.status).toBe("closed"); + expect(runtime.getEpisode(completed.result!.episodeId)?.openedAt).toBe("2099-01-01T10:02:00.000Z"); + }); + + it("rejects a supplied Session from another account or native conversation", () => { + const { service } = createTestService(); + const foreign = service.openSession({ namespace: { source: "codex", profileId: "default", userId: "other-user", sessionKey: "other-conversation" } }); + expect(() => service.completeSourceTurn(request({ sessionId: foreign.sessionId }))).toThrow("source_session_scope_conflict"); + }); +}); diff --git a/Memory/tests/service/session/turn-capture.test.ts b/Memory/tests/service/session/turn-capture.test.ts index 772fd2959..3bfbcc83c 100644 --- a/Memory/tests/service/session/turn-capture.test.ts +++ b/Memory/tests/service/session/turn-capture.test.ts @@ -444,7 +444,7 @@ describe("MemoryService / session / turn capture", () => { }; }; expect(properties.internal_info.summary).toBe(""); - expect(properties.internal_info.trace.tool_calls[0]?.output).toBeUndefined(); + expect(properties.internal_info.trace.tool_calls[0]?.output).toBe(expectedTrace.toolCalls[0]?.output); expect(defaultTrace.errorSignatures).toContain("SENTINEL_ERROR_CODE"); expect(expectedTrace.errorSignatures).not.toContain("SENTINEL_ERROR_CODE"); expect(properties.internal_info.trace.error_signatures).toEqual(expectedTrace.errorSignatures); diff --git a/Memory/tests/service/session/turn-tool-pairing.test.ts b/Memory/tests/service/session/turn-tool-pairing.test.ts new file mode 100644 index 000000000..b39a4f8af --- /dev/null +++ b/Memory/tests/service/session/turn-tool-pairing.test.ts @@ -0,0 +1,152 @@ +import { afterEach, describe, expect, it } from "vitest"; +import type { ToolCallPayload } from "../../../src/types.js"; +import { captureTurnSteps } from "../../../src/algorithm/plugin-algorithms.js"; +import { normalizeCompleteTurnToolCalls, sanitizeTurnCompleteRequest } from "../../../src/service/turn/turn-normalization.js"; +import { createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; + +const { cleanup, createTestService } = createMemoryServiceFixture(); +afterEach(cleanup); + +const calls = [ + { id: "call-a", name: "read_file", input: { path: "a.txt" } }, + { id: "call-b", name: "read_file", input: { path: "b.txt" } } +]; + +function normalize(toolCalls: unknown[], toolResults: unknown[]) { + return normalizeCompleteTurnToolCalls({ + sessionId: "session-tools", query: "Read the files.", answer: "Read complete.", toolCalls, toolResults + }); +} + +function capture(toolCalls: ToolCallPayload[], toolResults: unknown[]) { + return captureTurnSteps({ + episodeId: "episode-tools", sessionId: "session-tools", turnId: "turn-tools", + toolCalls, toolResults, createdAtIso: "2026-09-09T00:00:00.000Z" + })[0]!.toolCalls; +} + +describe("complete turn tool pairing", () => { + it("pairs repeated tool names by invocation ID when results arrive out of order", () => { + const result = normalize(calls, [ + { call_id: "call-b", output: "FILE_B", status: "failed", success: false, error: "read failed", errorCode: "ENOENT" }, + { toolCallId: "call-a", output: "FILE_A", status: "completed", success: true } + ]); + expect(result).toMatchObject([ + { id: "call-a", input: { path: "a.txt" }, output: "FILE_A", status: "completed", success: true }, + { id: "call-b", input: { path: "b.txt" }, output: "FILE_B", status: "failed", success: false, error: "read failed", errorCode: "ENOENT" } + ]); + expect(result[0]?.error).toBeUndefined(); + }); + + it("keeps A missing when only B has a result, including the downstream capture pass", () => { + const toolResults = [{ id: "call-b", output: "FILE_B", success: true }]; + const normalized = normalize(calls, toolResults); + expect(normalized[0]?.output).toBeUndefined(); + expect(normalized[1]?.output).toBe("FILE_B"); + const captured = capture(normalized, toolResults); + expect(captured[0]?.output).toBeUndefined(); + expect(captured[0]?.success).toBeUndefined(); + expect(captured[1]?.output).toBe("FILE_B"); + }); + + it("does not carry results from a different turn into the current call set", () => { + const result = normalize(calls, [{ id: "previous-turn-call", output: "OLD_RESULT" }]); + expect(result.map((call) => call.output)).toEqual([undefined, undefined]); + }); + + it("preserves call-contained output without assigning a mismatched result", () => { + const result = normalize([{ ...calls[0], output: "OWN_OUTPUT", success: true }], [ + { id: "call-b", output: "WRONG_OUTPUT", error: "unrelated failure" } + ]); + expect(result[0]).toMatchObject({ id: "call-a", output: "OWN_OUTPUT", success: true }); + expect(result[0]?.error).toBeUndefined(); + }); + + it("accepts native call_id and tool_call_id without replacing them with record IDs", () => { + expect(normalize([{ id: "record-a", call_id: "native-a", name: "exec" }], [ + { id: "result-record", tool_call_id: "native-a", output: "DONE" } + ])[0]).toMatchObject({ id: "native-a", output: "DONE" }); + }); + + it("retains aligned legacy results only when both complete arrays have no IDs", () => { + expect(normalize([{ name: "read" }, { name: "write" }], ["FILE", "SAVED"]) + .map((call) => call.output)).toEqual(["FILE", "SAVED"]); + expect(normalize([{ name: "read" }, { name: "write" }], ["SAVED"]) + .map((call) => call.output)).toEqual([undefined, undefined]); + expect(normalize([{ name: "read" }, calls[1]], ["UNIDENTIFIED", { id: "call-b", output: "B" }]) + .map((call) => call.output)).toEqual([undefined, "B"]); + expect(normalize([calls[0]], ["UNIDENTIFIED"])[0]?.output).toBeUndefined(); + }); + + it("preserves explicit errors without guessing failure from an ordinary message", () => { + const toolResults = [{ id: "call-a", error: "permission denied" }, { id: "call-b", output: "B", message: "read finished" }]; + const normalized = normalize(calls, toolResults); + expect(normalized[0]).toMatchObject({ error: "permission denied", success: false }); + expect(normalized[1]?.error).toBeUndefined(); + const captured = capture(calls, toolResults); + expect(captured[0]).toMatchObject({ error: "permission denied", success: false }); + expect(captured[1]).toMatchObject({ output: "B", success: true }); + expect(captured[1]?.error).toBeUndefined(); + }); + + it("retains empty slots in legacy arrays without moving later results forward", () => { + expect(normalize([{ name: "read" }, { name: "write" }], [undefined, "SAVED"]) + .map((call) => call.output)).toEqual([undefined, "SAVED"]); + }); + + it("does not guess between duplicate invocation IDs", () => { + expect(normalize([calls[0]], [{ id: "call-a", output: "FIRST" }, { id: "call-a", output: "SECOND" }])[0]?.output) + .toBeUndefined(); + expect(normalize([calls[0], calls[0]], [{ id: "call-a", output: "SHARED" }]) + .map((call) => call.output)).toEqual([undefined, undefined]); + }); + + it("does not assign a result after dropping a malformed call", () => { + expect(normalize([{ id: "invalid" }, calls[1]], [ + { id: "invalid", output: "INVALID" }, { id: "call-b", output: "B" } + ])).toMatchObject([{ id: "call-b", output: "B" }]); + expect(normalize([], [])).toEqual([]); + }); + + it("uses result ID to select recall sanitization without hiding another tool's output", () => { + const sanitized = sanitizeTurnCompleteRequest({ + sessionId: "session-tools", query: "Read the files.", answer: "Done.", + toolCalls: [{ call_id: "memory", name: "memmy_memory_search" }, calls[1]], + toolResults: [{ call_id: "call-b", output: "FILE_B" }, { call_id: "memory", output: "RECALLED_PRIVATE_CONTEXT" }] + }); + expect(sanitized.toolResults?.[0]).toMatchObject({ call_id: "call-b", output: "FILE_B" }); + expect(sanitized.toolResults?.[1]).toMatchObject({ toolCallId: "memory", output: "[memmy memory result omitted from capture: memmy_memory_search]" }); + expect(normalizeCompleteTurnToolCalls(sanitized).map((call) => call.output)).toEqual([ + "[memmy memory result omitted from capture: memmy_memory_search]", "FILE_B" + ]); + }); + + it("pairs direct algorithm inputs by ID and retains explicit result failures", () => { + const captured = capture(calls, [ + { id: "call-b", output: "B", status: "failed", success: false, error: "permission denied", errorCode: "EACCES" }, + { id: "call-a", output: "A", status: "completed", success: true } + ]); + expect(captured).toMatchObject([ + { id: "call-a", output: "A", status: "completed", success: true }, + { id: "call-b", output: "B", status: "failed", success: false, error: "permission denied", errorCode: "EACCES" } + ]); + }); + + it("persists paired input, output and status in L1 without treating successful text as an error", () => { + const { db, service } = createTestService(); + const session = service.openSession({ namespace: { source: "codex", profileId: "default", userId: "tool-pairing" } }); + const completed = service.completeTurn("turn-preserve-tools", { + sessionId: session.sessionId, + query: "Read the source file and verify the test result.", answer: "The source file was read and the test result was checked.", + toolCalls: calls, + toolResults: [{ id: "call-b", output: "B", status: "failed", success: false, error: "read failed" }, { id: "call-a", output: "A", status: "completed", success: true }] + }); + const row = db.db.prepare("SELECT properties_json FROM memories WHERE id = ?").get(completed.l1MemoryId) as { properties_json: string }; + const persisted = JSON.parse(row.properties_json).internal_info.trace.tool_calls; + expect(persisted).toMatchObject([ + { id: "call-a", input: { path: "a.txt" }, output: "A", status: "completed", success: true }, + { id: "call-b", input: { path: "b.txt" }, output: "B", status: "failed", success: false, error: "read failed" } + ]); + expect(persisted[0].error).toBeUndefined(); + }); +}); diff --git a/docs/cn/memory/sources.mdx b/docs/cn/memory/sources.mdx index 4d0c40c33..6d38b742b 100644 --- a/docs/cn/memory/sources.mdx +++ b/docs/cn/memory/sources.mdx @@ -199,7 +199,7 @@ Memmy 会在 `~/.cursor/hooks.json` 中追加自己的 Hook 条目,不会覆 - `plugins/memmy-memory.js` - `plugins/memmy-memory-config.json` - `commands/memmy-resume.md` -- `AGENTS.md` +- `../../../AGENTS.md` - `skills/memmy-memory/SKILL.md` 如果设置了 `OPENCODE_CONFIG_DIR` 或 `XDG_CONFIG_HOME`,文件会写入对应配置目录。 @@ -221,7 +221,7 @@ Memmy 会在 `~/.cursor/hooks.json` 中追加自己的 Hook 条目,不会覆 - `~/.openclaw/extensions/memmy-memory/` - `~/.openclaw/openclaw.json` - `~/.openclaw/skills/memmy-memory/SKILL.md` -- OpenClaw workspace 下的 `AGENTS.md`(默认 `~/.openclaw/workspace/AGENTS.md`) +- OpenClaw workspace 下的 `../../../AGENTS.md`(默认 `~/.openclaw/workspace/AGENTS.md`) OpenClaw 的 memory slot 只能有一个提供者。如果已安装其他记忆插件,Memmy 会在安装前提示:替换现有插件,或保留现有插件并只安装 Skill。 diff --git a/docs/en/memory/sources.mdx b/docs/en/memory/sources.mdx index b05dd1668..cfb872c56 100644 --- a/docs/en/memory/sources.mdx +++ b/docs/en/memory/sources.mdx @@ -199,7 +199,7 @@ Written or updated under `~/.config/opencode` by default: - `plugins/memmy-memory.js` - `plugins/memmy-memory-config.json` - `commands/memmy-resume.md` -- `AGENTS.md` +- `../../../AGENTS.md` - `skills/memmy-memory/SKILL.md` If `OPENCODE_CONFIG_DIR` or `XDG_CONFIG_HOME` is set, Memmy writes to the corresponding configuration directory instead. @@ -221,7 +221,7 @@ Written or updated by default: - `~/.openclaw/extensions/memmy-memory/` - `~/.openclaw/openclaw.json` - `~/.openclaw/skills/memmy-memory/SKILL.md` -- `AGENTS.md` in the OpenClaw workspace (default `~/.openclaw/workspace/AGENTS.md`) +- `../../../AGENTS.md` in the OpenClaw workspace (default `~/.openclaw/workspace/AGENTS.md`) OpenClaw has one memory-provider slot. If another memory plugin is already active, Memmy asks whether to replace it or keep it and install only the Skill. diff --git a/scripts/internal/mac/build-dmg.sh b/scripts/internal/mac/build-dmg.sh index 629ad0b56..060814ef8 100755 --- a/scripts/internal/mac/build-dmg.sh +++ b/scripts/internal/mac/build-dmg.sh @@ -690,6 +690,7 @@ verify_packaged_mac_unpacked_artifacts() { app_path="$(resolve_packaged_mac_app_path "$target_cpu")" local unpacked_runtime="$app_path/Contents/Resources/app.asar.unpacked/dist/runtime" local packaged_memory_runtime="$app_path/Contents/Resources/memory-runtime" + local packaged_agent_source_core="$packaged_memory_runtime/node_modules/@memmy/agent-source-core" local packaged_embedding_model="$app_path/Contents/Resources/embedding-models/$EMBEDDING_MODEL_ID" require_packaged_runtime_file "$app_path/Contents/Resources/app.asar" @@ -699,6 +700,14 @@ verify_packaged_mac_unpacked_artifacts() { require_packaged_runtime_file "$packaged_memory_runtime/memory-runtime.json" require_packaged_runtime_file "$packaged_memory_runtime/dist/src/server/index.js" require_packaged_runtime_file "$packaged_memory_runtime/dist/src/cli/index.js" + require_packaged_runtime_file "$packaged_memory_runtime/dist/src/agent-source/integration/workspace-bridge/memmy-workspace-bridge.mjs" + require_packaged_runtime_file "$packaged_agent_source_core/package.json" + require_packaged_runtime_file "$packaged_agent_source_core/dist/src/index.js" + require_packaged_runtime_file "$packaged_agent_source_core/dist/src/codex-source-turn.js" + if [ -L "$packaged_agent_source_core" ]; then + echo "Packaged offline Memory agent source core must not be a symbolic link." >&2 + exit 1 + fi require_packaged_runtime_file "$packaged_memory_runtime/node_modules/better-sqlite3/build/Release/better_sqlite3.node" require_packaged_runtime_glob "$packaged_memory_runtime/node_modules/sqlite-vec-darwin-$target_cpu/vec0.*" require_packaged_runtime_file "$packaged_memory_runtime/node_modules/onnxruntime-node/bin/napi-v3/darwin/$target_cpu/onnxruntime_binding.node" diff --git a/tests/packaged-runtime-config.test.mjs b/tests/packaged-runtime-config.test.mjs index 6aa6be51c..b972d926f 100644 --- a/tests/packaged-runtime-config.test.mjs +++ b/tests/packaged-runtime-config.test.mjs @@ -185,6 +185,76 @@ describe("packaged desktop runtime configuration", () => { }); }); + it("creates a standalone macOS Memory manifest and stages its workspace parser package", () => { + const root = fixtureRoot(); + const memoryDir = join(root, "Memory"); + const runtimeDir = join(root, "runtime"); + const coreDir = join(root, "AgentSourceCore"); + const buildScript = readFileSync(join( + dirname(fileURLToPath(import.meta.url)), + "..", "scripts", "internal", "mac", "build-dmg.sh", + ), "utf8"); + writeFixtureJson(join(memoryDir, "package.json"), { + version: "2.1.1", + dependencies: { "@memmy/agent-source-core": "0.0.0", zod: "4.4.3" }, + }); + writeFixtureJson(join(root, "package-lock.json"), { + lockfileVersion: 3, + requires: true, + packages: { + "node_modules/@memmy/agent-source-core": { resolved: "AgentSourceCore", link: true }, + AgentSourceCore: { name: "@memmy/agent-source-core", version: "0.0.0" }, + "node_modules/zod": { version: "4.4.3" }, + }, + }); + const manifestFunction = buildScript.slice( + buildScript.indexOf("create_memory_runtime_manifest() {"), + buildScript.indexOf("\nprune_better_sqlite3_build_artifacts()"), + ); + const generated = spawnSync("bash", ["-c", `${manifestFunction}\ncreate_memory_runtime_manifest "$MEMORY_RUNTIME_DIR"`], { + encoding: "utf8", + env: { ...process.env, ROOT_DIR: root, MEMORY_DIR: memoryDir, MEMORY_RUNTIME_DIR: runtimeDir, TARGET_CPU: "arm64" }, + }); + expect(generated.status, generated.stderr).toBe(0); + expect(JSON.parse(readFileSync(join(runtimeDir, "package.json"), "utf8")).dependencies).toEqual({ + "@memmy/agent-source-core": "file:../../../../../../AgentSourceCore", + zod: "4.4.3", + }); + const lock = JSON.parse(readFileSync(join(runtimeDir, "package-lock.json"), "utf8")); + expect(lock.packages[""].dependencies).toEqual({ + "@memmy/agent-source-core": "file:../../../../../../AgentSourceCore", + zod: "4.4.3", + }); + expect(lock.packages["node_modules/@memmy/agent-source-core"]).toBeUndefined(); + expect(lock.packages["node_modules/zod"].version).toBe("4.4.3"); + + writeFixtureJson(join(coreDir, "package.json"), { + name: "@memmy/agent-source-core", version: "0.0.0", type: "module", main: "./dist/src/index.js", + }); + mkdirSync(join(coreDir, "dist", "src"), { recursive: true }); + writeFileSync(join(coreDir, "dist", "src", "index.js"), 'export { readCodexSourceTurn } from "./codex-source-turn.js";\n'); + writeFileSync(join(coreDir, "dist", "src", "codex-source-turn.js"), 'export const readCodexSourceTurn = () => "packaged-parser";\n'); + mkdirSync(join(memoryDir, "dist", "src"), { recursive: true }); + mkdirSync(join(memoryDir, "dist", "viewer"), { recursive: true }); + mkdirSync(join(memoryDir, "adapters"), { recursive: true }); + mkdirSync(join(root, "App", "memmy-agent", "dist"), { recursive: true }); + mkdirSync(join(runtimeDir, "memmy-agent"), { recursive: true }); + const stageStart = buildScript.indexOf('mkdir -p "$RUNTIME_DIR/memory/dist"'); + const stageEnd = buildScript.indexOf('\nverify_office_rendering_bundle', stageStart); + const staged = spawnSync("bash", ["-c", buildScript.slice(stageStart, stageEnd)], { + encoding: "utf8", + env: { ...process.env, ROOT_DIR: root, RUNTIME_DIR: runtimeDir, MEMORY_DIR: memoryDir, AGENT_DIR: join(root, "App", "memmy-agent") }, + }); + expect(staged.status, staged.stderr).toBe(0); + expect(existsSync(join(runtimeDir, "memory", "AgentSourceCore", "package.json"))).toBe(true); + expect(existsSync(join(runtimeDir, "memory", "AgentSourceCore", "dist", "src", "index.js"))).toBe(true); + expect(buildScript).toContain('npm ci --prefix "$RUNTIME_DIR/memory" --omit=dev --install-links'); + expect(buildScript).toContain('require_packaged_runtime_file "$packaged_agent_source_core/dist/src/index.js"'); + expect(buildScript).toContain('require_packaged_runtime_file "$packaged_agent_source_core/dist/src/codex-source-turn.js"'); + expect(buildScript).toContain('require_packaged_runtime_file "$packaged_memory_runtime/dist/src/agent-source/integration/workspace-bridge/memmy-workspace-bridge.mjs"'); + expect(buildScript).toContain('[ -L "$packaged_agent_source_core" ]'); + }); + it("validates desktop and Memory ASAR versions against independent authorities", async () => { const root = fixtureRoot(); const verifier = join(