From 259fd913bed9a4fe1dac7a9d3d042ce898fead08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E4=B8=96=E8=80=80?= Date: Tue, 8 Sep 2026 11:52:21 +0800 Subject: [PATCH 01/13] fix(release): sanitize public release notes --- .github/workflows/github-draft-release-v2.yml | 18 +- scripts/sanitize-release-notes.mjs | 166 ++++++++++++++++++ tests/release-workflow.test.ts | 110 +++++++++++- 3 files changed, 280 insertions(+), 14 deletions(-) create mode 100644 scripts/sanitize-release-notes.mjs diff --git a/.github/workflows/github-draft-release-v2.yml b/.github/workflows/github-draft-release-v2.yml index badaa064e..ef10261c9 100644 --- a/.github/workflows/github-draft-release-v2.yml +++ b/.github/workflows/github-draft-release-v2.yml @@ -692,6 +692,13 @@ jobs: fi fi + sanitized_notes="${notes}.sanitized" + if ! node scripts/sanitize-release-notes.mjs "$notes" "$sanitized_notes"; then + echo "::error title=Release notes sanitization failed::Reserved Doc Agent metadata was malformed or could not be removed safely. Manual recovery: inspect the selected release notes source and keep internal audit data in the structured Release assets." >&2 + exit 1 + fi + mv "$sanitized_notes" "$notes" + cat >> "$notes" < - EOF - name: Build auditable release evidence diff --git a/scripts/sanitize-release-notes.mjs b/scripts/sanitize-release-notes.mjs new file mode 100644 index 000000000..8ecdb72b3 --- /dev/null +++ b/scripts/sanitize-release-notes.mjs @@ -0,0 +1,166 @@ +#!/usr/bin/env node + +import { readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; + +const RESERVED_MARKERS = [ + /^|$)/, + /^|$)/, +]; + +function isReservedMarker(value) { + return RESERVED_MARKERS.some((pattern) => pattern.test(value)); +} + +function reservedMarkerOffset(line) { + let offset = line.indexOf(""); + if (closeOffset === -1) continue; + + if (line.slice(closeOffset + 3).trim() !== "") { + throw new Error( + `Reserved release metadata must occupy complete lines (line ${lineNumber})`, + ); + } + + removingReservedComment = false; + continue; + } + + if (fence) { + publicLines.push(line); + if (isFenceClosing(line, fence)) fence = null; + continue; + } + + const openingFence = fenceOpening(line); + if (openingFence) { + fence = openingFence; + publicLines.push(line); + continue; + } + + const markerOffset = reservedMarkerOffset(line); + if (markerOffset === -1) { + publicLines.push(line); + continue; + } + + assertCommentOccupiesCompleteLines(line, markerOffset, lineNumber); + const closeOffset = line.indexOf("-->", markerOffset + 4); + if (closeOffset === -1) { + removingReservedComment = true; + reservedCommentStart = lineNumber; + continue; + } + + if (line.slice(closeOffset + 3).trim() !== "") { + throw new Error( + `Reserved release metadata must occupy complete lines (line ${lineNumber})`, + ); + } + } + + if (removingReservedComment) { + throw new Error( + `Unterminated reserved release metadata starting at line ${reservedCommentStart}`, + ); + } + + const publicMarkdown = publicLines.join("\n").trimEnd(); + if (publicMarkdown.trim() === "") { + throw new Error("Release notes contain no public content after sanitization"); + } + + const sanitized = `${publicMarkdown}\n`; + assertNoReservedMetadata(sanitized); + return sanitized; +} + +function assertNoReservedMetadata(markdown) { + const lines = markdown.split(/\r?\n/); + let fence = null; + + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index]; + if (fence) { + if (isFenceClosing(line, fence)) fence = null; + continue; + } + + const openingFence = fenceOpening(line); + if (openingFence) { + fence = openingFence; + continue; + } + + if (reservedMarkerOffset(line) !== -1) { + throw new Error( + `Reserved release metadata remains after sanitization (line ${index + 1})`, + ); + } + } +} + +function main() { + const [inputArg, outputArg, ...extraArgs] = process.argv.slice(2); + if (!inputArg || !outputArg || extraArgs.length > 0) { + throw new Error("Usage: node scripts/sanitize-release-notes.mjs "); + } + + const inputPath = resolve(inputArg); + const outputPath = resolve(outputArg); + if (inputPath === outputPath) { + throw new Error("Input and output paths must be different"); + } + + const markdown = readFileSync(inputPath, "utf8"); + const sanitized = sanitizeReleaseNotes(markdown); + writeFileSync(outputPath, sanitized, "utf8"); +} + +try { + main(); +} catch (error) { + console.error(`sanitize-release-notes: ${error instanceof Error ? error.message : error}`); + process.exitCode = 1; +} diff --git a/tests/release-workflow.test.ts b/tests/release-workflow.test.ts index b28aecd3f..12b0b0d15 100644 --- a/tests/release-workflow.test.ts +++ b/tests/release-workflow.test.ts @@ -9,6 +9,7 @@ const repoRoot = resolve(import.meta.dirname, ".."); const legacyWorkflowPath = resolve(repoRoot, ".github/workflows/github-release.yml"); const draftWorkflowPath = resolve(repoRoot, ".github/workflows/github-draft-release-v2.yml"); const releaseCompareScriptPath = resolve(repoRoot, "scripts/build-release-compare.mjs"); +const releaseNotesSanitizerPath = resolve(repoRoot, "scripts/sanitize-release-notes.mjs"); const ossIntegrityScriptPath = resolve(repoRoot, "scripts/internal/shared/oss-object-integrity.mjs"); const draftSource = readFileSync(draftWorkflowPath, "utf8"); const releaseCompareSource = readFileSync(releaseCompareScriptPath, "utf8"); @@ -57,6 +58,103 @@ function readJson(relativePath: string): { return JSON.parse(readFileSync(resolve(repoRoot, relativePath), "utf8")); } +function runReleaseNotesSanitizer(markdown: string) { + const tempDir = mkdtempSync(resolve(tmpdir(), "memmy-release-notes-sanitizer-")); + const inputPath = resolve(tempDir, "input.md"); + const outputPath = resolve(tempDir, "output.md"); + writeFileSync(inputPath, markdown); + + const result = spawnSync( + "node", + [releaseNotesSanitizerPath, inputPath, outputPath], + { cwd: repoRoot, encoding: "utf8" }, + ); + + return { + result, + output: existsSync(outputPath) ? readFileSync(outputPath, "utf8") : "", + }; +} + +describe("public release notes sanitizer", () => { + it("removes reserved audit comments but preserves public and fenced content", () => { + const { result, output } = runReleaseNotesSanitizer(` +# Memmy v1.1.3 + +Public release notes. + + + + + + + +\`\`\`markdown + + +\`\`\` + +~~~text + +~~~ +`); + + expect(result.status, result.stderr).toBe(0); + expect(output).toContain("# Memmy v1.1.3"); + expect(output).toContain("Public release notes."); + expect(output).toContain(""); + expect(output).toContain(""); + expect(output).toContain("inside: backtick-fence"); + expect(output).toContain("inside: tilde-fence"); + expect(output).not.toContain("memmy-official-changelog-v2"); + expect(output).not.toContain("target_sha: abc123"); + expect(output.endsWith("\n")).toBe(true); + }); + + it("fails closed for unterminated or inline reserved metadata", () => { + const unterminated = runReleaseNotesSanitizer(` +# Memmy + +\n", + ); + expect(inline.result.status).not.toBe(0); + expect(inline.result.stderr).toContain("must occupy complete lines"); + expect(inline.output).toBe(""); + + const afterOrdinaryComment = runReleaseNotesSanitizer( + " \n", + ); + expect(afterOrdinaryComment.result.status).not.toBe(0); + expect(afterOrdinaryComment.result.stderr).toContain("must occupy complete lines"); + expect(afterOrdinaryComment.output).toBe(""); + + const metadataOnly = runReleaseNotesSanitizer( + "\n", + ); + expect(metadataOnly.result.status).not.toBe(0); + expect(metadataOnly.result.stderr).toContain("no public content"); + expect(metadataOnly.output).toBe(""); + }); +}); + describe("Memmy release workflow metadata", () => { it("keeps Memmy metadata aligned while preserving the independent Memory version", () => { const version = readJson("package.json").version; @@ -451,6 +549,15 @@ describe("GitHub Draft Release v2 workflow", () => { expect(releaseNotes).toContain("Release notes generation produced an empty body"); expect(releaseNotes).toContain("RELEASE_NOTES_SOURCE.json"); expect(releaseNotes).toContain("QUALITY_REPORT.json"); + expect(existsSync(releaseNotesSanitizerPath)).toBe(true); + expect(releaseNotes).toContain( + 'node scripts/sanitize-release-notes.mjs "$notes" "$sanitized_notes"', + ); + expect(releaseNotes).toContain('mv "$sanitized_notes" "$notes"'); + expect(releaseNotes).toContain("Release notes sanitization failed"); + expect(releaseNotes).not.toContain("|$)/, /^|$)/, ]; +const CJK_RE = /[\u3040-\u30ff\u3400-\u9fff\uf900-\ufaff]/; function isReservedMarker(value) { return RESERVED_MARKERS.some((pattern) => pattern.test(value)); @@ -41,7 +42,130 @@ function assertCommentOccupiesCompleteLines(line, markerOffset, lineNumber) { } } -export function sanitizeReleaseNotes(markdown) { +function removeCjkHeadingSections(markdown) { + const lines = markdown.split(/\r?\n/); + const output = []; + let fence = null; + let htmlComment = false; + let skippedHeadingDepth = null; + + for (const line of lines) { + if (fence) { + if (skippedHeadingDepth === null) output.push(line); + if (isFenceClosing(line, fence)) fence = null; + continue; + } + + const openingFence = fenceOpening(line); + if (openingFence) { + fence = openingFence; + if (skippedHeadingDepth === null) output.push(line); + continue; + } + + if (htmlComment) { + if (skippedHeadingDepth === null) output.push(line); + if (line.includes("-->")) htmlComment = false; + continue; + } + if (line.includes("", line.indexOf("")) htmlComment = false; + continue; + } + const commentOffset = line.indexOf("", commentOffset + 4)) htmlComment = true; + const visiblePrefix = visibleLineForLanguageValidation(line.slice(0, commentOffset)); + if (CJK_RE.test(visiblePrefix)) { + throw new Error( + `English public release notes contain visible CJK text (line ${index + 1})`, + ); + } + continue; + } + + const visible = visibleLineForLanguageValidation(line); + if (CJK_RE.test(visible)) { + throw new Error( + `English public release notes contain visible CJK text (line ${index + 1})`, + ); + } + if ( + /[A-Za-z]/.test(visible) && + !/^[ \t]{0,3}#{1,6}[ \t]/.test(visible) && + !/^[ \t]*(?:[-*_][ \t]*){3,}$/.test(visible) + ) { + hasBodyContent = true; + } + } + + if (!hasBodyContent) { + throw new Error("English public release notes contain no English body content"); + } +} + +function normalizePublicLanguage(markdown, publicLanguage) { + if (!publicLanguage) return markdown; + if (publicLanguage !== "en") { + throw new Error(`Unsupported public release language: ${publicLanguage}`); + } + const normalized = removeCjkHeadingSections(markdown).trimEnd(); + assertEnglishPublicBody(normalized); + return normalized; +} + +export function sanitizeReleaseNotes(markdown, { publicLanguage = "" } = {}) { const lines = markdown.split(/\r?\n/); const publicLines = []; let fence = null; @@ -106,7 +230,10 @@ export function sanitizeReleaseNotes(markdown) { ); } - const publicMarkdown = publicLines.join("\n").trimEnd(); + const publicMarkdown = normalizePublicLanguage( + publicLines.join("\n").trimEnd(), + publicLanguage, + ); if (publicMarkdown.trim() === "") { throw new Error("Release notes contain no public content after sanitization"); } @@ -142,9 +269,17 @@ function assertNoReservedMetadata(markdown) { } function main() { - const [inputArg, outputArg, ...extraArgs] = process.argv.slice(2); - if (!inputArg || !outputArg || extraArgs.length > 0) { - throw new Error("Usage: node scripts/sanitize-release-notes.mjs "); + const [inputArg, outputArg, languageFlag, languageArg, ...extraArgs] = process.argv.slice(2); + const hasLanguage = languageFlag !== undefined || languageArg !== undefined; + if ( + !inputArg || + !outputArg || + extraArgs.length > 0 || + (hasLanguage && (languageFlag !== "--language" || !languageArg)) + ) { + throw new Error( + "Usage: node scripts/sanitize-release-notes.mjs [--language en]", + ); } const inputPath = resolve(inputArg); @@ -154,7 +289,9 @@ function main() { } const markdown = readFileSync(inputPath, "utf8"); - const sanitized = sanitizeReleaseNotes(markdown); + const sanitized = sanitizeReleaseNotes(markdown, { + publicLanguage: languageArg || "", + }); writeFileSync(outputPath, sanitized, "utf8"); } diff --git a/tests/release-workflow.test.ts b/tests/release-workflow.test.ts index 12b0b0d15..ada6a14c1 100644 --- a/tests/release-workflow.test.ts +++ b/tests/release-workflow.test.ts @@ -58,7 +58,7 @@ function readJson(relativePath: string): { return JSON.parse(readFileSync(resolve(repoRoot, relativePath), "utf8")); } -function runReleaseNotesSanitizer(markdown: string) { +function runReleaseNotesSanitizer(markdown: string, publicLanguage?: "en") { const tempDir = mkdtempSync(resolve(tmpdir(), "memmy-release-notes-sanitizer-")); const inputPath = resolve(tempDir, "input.md"); const outputPath = resolve(tempDir, "output.md"); @@ -66,7 +66,12 @@ function runReleaseNotesSanitizer(markdown: string) { const result = spawnSync( "node", - [releaseNotesSanitizerPath, inputPath, outputPath], + [ + releaseNotesSanitizerPath, + inputPath, + outputPath, + ...(publicLanguage ? ["--language", publicLanguage] : []), + ], { cwd: repoRoot, encoding: "utf8" }, ); @@ -153,6 +158,84 @@ schema_version: 2 expect(metadataOnly.result.stderr).toContain("no public content"); expect(metadataOnly.output).toBe(""); }); + + it("keeps only English sections when a reviewed source contains parallel Chinese sections", () => { + const { result, output } = runReleaseNotesSanitizer( + `# Memmy v1.1.3 + +## Fixes + +- Fixed packaged Memory startup. + +## 修复 + +- 修复随包 Memory 的启动问题。 + +## Upgrade notes + +- Memory remains independently versioned. + +## 升级说明 + +- Memory 继续独立发版。 +`, + "en", + ); + + expect(result.status, result.stderr).toBe(0); + expect(output).toContain("## Fixes"); + expect(output).toContain("Fixed packaged Memory startup."); + expect(output).toContain("## Upgrade notes"); + expect(output).not.toMatch(/[\u3040-\u30ff\u3400-\u9fff\uf900-\ufaff]/); + expect(output.match(/^## Fixes$/gm)).toHaveLength(1); + }); + + it("rejects Chinese prose that remains inside an English section", () => { + const { result, output } = runReleaseNotesSanitizer( + `# Memmy v1.1.3 + +## Fixes + +- 修复随包 Memory 的启动问题。 +`, + "en", + ); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("English public release notes contain visible CJK text"); + expect(output).toBe(""); + }); + + it("allows CJK characters in code spans while validating English prose", () => { + const { result, output } = runReleaseNotesSanitizer( + `# Memmy v1.1.3 + +## Fixes + +- Fixed startup when the configured path is \`C:\\\\用户\\\\Memmy\`. +`, + "en", + ); + + expect(result.status, result.stderr).toBe(0); + expect(output).toContain("`C:\\\\用户\\\\Memmy`"); + }); + + it("normalizes the real v1.1.3 reviewed notes to a single English public body", () => { + const notes = readFileSync( + resolve(repoRoot, ".github/release-notes/v1.1.3.md"), + "utf8", + ); + const { result, output } = runReleaseNotesSanitizer(notes, "en"); + + expect(result.status, result.stderr).toBe(0); + expect(output).toContain("# Memmy v1.1.3"); + expect(output).toContain("## Fixes"); + expect(output).toContain("## Upgrade notes"); + expect(output).not.toMatch(/[\u3040-\u30ff\u3400-\u9fff\uf900-\ufaff]/); + expect(output.match(/^## Fixes$/gm)).toHaveLength(1); + expect(output.match(/^## Upgrade notes$/gm)).toHaveLength(1); + }); }); describe("Memmy release workflow metadata", () => { @@ -537,6 +620,7 @@ describe("GitHub Draft Release v2 workflow", () => { expect(releaseNotes).toContain("DOC_AGENT_RELEASE_NOTES_REQUEST.json"); expect(releaseNotes).toContain("MEMMY_RELEASE_STYLE_EXAMPLES.json"); expect(releaseNotes).toContain("candidate_count: 3"); + expect(releaseNotes).toContain('public_release_language: "en"'); expect(releaseNotes).toContain(".release_notes_md // .release_notes_markdown"); expect(releaseNotes).toContain("Doc Agent draft configuration missing"); expect(releaseNotes).toContain("Doc Agent draft generation failed"); @@ -551,10 +635,12 @@ describe("GitHub Draft Release v2 workflow", () => { expect(releaseNotes).toContain("QUALITY_REPORT.json"); expect(existsSync(releaseNotesSanitizerPath)).toBe(true); expect(releaseNotes).toContain( - 'node scripts/sanitize-release-notes.mjs "$notes" "$sanitized_notes"', + 'node scripts/sanitize-release-notes.mjs "$notes" "$sanitized_notes" --language en', ); expect(releaseNotes).toContain('mv "$sanitized_notes" "$notes"'); expect(releaseNotes).toContain("Release notes sanitization failed"); + expect(releaseNotes).toContain('public_release_language: "en"'); + expect(releaseNotes).toContain("language_validation"); expect(releaseNotes).not.toContain(" + + EOF - name: Build auditable release evidence if: ${{ steps.release.outputs.preflight_level == 'full' }} diff --git a/.github/workflows/memory-release.yml b/.github/workflows/memory-release.yml index b4a9cf02e..3dfe2991e 100644 --- a/.github/workflows/memory-release.yml +++ b/.github/workflows/memory-release.yml @@ -6,7 +6,7 @@ on: version: description: Memory version (X.Y.Z) required: true - default: 2.1.2 + default: 2.1.1 push: tags: - "memory-v*" diff --git a/.github/workflows/yunxiao-github-sync.yml b/.github/workflows/yunxiao-github-sync.yml deleted file mode 100644 index 9f3ddeedb..000000000 --- a/.github/workflows/yunxiao-github-sync.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Sync GitHub to Yunxiao - -on: - issues: - types: [opened, closed, reopened] - pull_request_target: - types: [opened, closed, reopened] - workflow_dispatch: - inputs: - mode: - description: 运行模式 - required: true - type: choice - default: preflight - options: - - preflight - - backfill - state: - description: 回填 GitHub 数据范围 - required: false - type: choice - default: open - options: - - open - - all - apply: - description: 回填时是否实际写入云效 - required: false - type: boolean - default: false - -permissions: - contents: read - issues: read - pull-requests: read - -concurrency: - group: yunxiao-github-sync-${{ github.repository }} - cancel-in-progress: false - -jobs: - sync: - runs-on: ubuntu-latest - timeout-minutes: 30 - - steps: - # pull_request_target uses the trusted default branch. Never execute the - # workflow or script from an untrusted PR branch. - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.repository.default_branch }} - persist-credentials: false - - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Run sync - env: - YUNXIAO_TOKEN: ${{ secrets.YUNXIAO_TOKEN }} - YUNXIAO_API_BASE_URL: ${{ vars.YUNXIAO_API_BASE_URL }} - # memmy-agent defaults to the memmy Yunxiao project. Repository - # Variables can override these values for a fork or another space. - YUNXIAO_PROJECT_ID: ${{ vars.YUNXIAO_PROJECT_ID || '1832b179386e24414d3891e244' }} - YUNXIAO_PROJECT_NAME: ${{ vars.YUNXIAO_PROJECT_NAME || 'memmy' }} - YUNXIAO_PARENT_ID: ${{ vars.YUNXIAO_PARENT_ID }} - YUNXIAO_WORKITEM_CATEGORY: ${{ vars.YUNXIAO_WORKITEM_CATEGORY }} - YUNXIAO_WORKITEM_TYPE_NAME: ${{ vars.YUNXIAO_WORKITEM_TYPE_NAME }} - YUNXIAO_PRIORITY_NAME: ${{ vars.YUNXIAO_PRIORITY_NAME }} - YUNXIAO_DAYS_TO_FINISH: ${{ vars.YUNXIAO_DAYS_TO_FINISH }} - # Optional. Empty means Yunxiao leaves the new work item unassigned. - YUNXIAO_DEFAULT_ASSIGNEE_NAME: ${{ vars.YUNXIAO_DEFAULT_ASSIGNEE_NAME }} - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - args=(--mode "${{ inputs.mode }}" --state "${{ inputs.state }}") - if [ "${{ inputs.mode }}" = "backfill" ] && [ "${{ inputs.apply }}" != "true" ]; then - args+=(--dry-run) - fi - elif [ "${{ github.event_name }}" = "issues" ] || [ "${{ github.event_name }}" = "pull_request_target" ]; then - args=(--mode event --apply) - else - args=(--mode preflight) - fi - - python scripts/yunxiao_github_sync.py "${args[@]}" diff --git a/AgentSourceCore/src/index.test.ts b/AgentSourceCore/src/index.test.ts index 16cdccb42..b3eea8117 100644 --- a/AgentSourceCore/src/index.test.ts +++ b/AgentSourceCore/src/index.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { TURN_CONTENT_MAX_BYTES, conversationContentHash, orderedTurns, renderTurn, renderTurnClipped, type ConversationMessage } from "./index.js"; +import { TURN_CONTENT_MAX_BYTES, conversationContentHash, orderedTurns, renderTurnClipped, type ConversationMessage } from "./index.js"; const message = (id: string, role: ConversationMessage["role"], content: string, createdAt: string): ConversationMessage => ({ messageId: id, sourceId: "fixture", conversationId: "conversation", role, content, createdAt, @@ -45,29 +45,4 @@ describe("agent source core", () => { const messages = [message("u", "user", "hello", "2026-01-01T00:00:00Z"), message("a", "assistant", "world", "2026-01-01T00:00:01Z")]; expect(renderTurnClipped(messages, TURN_CONTENT_MAX_BYTES)).toBe("## user\n\nhello\n\n## assistant\n\nworld"); }); - - it.each([ - ["ASCII", "x".repeat(600_000)], - ["Chinese and emoji", "汉🙂".repeat(160_000)], - ["escaped control characters", "\u0001".repeat(400_000)], - ])("bounds both raw UTF-8 and JSON bytes for %s", (_label, text) => { - const messages = [message("u", "user", "request", "2026-01-01T00:00:00Z"), message("t", "tool", text, "2026-01-01T00:00:01Z")]; - const content = renderTurnClipped(messages); - expect(Buffer.byteLength(content)).toBeLessThanOrEqual(TURN_CONTENT_MAX_BYTES); - expect(Buffer.byteLength(JSON.stringify(content))).toBeLessThanOrEqual(1024 * 1024); - expect(content).not.toContain("\ufffd"); - const truncation = /\n\n\[\.\.\. truncated (\d+) bytes of tool output \.\.\.\]$/u.exec(content); - expect(truncation).not.toBeNull(); - expect(Number(truncation![1])).toBe(Buffer.byteLength(renderTurn(messages)) - Buffer.byteLength(content.slice(0, truncation!.index))); - }); - - it("preserves quoted text and newlines when both budgets allow the complete turn", () => { - const messages = [message("u", "user", '"\\\n'.repeat(150_000), "2026-01-01T00:00:00Z")]; - expect(renderTurnClipped(messages)).toBe(renderTurn(messages)); - }); - - it.each([0, 1, 8, 48])("includes the truncation marker inside a %i-byte budget", (maxBytes) => { - const content = renderTurnClipped([message("u", "user", "x".repeat(1000), "2026-01-01T00:00:00Z")], maxBytes); - expect(Buffer.byteLength(content)).toBeLessThanOrEqual(maxBytes); - }); }); diff --git a/AgentSourceCore/src/index.ts b/AgentSourceCore/src/index.ts index e8713cfbb..b00d32933 100644 --- a/AgentSourceCore/src/index.ts +++ b/AgentSourceCore/src/index.ts @@ -249,9 +249,8 @@ export function legacyTurnId(turn: ImportedTurn): string { return `${turn.sourceId}:${createHash("sha256").update(stableTurnIdentity(turn)).digest("hex").slice(0, 24)}`; } -/** Raw UTF-8 content limit; JSON escaping has a separate transport budget. */ +/** Leaves ample room for JSON escaping and the add-memory envelope. */ export const TURN_CONTENT_MAX_BYTES = 512 * 1024; -const TURN_CONTENT_MAX_JSON_BYTES = 1024 * 1024; /** * Renders a whole turn as one memory body. Agent-source scans deliberately keep @@ -263,31 +262,19 @@ const TURN_CONTENT_MAX_JSON_BYTES = 1024 * 1024; export function renderTurnClipped(messages: readonly ConversationMessage[], maxBytes = TURN_CONTENT_MAX_BYTES): string { const content = renderTurn(messages); const bytes = Buffer.byteLength(content); - if (bytes <= maxBytes && jsonContentBytes(content) + 2 <= TURN_CONTENT_MAX_JSON_BYTES) return content; - const marker = (omitted: number) => `\n\n[... truncated ${omitted} bytes of tool output ...]`; - // Reserving the largest possible omission count also bounds the final marker. - const reservedMarker = marker(bytes); - const markerBytes = Buffer.byteLength(reservedMarker); - const rawBudget = Math.max(0, maxBytes); - if (rawBudget <= markerBytes) return clipUtf8(reservedMarker, rawBudget, TURN_CONTENT_MAX_JSON_BYTES - 2); - const prefix = clipUtf8(content, rawBudget - markerBytes, TURN_CONTENT_MAX_JSON_BYTES - 2 - jsonContentBytes(reservedMarker)); - return `${prefix}${marker(bytes - Buffer.byteLength(prefix))}`; + if (bytes <= maxBytes) return content; + const marker = `\n\n[... truncated ${bytes - maxBytes} bytes of tool output ...]`; + const budget = Math.max(0, maxBytes - Buffer.byteLength(marker)); + return `${clipUtf8(content, budget)}${marker}`; } -function jsonContentBytes(value: string): number { - return Buffer.byteLength(JSON.stringify(value)) - 2; -} - -function clipUtf8(value: string, maxBytes: number, maxJsonBytes: number): string { +function clipUtf8(value: string, maxBytes: number): string { let bytes = 0; - let jsonBytes = 0; let end = 0; for (const character of value) { const characterBytes = Buffer.byteLength(character); - const characterJsonBytes = jsonContentBytes(character); - if (bytes + characterBytes > maxBytes || jsonBytes + characterJsonBytes > maxJsonBytes) break; + if (bytes + characterBytes > maxBytes) break; bytes += characterBytes; - jsonBytes += characterJsonBytes; end += character.length; } return value.slice(0, end); diff --git a/App/backend/src/project-version.ts b/App/backend/src/project-version.ts index 9ee6bb759..cc846a3e3 100644 --- a/App/backend/src/project-version.ts +++ b/App/backend/src/project-version.ts @@ -1,2 +1,2 @@ /** Generated from the root package.json by scripts/sync-project-version.mjs. */ -export const MEMMY_VERSION = "1.1.4"; +export const MEMMY_VERSION = "1.1.3"; diff --git a/App/backend/src/services/tests/agent-source-service.test.ts b/App/backend/src/services/tests/agent-source-service.test.ts index e21419449..2bbd25cd2 100644 --- a/App/backend/src/services/tests/agent-source-service.test.ts +++ b/App/backend/src/services/tests/agent-source-service.test.ts @@ -1,7 +1,6 @@ /** Agent source service tests. */ import { DatabaseSync } from "node:sqlite"; import { MANAGED_AGENT_DISCOVERY_PENDING_DATA_PATH } from "@memmy/local-api-contracts"; -import { legacyTurnId, legacyTurnRequestId } from "@memmy/agent-source-core"; import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -40,109 +39,6 @@ afterEach(() => { }); describe("agent source service", () => { - describe("persistent scan turn boundaries", () => { - it("stores one oversized tool turn once and reuses its legacy idempotency keys", async () => { - tempDir = mkdtempSync(join(tmpdir(), "memmy-persistent-one-turn-")); - const messages = createCompleteMemoryMessages("cursor", 1, "2026-05-28T10:00:00.000Z", { includeTool: true }) - .map((message) => message.role === "tool" ? { ...message, content: "Tool calls:\n\n- tool_1\n\n".repeat(30_000) } : message); - const turn = { sourceId: "cursor", conversationId: messages[0]!.conversationId, turnIndex: 0, messages }; - const memoryClient = createMockMemoryClient(); - const added: Parameters[0][] = []; - const service = createService({ - scanStoreDirectory: tempDir, - adapters: [createFakeAdapter("cursor", messages)], - memoryClient: { - ...memoryClient, - async addMemory(input) { - added.push(input); - return { ...await memoryClient.addMemory(input), id: "oversized-turn", duplicate: added.length > 1 }; - }, - async getMemoryProcessingStatus(ids) { - return { items: ids.map((memoryId) => ({ memoryId, state: "ready" as const, attemptCount: 0, manualRetryCount: 0, retryAction: "retry" as const, updatedAt: "2026-05-28T10:00:00.000Z" })), serverTime: "2026-05-28T10:00:00.000Z" }; - } - } - }); - - const first = await service.scanOne("cursor", { mode: "full" }); - expect(first.errors).toEqual([]); - expect(added).toHaveLength(1); - expect(added[0]).toMatchObject({ requestId: legacyTurnRequestId(turn), turnId: legacyTurnId(turn) }); - expect(added[0]?.content).toContain("truncated"); - const replay = await service.scanOne("cursor", { mode: "full" }); - expect(replay.errors).toEqual([]); - expect(added).toHaveLength(2); - expect(added[1]?.requestId).toBe(added[0]?.requestId); - expect(added[1]?.turnId).toBe(added[0]?.turnId); - expect(replay.skipped).toBe(messages.length); - expect(replay.memoryIdCount).toBe(0); - }); - - it.each([ - ["after the watermark", "2026-05-28T10:01:53.000Z", ["query 1"]], - ["ending exactly at the watermark", "2026-05-28T10:01:52.000Z", ["query 2", "query 1"]], - ["when the newest turn ends exactly at the watermark", "2026-05-28T10:02:02.000Z", ["query 1"]] - ] as const)("imports only complete turns %s from a changed long conversation", async (_label, since, expectedTitles) => { - tempDir = mkdtempSync(join(tmpdir(), "memmy-persistent-boundary-")); - const repository = createRepository(); - repository.upsertSource({ sourceId: "cursor", displayName: "Cursor", dataPath: "/tmp/cursor", builtin: true }); - repository.upsertScanWatermark({ sourceId: "cursor", mode: "incremental", baselineAt: since, latestSeenCreatedAt: since, updatedAt: since }); - const memoryClient = createMockMemoryClient(); - const added: Parameters[0][] = []; - const service = createService({ - repository, - scanStoreDirectory: tempDir, - adapters: [createFakeAdapter("cursor", createCompleteMemoryMessages("cursor", 3, "2026-05-28T10:02:00.000Z") - .map((message) => ({ ...message, conversationId: "long-conversation" })))], - memoryClient: { - ...memoryClient, - async addMemory(input) { added.push(input); return memoryClient.addMemory(input); }, - async getMemoryProcessingStatus(ids) { - return { items: ids.map((memoryId) => ({ memoryId, state: "ready" as const, attemptCount: 0, manualRetryCount: 0, retryAction: "retry" as const, updatedAt: since })), serverTime: since }; - } - } - }); - - const result = await service.scanOne("cursor", { mode: "incremental" }); - - expect(result.errors).toEqual([]); - expect(added.map((input) => input.title)).toEqual(expectedTitles); - for (const input of added) expect(input.content).toContain(String(input.title).replace("query", "answer")); - expect(repository.getScanWatermark("cursor")?.latestSeenCreatedAt).toBe("2026-05-28T10:02:02.000Z"); - }); - - it.each(["full", "initial_subset"] as const)("preserves the %s history selection with an existing watermark", async (mode) => { - tempDir = mkdtempSync(join(tmpdir(), "memmy-persistent-mode-")); - const repository = createRepository(); - const boundary = "2026-05-28T10:02:02.000Z"; - repository.upsertSource({ sourceId: "cursor", displayName: "Cursor", dataPath: "/tmp/cursor", builtin: true }); - repository.upsertScanWatermark({ sourceId: "cursor", mode: "incremental", baselineAt: boundary, latestSeenCreatedAt: boundary, updatedAt: boundary }); - const added: Parameters[0][] = []; - const memoryClient = createMockMemoryClient(); - const count = mode === "initial_subset" ? 1001 : 3; - const service = createService({ - repository, - scanStoreDirectory: tempDir, - adapters: [createFakeAdapter("cursor", createCompleteMemoryMessages("cursor", count, "2026-05-28T10:02:00.000Z") - .map((message) => ({ ...message, conversationId: "long-conversation" })))], - memoryClient: { - ...memoryClient, - async addMemory(input) { added.push(input); return memoryClient.addMemory(input); }, - async getMemoryProcessingStatus(ids) { - return { items: ids.map((memoryId) => ({ memoryId, state: "ready" as const, attemptCount: 0, manualRetryCount: 0, retryAction: "retry" as const, updatedAt: boundary })), serverTime: boundary }; - } - } - }); - - const result = await service.scanOne("cursor", { mode }); - - expect(result.errors).toEqual([]); - expect(added).toHaveLength(mode === "initial_subset" ? 1000 : 3); - expect(added.map((input) => input.title)).toContain("query 1"); - expect(added.map((input) => input.title)).toContain(`query ${mode === "initial_subset" ? 1000 : 3}`); - if (mode === "initial_subset") expect(added.map((input) => input.title)).not.toContain("query 1001"); - }); - }); - it("lists builtin registry sources together with persisted manual sources", async () => { const repository = createRepository(); repository.upsertSource({ @@ -1603,7 +1499,6 @@ function createService( memoryClient?: MemoryClient; agentSourceAnalytics?: AgentSourceLifecycleAnalytics; getScanPermission?: () => Promise; - scanStoreDirectory?: string; } = {} ): AgentSourceService { return createAgentSourceService({ @@ -1613,7 +1508,6 @@ function createService( memoryClient: options.memoryClient ?? createMockMemoryClient(), agentSourceAnalytics: options.agentSourceAnalytics, getScanPermission: options.getScanPermission, - scanStoreDirectory: options.scanStoreDirectory, skillDistributionService: options.skillDistributionService ?? ({ diff --git a/App/backend/src/services/tests/agent-source-wire-budget.test.ts b/App/backend/src/services/tests/agent-source-wire-budget.test.ts deleted file mode 100644 index 6e1aa5981..000000000 --- a/App/backend/src/services/tests/agent-source-wire-budget.test.ts +++ /dev/null @@ -1,61 +0,0 @@ -import type { Server } from "node:http"; -import { afterEach, describe, expect, it } from "vitest"; -import { legacyTurnId, legacyTurnRequestId, renderTurnClipped, type ConversationMessage } from "@memmy/agent-source-core"; -import { createHttpMemoryClient } from "../../adapters/outbound/memory-client/http-memory-client.js"; -import { createMemoryHttpServer, closeMemoryHttpServer } from "../../../../../Memory/src/server/http.js"; -import type { MemoryService } from "../../../../../Memory/src/service/memory-service.js"; -import type { AgentSourceExecutor } from "../../../../../Memory/src/agent-source/runtime.js"; - -const servers: Server[] = []; -afterEach(async () => { - await Promise.all(servers.splice(0).map((server) => closeMemoryHttpServer(server))); -}); - -describe("scanned turn HTTP request budget", () => { - it.each([ - ["quoted JSON tool output", '{"value":"quoted \\ path"}\n'.repeat(30_000)], - ["JSON-escaped control characters", "\u0001".repeat(400_000)], - ])("stores %s as one memory through the actual Memory HTTP body limit", async (_label, toolContent) => { - const stored: Array<{ content: string }> = []; - const server = createMemoryHttpServer({ - service: { - idempotent(_operation: unknown, _input: unknown, _audit: unknown, run: () => unknown) { return run(); }, - addMemory(input: { content: string }) { - stored.push(input); - return { id: "wire-fixture", kind: "trace", memoryLayer: "L1", status: "activated", title: "one turn", summary: "stored", tags: [], createdAt: "2026-09-08T00:00:00.000Z", serverTime: "2026-09-08T00:00:00.000Z" }; - }, - } as unknown as MemoryService, - auth: { localServiceToken: "fixture-token" }, - agentSourceExecutor: { dispose() {} } as unknown as AgentSourceExecutor, - pluginRuntimeAnalytics: { track() {}, async trackAwait() {}, async flush() {} }, - workerStartupFallbackMs: 60_000, - }); - servers.push(server); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - const address = server.address(); - if (!address || typeof address === "string") throw new Error("expected TCP address"); - const bodies: string[] = []; - const client = createHttpMemoryClient({ baseUrl: `http://127.0.0.1:${address.port}`, token: "fixture-token", timeoutMs: 5_000, maxRetries: 0 }, { - fetchImpl: async (url, init) => { bodies.push(String(init?.body)); return fetch(url, init); }, - }); - const message = (messageId: string, role: ConversationMessage["role"], content: string): ConversationMessage => ({ - messageId, sourceId: "fixture", conversationId: "conversation", role, content, - createdAt: "2026-09-08T00:00:00.000Z", workspacePath: null, gitRoot: null, rawMeta: {}, - }); - const turn = { sourceId: "fixture", conversationId: "conversation", turnIndex: 0, messages: [ - message("u", "user", "one turn"), message("t", "tool", toolContent), message("a", "assistant", "finished"), - ] }; - const content = renderTurnClipped(turn.messages); - - await expect(client.addMemory({ - requestId: legacyTurnRequestId(turn), adapterId: "agent-source:fixture", content, - layer: "L1", title: "one turn", tags: ["agent-source", "fixture"], source: "fixture", - turnId: legacyTurnId(turn), createdAt: turn.messages[0]!.createdAt, deferProcessing: true, - })).resolves.toMatchObject({ id: "wire-fixture" }); - - expect(bodies).toHaveLength(1); - expect(Buffer.byteLength(bodies[0]!)).toBeLessThanOrEqual(2 * 1024 * 1024); - expect(stored).toHaveLength(1); - expect(stored[0]?.content).toBe(content); - }); -}); diff --git a/App/memmy-agent/package-lock.json b/App/memmy-agent/package-lock.json index 3472def38..6f0758424 100644 --- a/App/memmy-agent/package-lock.json +++ b/App/memmy-agent/package-lock.json @@ -1,12 +1,12 @@ { "name": "memmy-agent", - "version": "1.1.4", + "version": "1.1.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "memmy-agent", - "version": "1.1.4", + "version": "1.1.3", "dependencies": { "@anthropic-ai/sdk": "^0.100.1", "@aws-sdk/client-bedrock-runtime": "^3.1061.0", diff --git a/App/memmy-agent/package.json b/App/memmy-agent/package.json index 96294d0c7..455c0596d 100644 --- a/App/memmy-agent/package.json +++ b/App/memmy-agent/package.json @@ -1,6 +1,6 @@ { "name": "memmy-agent", - "version": "1.1.4", + "version": "1.1.3", "description": "TypeScript refactor of memmy's agent runtime.", "type": "module", "main": "./dist/index.js", diff --git a/App/shell/desktop/package.json b/App/shell/desktop/package.json index cf384e397..73731f217 100644 --- a/App/shell/desktop/package.json +++ b/App/shell/desktop/package.json @@ -1,6 +1,6 @@ { "name": "@memmy/desktop", - "version": "1.1.4", + "version": "1.1.3", "private": true, "type": "module", "description": "Memmy desktop client.", diff --git a/App/shell/desktop/tests/packaged-runtime-boundary.test.ts b/App/shell/desktop/tests/packaged-runtime-boundary.test.ts index c11670376..35d1fb2fc 100644 --- a/App/shell/desktop/tests/packaged-runtime-boundary.test.ts +++ b/App/shell/desktop/tests/packaged-runtime-boundary.test.ts @@ -124,7 +124,7 @@ describe("desktop packaged runtime boundaries", () => { yaml: expect.any(String), zod: expect.any(String) }); - expect(memoryPackage.version).toBe("2.1.2"); + expect(memoryPackage.version).toBe("2.1.1"); expect(memoryPackage.dependencies ?? {}).not.toHaveProperty("@memmy/local-api-contracts"); expect(memoryPackage.dependencies ?? {}).not.toHaveProperty("@memmy/migrations"); expect(memoryPackage.scripts?.prebuild).toBeUndefined(); @@ -1565,9 +1565,9 @@ describe("desktop packaged runtime boundaries", () => { expect(source).toContain('cp -R "$MEMORY_DIR/dist/viewer" "$RUNTIME_DIR/memory/dist/viewer"'); expect(source).toContain('cp -R "$MEMORY_DIR/adapters" "$RUNTIME_DIR/memory/adapters"'); expect(source).toContain( - 'npm install --prefix "$RUNTIME_DIR/memory" --package-lock-only --ignore-scripts --install-links --os=darwin --cpu="$TARGET_CPU"' + 'npm install --prefix "$RUNTIME_DIR/memory" --package-lock-only --ignore-scripts --os=darwin --cpu="$TARGET_CPU"' ); - expect(source).toContain('npm ci --prefix "$RUNTIME_DIR/memory" --omit=dev --install-links --os=darwin --cpu="$TARGET_CPU"'); + expect(source).toContain('npm ci --prefix "$RUNTIME_DIR/memory" --omit=dev --os=darwin --cpu="$TARGET_CPU"'); expect(source).not.toContain('MEMORY_RUNTIME_CONTRACTS_DIR'); expect(source).not.toContain('MEMORY_RUNTIME_MIGRATIONS_DIR'); expect(source).toContain("node_modules/.bin/electron-rebuild"); diff --git a/Memory/package.json b/Memory/package.json index 4663df95b..fc8bd3f98 100644 --- a/Memory/package.json +++ b/Memory/package.json @@ -1,6 +1,6 @@ { "name": "@memmy/memory", - "version": "2.1.2", + "version": "2.1.1", "private": true, "type": "module", "main": "./dist/src/index.js", diff --git a/Memory/src/cli/npm/package.json b/Memory/src/cli/npm/package.json index f41e421a0..e49cf4e97 100644 --- a/Memory/src/cli/npm/package.json +++ b/Memory/src/cli/npm/package.json @@ -1,6 +1,6 @@ { "name": "@memtensor/memmy-memory-cli", - "version": "2.1.2", + "version": "2.1.1", "description": "Memmy Memory CLI for local agent memory.", "type": "module", "bin": { diff --git a/Memory/src/storage/db.ts b/Memory/src/storage/db.ts index 8e408b61d..2fbba53fd 100644 --- a/Memory/src/storage/db.ts +++ b/Memory/src/storage/db.ts @@ -1,6 +1,5 @@ import Database from "better-sqlite3"; -import { createHash, randomUUID } from "node:crypto"; -import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { copyFileSync, existsSync, mkdirSync } from "node:fs"; import { basename, dirname, join, resolve } from "node:path"; import { homedir, tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; @@ -84,34 +83,27 @@ function packagedNativeBindingPath(): string | undefined { const moduleDir = dirname(fileURLToPath(import.meta.url)); const candidates = [ resolve(moduleDir, "../../../node_modules/better-sqlite3/build/Release/better_sqlite3.node"), - resolve(moduleDir, "node_modules/better-sqlite3/build/Release/better_sqlite3.node"), - resolve(moduleDir, "../../../../node_modules/better-sqlite3/build/Release/better_sqlite3.node") + resolve(moduleDir, "node_modules/better-sqlite3/build/Release/better_sqlite3.node") ]; const source = candidates.find((candidate) => existsSync(candidate)); if (!source) return undefined; - return packagedNativeAssetPath(source); + const targetDirectory = join(tmpdir(), "memmy-memory-native"); + const target = join(targetDirectory, "better_sqlite3.node"); + // Refresh on every start so an upgraded package cannot reuse a stale + // native addon left by a previous executable with the same temp path. + mkdirSync(targetDirectory, { recursive: true, mode: 0o700 }); + copyFileSync(source, target); + return target; } function packagedNativeAssetPath(source: string): string { if (!(process as NodeJS.Process & { pkg?: unknown }).pkg) return source; if (!existsSync(source)) return source; - const contents = readFileSync(source); - const digest = createHash("sha256").update(contents).digest("hex"); - // Keep upgrades separate from native libraries already loaded by older processes. - const targetDirectory = join(tmpdir(), "memmy-memory-native", digest); + const targetDirectory = join(tmpdir(), "memmy-memory-native"); const target = join(targetDirectory, basename(source)); - const isComplete = () => existsSync(target) && readFileSync(target).equals(contents); - if (isComplete()) return target; - mkdirSync(targetDirectory, { recursive: true, mode: 0o700 }); - const temporaryPath = `${target}.${process.pid}.${randomUUID()}.tmp`; - try { - writeFileSync(temporaryPath, contents, { flag: "wx", mode: 0o600 }); - renameSync(temporaryPath, target); - } catch (error) { - // Another extractor may publish and load the same asset before our rename. - if (!isComplete()) throw error; - } finally { - rmSync(temporaryPath, { force: true }); + if (!existsSync(target)) { + mkdirSync(targetDirectory, { recursive: true, mode: 0o700 }); + copyFileSync(source, target); } return target; } diff --git a/Memory/src/version.ts b/Memory/src/version.ts index c8f47e915..96aa657ed 100644 --- a/Memory/src/version.ts +++ b/Memory/src/version.ts @@ -1,5 +1,5 @@ /** Independent Memory/Local Plugin release identity. */ -export const MEMORY_SERVICE_VERSION = "2.1.2"; +export const MEMORY_SERVICE_VERSION = "2.1.1"; export const MEMORY_VIEWER_VERSION = MEMORY_SERVICE_VERSION; export const MEMORY_PROTOCOL_VERSION = 1; diff --git a/Memory/tests/agent-source-runtime.test.ts b/Memory/tests/agent-source-runtime.test.ts index 0fc0d9c24..ba25c7ffe 100644 --- a/Memory/tests/agent-source-runtime.test.ts +++ b/Memory/tests/agent-source-runtime.test.ts @@ -2,7 +2,6 @@ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "nod import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { legacyTurnId, legacyTurnRequestId } from "@memmy/agent-source-core"; import { createAgentSourceExecutor, createBuiltinSourceRegistry @@ -22,83 +21,6 @@ afterEach(() => { }); describe("standalone Agent source executor", () => { - it("keeps an oversized standalone tool turn as one memory with stable legacy dedup keys", async () => { - const root = tempRoot(); - const adapter = longConversationAdapter(1, "Tool calls:\n\n- tool_1\n\n".repeat(30_000)); - const messages = []; - for await (const message of adapter.scan({})) messages.push(message); - const turn = { sourceId: "fixture-agent", conversationId: messages[0]!.conversationId, turnIndex: 0, messages }; - const addMemory = vi.fn((input: { title: string; content: string; requestId: string; turnId: string }) => ({ id: input.title, duplicate: false })); - const enqueuePendingImportSummaries = vi.fn(); - const executor = createAgentSourceExecutor({ - service: { addMemory, enqueuePendingImportSummaries } as unknown as MemoryService, - configPath: join(root, "config.yaml"), statePath: join(root, "agent-sources.json"), - sourceRegistry: createSourceRegistry([adapter]), - }); - try { - await executor.startScan({ sourceId: "fixture-agent", mode: "full" }); - await vi.waitFor(() => expect(executor.scanStatus().running).toBe(false), { timeout: 5_000 }); - expect(executor.scanStatus().error).toBeNull(); - expect(addMemory).toHaveBeenCalledOnce(); - expect(addMemory.mock.calls[0]?.[0]).toMatchObject({ requestId: legacyTurnRequestId(turn), turnId: legacyTurnId(turn) }); - expect(addMemory.mock.calls[0]?.[0].content).toContain("truncated"); - expect(enqueuePendingImportSummaries).toHaveBeenCalledWith(1000, ["question-0"]); - } finally { await executor.dispose(); } - }); - - it.each([ - ["after the watermark", "2026-08-28T01:00:13.000Z", ["question-2"]], - ["ending exactly at the watermark", "2026-08-28T01:00:12.000Z", ["question-1", "question-2"]], - ["when the newest turn ends exactly at the watermark", "2026-08-28T01:00:22.000Z", ["question-2"]] - ] as const)("imports only complete turns %s from a changed long conversation", async (_label, latestSeenAt, expectedTitles) => { - const root = tempRoot(); - const statePath = join(root, "agent-sources.json"); - writeFileSync(statePath, JSON.stringify({ version: 2, sources: { "fixture-agent": { - status: "not_connected", messageCount: 0, lastScannedAt: latestSeenAt, latestSeenAt, - } } })); - const addMemory = vi.fn((input: { title: string; content: string }) => ({ id: input.title, duplicate: false })); - const executor = createAgentSourceExecutor({ - service: { addMemory, enqueuePendingImportSummaries: vi.fn() } as unknown as MemoryService, - configPath: join(root, "config.yaml"), statePath, - sourceRegistry: createSourceRegistry([longConversationAdapter(3)]), - }); - try { - await executor.startScan({ sourceId: "fixture-agent", mode: "incremental" }); - await vi.waitFor(() => expect(executor.scanStatus().running).toBe(false), { timeout: 5_000 }); - - expect(executor.scanStatus().error).toBeNull(); - expect(addMemory.mock.calls.map(([input]) => input.title)).toEqual(expectedTitles); - for (const [input] of addMemory.mock.calls) expect(input.content).toContain(input.title.replace("question", "answer")); - const saved = JSON.parse(readFileSync(statePath, "utf8")); - expect(saved.sources["fixture-agent"].latestSeenAt).toBe("2026-08-28T01:00:22.000Z"); - } finally { await executor.dispose(); } - }); - - it.each(["full", "initial_subset"] as const)("preserves standalone %s history selection with an existing watermark", async (mode) => { - const root = tempRoot(); - const statePath = join(root, "agent-sources.json"); - writeFileSync(statePath, JSON.stringify({ version: 2, sources: { "fixture-agent": { - status: "not_connected", messageCount: 0, lastScannedAt: "2026-09-01T00:00:00.000Z", latestSeenAt: "2026-09-01T00:00:00.000Z", - } } })); - const addMemory = vi.fn((input: { title: string }) => ({ id: input.title, duplicate: false })); - const executor = createAgentSourceExecutor({ - service: { addMemory, enqueuePendingImportSummaries: vi.fn() } as unknown as MemoryService, - configPath: join(root, "config.yaml"), statePath, - sourceRegistry: createSourceRegistry([longConversationAdapter(mode === "initial_subset" ? 1001 : 3)]), - }); - try { - await executor.startScan({ sourceId: "fixture-agent", mode }); - await vi.waitFor(() => expect(executor.scanStatus().running).toBe(false), { timeout: 5_000 }); - - expect(executor.scanStatus().error).toBeNull(); - const titles = addMemory.mock.calls.map(([input]) => input.title); - expect(titles).toHaveLength(mode === "initial_subset" ? 1000 : 3); - expect(titles).toContain(mode === "initial_subset" ? "question-1000" : "question-2"); - if (mode === "initial_subset") expect(titles).not.toContain("question-0"); - else expect(titles).toContain("question-0"); - } finally { await executor.dispose(); } - }); - it("also waits for a canceled scan when a replacement scan starts before disposal", async () => { const root = tempRoot(); let finishFirst!: () => void; @@ -636,21 +558,6 @@ function fixtureMessage( } as const; } -function longConversationAdapter(turnCount: number, toolContent?: string): SourceAdapter { - return { - descriptor: { sourceId: "fixture-agent", displayName: "Fixture Agent", builtin: true, dataPath: "/synthetic-history" }, - async detect() { return true; }, - async *scan() { - for (let index = 0; index < turnCount; index += 1) { - const userAt = Date.parse("2026-08-28T01:00:00.000Z") + index * 10_000; - yield { ...fixtureMessage("user", `user-${index}`, new Date(userAt).toISOString()), content: `question-${index}` }; - if (toolContent) yield { ...fixtureMessage("user", `tool-${index}`, new Date(userAt + 1_000).toISOString()), role: "tool", content: toolContent }; - yield { ...fixtureMessage("assistant", `assistant-${index}`, new Date(userAt + 2_000).toISOString()), content: `answer-${index}` }; - } - }, - }; -} - async function waitForFakeTimerScan(executor: ReturnType): Promise { for (let attempt = 0; attempt < 100; attempt += 1) { if (!executor.scanStatus().running) return; diff --git a/Memory/tests/project-version.test.ts b/Memory/tests/project-version.test.ts index 7e8087502..415667b7b 100644 --- a/Memory/tests/project-version.test.ts +++ b/Memory/tests/project-version.test.ts @@ -14,7 +14,7 @@ describe("Memory service version", () => { readFileSync(resolve(fileURLToPath(import.meta.url), "../../viewer/package.json"), "utf8") ); - expect(PROJECT_VERSION).toBe("2.1.2"); + expect(PROJECT_VERSION).toBe("2.1.1"); expect(PROJECT_VERSION).toBe(manifest.version); expect(cliManifest.version).toBe(manifest.version); expect(viewerManifest.version).toBe(manifest.version); diff --git a/Memory/tests/storage-db-pkg.test.ts b/Memory/tests/storage-db-pkg.test.ts deleted file mode 100644 index dc56e46be..000000000 --- a/Memory/tests/storage-db-pkg.test.ts +++ /dev/null @@ -1,238 +0,0 @@ -import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { MemoryDb } from "../src/storage/db.js"; - -const fixture = vi.hoisted(() => ({ - systemTmp: "", - root: "", - temp: "", - modulePath: "", - vectorSource: "", - interleave: undefined as undefined | { asset: string; run(): void }, - renameConflict: undefined as undefined | { publishCompleteAsset: boolean }, - bindings: [] as Array<{ path?: string; bytes?: Buffer; readonly: boolean }>, - extensions: [] as Array<{ path: string; bytes?: Buffer }>, - writes: [] as string[], -})); - -vi.mock("node:os", async (importOriginal) => { - const original = await importOriginal(); - fixture.systemTmp = original.tmpdir(); - return { ...original, tmpdir: () => fixture.temp }; -}); - -vi.mock("node:url", async (importOriginal) => ({ - ...await importOriginal(), - // The compatibility branch assumes a compiled repository-layout snapshot. - fileURLToPath: () => fixture.modulePath, -})); - -vi.mock("node:fs", async (importOriginal) => { - const original = await importOriginal(); - function interleaveWrite(target: string, bytes: Buffer): void { - fixture.writes.push(target); - const interleave = fixture.interleave; - if (!interleave || !target.includes(interleave.asset)) return; - fixture.interleave = undefined; - original.writeFileSync(target, bytes.subarray(0, Math.max(1, Math.floor(bytes.length / 2)))); - interleave.run(); - } - return { - ...original, - copyFileSync: vi.fn((source: string, target: string) => { - interleaveWrite(target, original.readFileSync(source)); - original.copyFileSync(source, target); - }), - renameSync: vi.fn((source: string, target: string) => { - const conflict = fixture.renameConflict; - if (conflict) { - fixture.renameConflict = undefined; - if (conflict.publishCompleteAsset) original.copyFileSync(source, target); - throw Object.assign(new Error("native cache publication denied"), { code: "EACCES" }); - } - original.renameSync(source, target); - }), - writeFileSync: vi.fn((target: string, contents: string | Buffer, options?: import("node:fs").WriteFileOptions) => { - if (target.startsWith(join(fixture.temp, "memmy-memory-native"))) { - const bytes = Buffer.isBuffer(contents) ? contents : Buffer.from(contents); - const interleaved = fixture.interleave && target.includes(fixture.interleave.asset); - interleaveWrite(target, bytes); - // Finish the same simulated write after a second process observes the filesystem. - if (interleaved) { - original.writeFileSync(target, contents, typeof options === "object" ? { ...options, flag: "w" } : options); - return; - } - } - original.writeFileSync(target, contents, options); - }), - }; -}); - -vi.mock("better-sqlite3", () => ({ - default: vi.fn(function (_path: string, options: { nativeBinding?: string; readonly: boolean }) { - fixture.bindings.push({ - path: options.nativeBinding, - bytes: options.nativeBinding ? readFileSync(options.nativeBinding) : undefined, - readonly: options.readonly, - }); - return { - loadExtension(path: string) { - fixture.extensions.push({ path, bytes: existsSync(path) ? readFileSync(path) : undefined }); - }, - prepare: () => ({ get: () => ({ version: "v0.1.9" }) }), - pragma: vi.fn(), - close: vi.fn(), - }; - }), -})); -vi.mock("sqlite-vec", () => ({ getLoadablePath: () => fixture.vectorSource })); -vi.mock("../src/storage/schema.js", () => ({ getSchemaVersion: vi.fn(), migrate: vi.fn(), SCHEMA_VERSION: 1 })); -vi.mock("../src/storage/sqlite-vec-store.js", () => ({ SQLITE_VEC_VERSION: "0.1.9" })); - -const processWithPkg = process as NodeJS.Process & { pkg?: unknown }; -let originalPkg: PropertyDescriptor | undefined; -let bindingSource: string; -const bindingBytes = Buffer.from("fixture sqlite binding version one"); -const vectorBytes = Buffer.from("fixture vector extension version one"); - -beforeEach(() => { - fixture.root = mkdtempSync(join(fixture.systemTmp, "memmy-storage-pkg-test-")); - fixture.temp = join(fixture.root, "tmp"); - fixture.modulePath = join(fixture.root, "snapshot", "product", "Memory", "dist", "src", "storage", "db.js"); - bindingSource = join(fixture.root, "snapshot", "product", "node_modules", "better-sqlite3", "build", "Release", "better_sqlite3.node"); - fixture.vectorSource = join(fixture.root, "snapshot", "product", "node_modules", "sqlite-vec-test", "vec0.dylib"); - for (const path of [bindingSource, fixture.vectorSource]) mkdirSync(dirname(path), { recursive: true }); - writeFileSync(bindingSource, bindingBytes); - writeFileSync(fixture.vectorSource, vectorBytes); - fixture.bindings = []; - fixture.extensions = []; - fixture.writes = []; - originalPkg = Object.getOwnPropertyDescriptor(process, "pkg"); - Object.defineProperty(process, "pkg", { configurable: true, writable: true, value: { entrypoint: "fixture" } }); -}); - -afterEach(() => { - fixture.interleave = undefined; - fixture.renameConflict = undefined; - if (originalPkg) Object.defineProperty(process, "pkg", originalPkg); - else delete processWithPkg.pkg; - rmSync(fixture.root, { recursive: true, force: true }); - vi.clearAllMocks(); -}); - -function openDatabase(): MemoryDb { - return new MemoryDb({ path: join(fixture.root, "memory.sqlite"), readonly: true }); -} - -describe("Memory SQLite assets in a pkg snapshot", () => { - it("extracts the compiled-layout binding and vector extension before passing them to SQLite", () => { - openDatabase().close(); - expect(fixture.bindings).toHaveLength(1); - expect(fixture.bindings[0]).toMatchObject({ readonly: true, bytes: bindingBytes }); - expect(fixture.extensions[0]?.bytes).toEqual(vectorBytes); - for (const path of [fixture.bindings[0]?.path, fixture.extensions[0]?.path]) { - expect(path?.startsWith(join(fixture.temp, "memmy-memory-native"))).toBe(true); - } - }); - - it("reuses a complete extraction when source bytes are unchanged", () => { - openDatabase().close(); - const firstWrites = [...fixture.writes]; - openDatabase().close(); - expect(fixture.bindings[1]?.path).toBe(fixture.bindings[0]?.path); - expect(fixture.extensions[1]?.path).toBe(fixture.extensions[0]?.path); - expect(fixture.writes).toEqual(firstWrites); - }); - - it("uses new source bytes after an upgrade without overwriting assets used by an older process", () => { - openDatabase().close(); - const oldBinding = fixture.bindings[0]!.path!; - const oldVector = fixture.extensions[0]!.path; - const upgradedBinding = Buffer.from("fixture sqlite binding version two"); - const upgradedVector = Buffer.from("fixture vector extension version two"); - writeFileSync(bindingSource, upgradedBinding); - writeFileSync(fixture.vectorSource, upgradedVector); - openDatabase().close(); - expect(fixture.bindings[1]?.bytes).toEqual(upgradedBinding); - expect(fixture.extensions[1]?.bytes).toEqual(upgradedVector); - expect(fixture.bindings[1]?.path).not.toBe(oldBinding); - expect(fixture.extensions[1]?.path).not.toBe(oldVector); - expect(readFileSync(oldBinding)).toEqual(bindingBytes); - expect(readFileSync(oldVector)).toEqual(vectorBytes); - }); - - it("ignores old shared basename caches", () => { - const oldCache = join(fixture.temp, "memmy-memory-native"); - mkdirSync(oldCache, { recursive: true }); - writeFileSync(join(oldCache, "better_sqlite3.node"), "old binding"); - writeFileSync(join(oldCache, "vec0.dylib"), "old vector"); - openDatabase().close(); - expect(fixture.bindings[0]?.bytes).toEqual(bindingBytes); - expect(fixture.extensions[0]?.bytes).toEqual(vectorBytes); - }); - - it("repairs an existing incomplete extraction before loading it", () => { - openDatabase().close(); - writeFileSync(fixture.bindings[0]!.path!, "incomplete binding"); - writeFileSync(fixture.extensions[0]!.path, "incomplete vector"); - openDatabase().close(); - expect(fixture.bindings[1]?.bytes).toEqual(bindingBytes); - expect(fixture.extensions[1]?.bytes).toEqual(vectorBytes); - }); - - it.each(["better_sqlite3.node", "vec0.dylib"])("never exposes an incomplete %s to a concurrent opener", (asset) => { - fixture.interleave = { asset, run: () => openDatabase().close() }; - openDatabase().close(); - expect(fixture.interleave).toBeUndefined(); - expect(fixture.bindings).toHaveLength(2); - expect(fixture.bindings.every((entry) => entry.bytes?.equals(bindingBytes))).toBe(true); - expect(fixture.extensions.every((entry) => entry.bytes?.equals(vectorBytes))).toBe(true); - expect(readdirSync(join(fixture.temp, "memmy-memory-native"), { recursive: true }).some((path) => String(path).endsWith(".tmp"))).toBe(false); - }); - - it("uses an identical asset published by another process if replacement is denied", () => { - fixture.renameConflict = { publishCompleteAsset: true }; - openDatabase().close(); - expect(fixture.renameConflict).toBeUndefined(); - expect(fixture.bindings[0]?.bytes).toEqual(bindingBytes); - expect(fixture.extensions[0]?.bytes).toEqual(vectorBytes); - expect(readdirSync(join(fixture.temp, "memmy-memory-native"), { recursive: true }).some((path) => String(path).endsWith(".tmp"))).toBe(false); - }); - - it("fails a denied publication without loading an incomplete asset or leaving temporary files", () => { - fixture.renameConflict = { publishCompleteAsset: false }; - expect(openDatabase).toThrow("native cache publication denied"); - expect(fixture.bindings).toEqual([]); - expect(readdirSync(join(fixture.temp, "memmy-memory-native"), { recursive: true }).some((path) => String(path).endsWith(".tmp"))).toBe(false); - }); - - it("preserves native loader fallback when snapshot assets are absent", () => { - rmSync(bindingSource); - rmSync(fixture.vectorSource); - openDatabase().close(); - expect(fixture.bindings[0]?.path).toBeUndefined(); - expect(fixture.extensions[0]?.path).toBe(fixture.vectorSource); - expect(existsSync(join(fixture.temp, "memmy-memory-native"))).toBe(false); - }); - - it("keeps ordinary Node native resolution and the original vector path", () => { - delete processWithPkg.pkg; - openDatabase().close(); - expect(fixture.bindings[0]?.path).toBeUndefined(); - expect(fixture.extensions[0]?.path).toBe(fixture.vectorSource); - expect(existsSync(join(fixture.temp, "memmy-memory-native"))).toBe(false); - }); - - it("keeps Electron's existing app.asar.unpacked vector preference", () => { - delete processWithPkg.pkg; - fixture.vectorSource = join(fixture.root, "Memmy.app", "app.asar", "node_modules", "sqlite-vec-test", "vec0.dylib"); - const unpacked = fixture.vectorSource.replace("app.asar", "app.asar.unpacked"); - mkdirSync(dirname(unpacked), { recursive: true }); - writeFileSync(unpacked, vectorBytes); - openDatabase().close(); - expect(fixture.bindings[0]?.path).toBeUndefined(); - expect(fixture.extensions[0]?.path).toBe(unpacked); - expect(existsSync(join(fixture.temp, "memmy-memory-native"))).toBe(false); - }); -}); diff --git a/Memory/viewer/package.json b/Memory/viewer/package.json index 16e95464c..a8c022140 100644 --- a/Memory/viewer/package.json +++ b/Memory/viewer/package.json @@ -1,6 +1,6 @@ { "name": "@memmy/memory-viewer", - "version": "2.1.2", + "version": "2.1.1", "private": true, "type": "module" } diff --git a/docs/github-yunxiao-sync.md b/docs/github-yunxiao-sync.md deleted file mode 100644 index 22203fe5f..000000000 --- a/docs/github-yunxiao-sync.md +++ /dev/null @@ -1,77 +0,0 @@ -# GitHub Issue 到云效同步 - -这个仓库使用 GitHub Actions 直接调用云效 OpenAPI，把 GitHub Issue 和 Pull Request 映射成云效工作项。`coding agent` 继续负责开发任务、分支和 PR，不参与这条 CRUD 同步链路。 - -同步链路如下： - -```text -GitHub issues / pull_request_target - -> .github/workflows/yunxiao-github-sync.yml - -> scripts/yunxiao_github_sync.py - -> 云效 OpenAPI - -> 目标空间（project）/目录（parentId）中的工作项 -``` - -## 仓库配置 - -在每个 GitHub 源仓库的 `Settings -> Secrets and variables -> Actions` 中配置： - -| 名称 | 类型 | 说明 | -| --- | --- | --- | -| `YUNXIAO_TOKEN` | Secret | 云效 PAT，只授予目标空间的工作项读写权限 | -| `YUNXIAO_PROJECT_ID` | Variable | 目标云效空间/项目 ID | -| `YUNXIAO_PROJECT_NAME` | Variable | 目标空间/项目名称，用于预检 | -| `YUNXIAO_PARENT_ID` | Variable | 目标目录对应的父工作项 ID；没有目录时留空 | -| `YUNXIAO_DEFAULT_ASSIGNEE_NAME` | Variable | 可选，默认负责人姓名；留空则不自动分配负责人 | -| `YUNXIAO_WORKITEM_CATEGORY` | Variable | 工作项大类，默认 `Req` | -| `YUNXIAO_WORKITEM_TYPE_NAME` | Variable | 工作项类型名称，默认 `需求` | -| `YUNXIAO_PRIORITY_NAME` | Variable | 默认优先级，默认 `中` | -| `YUNXIAO_DAYS_TO_FINISH` | Variable | 计划完成时间距创建时间的天数，默认 `7` | -| `YUNXIAO_API_BASE_URL` | Variable | 可选，默认 `https://openapi-rdc.aliyuncs.com` | - -`memmy-agent` 默认使用云效项目 `memmy`，项目 ID 为 -`1832b179386e24414d3891e244`。该默认值已经写入 workflow；如果仓库 Variables -中配置了同名变量，则以 Variables 为准。 - -`YUNXIAO_DEFAULT_ASSIGNEE_NAME` 可以暂时留空。这样工作项仍会进入 -`memmy` 需求空间，由云效用户后续人工分配负责人。 - -预检会按语义解析云效工作流状态：合并的 PR 优先使用“已完成”，如果目标项目 -使用“开发完成”则自动使用该名称；取消状态同理支持“已取消”“已关闭”“关闭”。 - -每个源仓库可以使用不同的 `YUNXIAO_PROJECT_ID` 和 `YUNXIAO_PARENT_ID`。例如： - -| GitHub 仓库 | 云效空间 | 云效目录 | -| --- | --- | --- | -| `MemTensor/memmy-agent` | `memmy` (`1832b179386e24414d3891e244`) | 留空，直接进入需求空间 | -| `MemTensor/MemOS` | MemOS 开源项目管理 | MemOS 目录 | -| `MemTensor/MemOS-Cloud-CLI` | CLI 项目 | CLI 目录 | - -你提供的页面 URL 中 `viewIdentifier=d7f112f9d023e2108fa1b0d8` 是云效“需求”列表视图标识， -不是创建工作项 API 的 `parentId`。当前需求是写入 `memmy` 空间，因此 -`YUNXIAO_PARENT_ID` 应保持为空。只有确认了一个真实的父工作项/目录 ID 后，才配置 -`YUNXIAO_PARENT_ID`。 - -如果云效页面中的“目录”只是列表视图，而不是父工作项，应改为配置不同的 -`YUNXIAO_PROJECT_ID` 或保持 `YUNXIAO_PARENT_ID` 为空。 - -## 运行方式 - -第一次接入按以下顺序运行 `Sync GitHub to Yunxiao`： - -1. `mode=preflight`：只检查 Token、空间、工作项类型、状态、优先级和负责人。 -2. `mode=backfill`、`state=open`、`apply=false`：查看回填 dry-run 结果。 -3. 确认结果后再次运行 `mode=backfill`、`state=open`、`apply=true`。 -4. 需要把历史关闭项也纳入同步时，将 `state` 设为 `all`。 - -实时事件会处理 `opened`、`closed` 和 `reopened`。已存在的云效工作项只更新状态，不重复创建。 - -幂等键包含完整源仓库名，例如： - -```text -[GitHub MemTensor/memmy-agent Issue #123] -``` - -因此不同仓库即使使用相同的 Issue 编号，也不会互相覆盖。Issue 标题、链接、正文和 GitHub labels 会写入新建工作项；评论暂不同步。 - -其他 GitHub 仓库接入时复制这个 workflow 和脚本，再为该仓库设置自己的 Variables/Secret 即可。仓库数量继续增长后，可以把脚本移动到同组织的私有 reusable workflow 仓库，源仓库只保留一个调用文件，映射仍由各源仓库自己的 Variables 管理。 diff --git a/package-lock.json b/package-lock.json index e40d978b5..e1d7486a5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "memmy-agent", - "version": "1.1.4", + "version": "1.1.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "memmy-agent", - "version": "1.1.4", + "version": "1.1.3", "workspaces": [ "Migrations", "AgentSourceCore", @@ -582,7 +582,7 @@ }, "App/shell/desktop": { "name": "@memmy/desktop", - "version": "1.1.4", + "version": "1.1.3", "dependencies": { "@memmy/backend": "0.0.0", "@memmy/desktop-interface": "0.0.0", @@ -719,7 +719,7 @@ }, "Memory": { "name": "@memmy/memory", - "version": "2.1.2", + "version": "2.1.1", "dependencies": { "@huggingface/transformers": "^3.8.0", "@memmy/agent-source-core": "0.0.0", diff --git a/package.json b/package.json index d49422685..9291482b8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "memmy-agent", - "version": "1.1.4", + "version": "1.1.3", "private": true, "type": "module", "description": "Local-first agent memory substrate with desktop and CLI surfaces.", diff --git a/scripts/sanitize-release-notes.mjs b/scripts/sanitize-release-notes.mjs deleted file mode 100644 index ffb229b5f..000000000 --- a/scripts/sanitize-release-notes.mjs +++ /dev/null @@ -1,303 +0,0 @@ -#!/usr/bin/env node - -import { readFileSync, writeFileSync } from "node:fs"; -import { resolve } from "node:path"; - -const RESERVED_MARKERS = [ - /^|$)/, - /^|$)/, -]; -const CJK_RE = /[\u3040-\u30ff\u3400-\u9fff\uf900-\ufaff]/; - -function isReservedMarker(value) { - return RESERVED_MARKERS.some((pattern) => pattern.test(value)); -} - -function reservedMarkerOffset(line) { - let offset = line.indexOf("")) htmlComment = false; - continue; - } - if (line.includes("", line.indexOf("")) htmlComment = false; - continue; - } - const commentOffset = line.indexOf("", commentOffset + 4)) htmlComment = true; - const visiblePrefix = visibleLineForLanguageValidation(line.slice(0, commentOffset)); - if (CJK_RE.test(visiblePrefix)) { - throw new Error( - `English public release notes contain visible CJK text (line ${index + 1})`, - ); - } - continue; - } - - const visible = visibleLineForLanguageValidation(line); - if (CJK_RE.test(visible)) { - throw new Error( - `English public release notes contain visible CJK text (line ${index + 1})`, - ); - } - if ( - /[A-Za-z]/.test(visible) && - !/^[ \t]{0,3}#{1,6}[ \t]/.test(visible) && - !/^[ \t]*(?:[-*_][ \t]*){3,}$/.test(visible) - ) { - hasBodyContent = true; - } - } - - if (!hasBodyContent) { - throw new Error("English public release notes contain no English body content"); - } -} - -function normalizePublicLanguage(markdown, publicLanguage) { - if (!publicLanguage) return markdown; - if (publicLanguage !== "en") { - throw new Error(`Unsupported public release language: ${publicLanguage}`); - } - const normalized = removeCjkHeadingSections(markdown).trimEnd(); - assertEnglishPublicBody(normalized); - return normalized; -} - -export function sanitizeReleaseNotes(markdown, { publicLanguage = "" } = {}) { - const lines = markdown.split(/\r?\n/); - const publicLines = []; - let fence = null; - let removingReservedComment = false; - let reservedCommentStart = 0; - - for (let index = 0; index < lines.length; index += 1) { - const line = lines[index]; - const lineNumber = index + 1; - - if (removingReservedComment) { - const closeOffset = line.indexOf("-->"); - if (closeOffset === -1) continue; - - if (line.slice(closeOffset + 3).trim() !== "") { - throw new Error( - `Reserved release metadata must occupy complete lines (line ${lineNumber})`, - ); - } - - removingReservedComment = false; - continue; - } - - if (fence) { - publicLines.push(line); - if (isFenceClosing(line, fence)) fence = null; - continue; - } - - const openingFence = fenceOpening(line); - if (openingFence) { - fence = openingFence; - publicLines.push(line); - continue; - } - - const markerOffset = reservedMarkerOffset(line); - if (markerOffset === -1) { - publicLines.push(line); - continue; - } - - assertCommentOccupiesCompleteLines(line, markerOffset, lineNumber); - const closeOffset = line.indexOf("-->", markerOffset + 4); - if (closeOffset === -1) { - removingReservedComment = true; - reservedCommentStart = lineNumber; - continue; - } - - if (line.slice(closeOffset + 3).trim() !== "") { - throw new Error( - `Reserved release metadata must occupy complete lines (line ${lineNumber})`, - ); - } - } - - if (removingReservedComment) { - throw new Error( - `Unterminated reserved release metadata starting at line ${reservedCommentStart}`, - ); - } - - const publicMarkdown = normalizePublicLanguage( - publicLines.join("\n").trimEnd(), - publicLanguage, - ); - if (publicMarkdown.trim() === "") { - throw new Error("Release notes contain no public content after sanitization"); - } - - const sanitized = `${publicMarkdown}\n`; - assertNoReservedMetadata(sanitized); - return sanitized; -} - -function assertNoReservedMetadata(markdown) { - const lines = markdown.split(/\r?\n/); - let fence = null; - - for (let index = 0; index < lines.length; index += 1) { - const line = lines[index]; - if (fence) { - if (isFenceClosing(line, fence)) fence = null; - continue; - } - - const openingFence = fenceOpening(line); - if (openingFence) { - fence = openingFence; - continue; - } - - if (reservedMarkerOffset(line) !== -1) { - throw new Error( - `Reserved release metadata remains after sanitization (line ${index + 1})`, - ); - } - } -} - -function main() { - const [inputArg, outputArg, languageFlag, languageArg, ...extraArgs] = process.argv.slice(2); - const hasLanguage = languageFlag !== undefined || languageArg !== undefined; - if ( - !inputArg || - !outputArg || - extraArgs.length > 0 || - (hasLanguage && (languageFlag !== "--language" || !languageArg)) - ) { - throw new Error( - "Usage: node scripts/sanitize-release-notes.mjs [--language en]", - ); - } - - const inputPath = resolve(inputArg); - const outputPath = resolve(outputArg); - if (inputPath === outputPath) { - throw new Error("Input and output paths must be different"); - } - - const markdown = readFileSync(inputPath, "utf8"); - const sanitized = sanitizeReleaseNotes(markdown, { - publicLanguage: languageArg || "", - }); - writeFileSync(outputPath, sanitized, "utf8"); -} - -try { - main(); -} catch (error) { - console.error(`sanitize-release-notes: ${error instanceof Error ? error.message : error}`); - process.exitCode = 1; -} diff --git a/scripts/test_yunxiao_github_sync.py b/scripts/test_yunxiao_github_sync.py deleted file mode 100644 index aa31e8a99..000000000 --- a/scripts/test_yunxiao_github_sync.py +++ /dev/null @@ -1,182 +0,0 @@ -from __future__ import annotations - -import importlib.util -import unittest -from pathlib import Path - - -MODULE_PATH = Path(__file__).with_name("yunxiao_github_sync.py") -SPEC = importlib.util.spec_from_file_location("yunxiao_github_sync", MODULE_PATH) -assert SPEC is not None and SPEC.loader is not None -MODULE = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(MODULE) - - -class YunxiaoSyncTests(unittest.TestCase): - def test_source_key_includes_repository(self) -> None: - item = {"number": 42, "title": "Fix scheduler"} - self.assertEqual( - MODULE.build_source_key("issue", item, "MemTensor/memmy-agent"), - "[GitHub MemTensor/memmy-agent Issue #42]", - ) - - def test_title_includes_source_key(self) -> None: - item = {"number": 42, "title": "Fix scheduler"} - self.assertEqual( - MODULE.build_title("issue", item, "MemTensor/memmy-agent"), - "[GitHub MemTensor/memmy-agent Issue #42] Fix scheduler", - ) - - def test_source_status(self) -> None: - self.assertEqual(MODULE.source_status("issue", {"state": "open"}), "待处理") - self.assertEqual( - MODULE.source_status("pr", {"state": "closed", "merged": True}), - "已完成", - ) - self.assertEqual( - MODULE.source_status("issue", {"state": "closed"}), - "已取消", - ) - - def test_resolve_statuses_accepts_memmy_workflow_names(self) -> None: - workflow = [ - {"id": "pending", "name": "待处理"}, - {"id": "developing", "name": "开发中"}, - {"id": "developed", "name": "开发完成"}, - {"id": "cancelled", "name": "已取消"}, - ] - self.assertEqual( - MODULE.resolve_statuses(workflow), - { - "待处理": "pending", - "已完成": "developed", - "已取消": "cancelled", - }, - ) - - def test_create_payload_contains_parent(self) -> None: - calls = [] - - def transport(method, path, body=None): - calls.append((method, path, body)) - if "search" in path: - return {"data": {"workitems": []}} - return {"id": "new-id"} - - cfg = { - "project_id": "project", - "workitem_category": "Req", - "type_id": "type", - "assignee_id": "assignee", - "priority_id": "priority", - "parent_id": "directory", - "statuses": {"待处理": "pending", "已取消": "cancelled"}, - } - item = { - "number": 1, - "title": "Test", - "html_url": "https://github.com/MemTensor/memmy-agent/issues/1", - "body": "Details", - "created_at": "2026-09-10T00:00:00Z", - "state": "open", - "labels": [], - } - result = MODULE.sync_one( - "org", - cfg, - "issue", - item, - repository="MemTensor/memmy-agent", - apply=True, - label_ids={}, - days_to_finish=7, - create_closed=False, - client=MODULE.YunxiaoClient(transport), - ) - self.assertEqual(result, "created") - self.assertEqual(calls[2][2]["parentId"], "directory") - self.assertIn("Details", calls[2][2]["description"]) - - def test_reopened_item_updates_to_pending(self) -> None: - calls = [] - - def transport(method, path, body=None): - calls.append((method, path, body)) - if "search" in path: - return {"data": {"workitems": [{"id": "existing"}]}} - return {} - - cfg = { - "project_id": "project", - "workitem_category": "Req", - "type_id": "type", - "assignee_id": "assignee", - "priority_id": "priority", - "parent_id": "", - "statuses": {"待处理": "pending", "已取消": "cancelled"}, - } - result = MODULE.sync_one( - "org", - cfg, - "issue", - { - "number": 2, - "title": "Reopen", - "html_url": "https://github.com/MemTensor/memmy-agent/issues/2", - "created_at": "2026-09-10T00:00:00Z", - "state": "open", - "labels": [], - }, - repository="MemTensor/memmy-agent", - apply=True, - label_ids={}, - days_to_finish=7, - create_closed=False, - client=MODULE.YunxiaoClient(transport), - ) - self.assertEqual(result, "updated-status") - self.assertEqual(calls[-1][2], {"status": "pending"}) - - def test_all_state_backfill_can_create_closed_item(self) -> None: - calls = [] - - def transport(method, path, body=None): - calls.append((method, path, body)) - if "search" in path: - return {"data": {"workitems": []}} - return {"id": "new-id"} - - cfg = { - "project_id": "project", - "workitem_category": "Req", - "type_id": "type", - "assignee_id": "assignee", - "priority_id": "priority", - "parent_id": "", - "statuses": {"待处理": "pending", "已取消": "cancelled"}, - } - result = MODULE.sync_one( - "org", - cfg, - "issue", - { - "number": 3, - "title": "Closed before rollout", - "html_url": "https://github.com/MemTensor/memmy-agent/issues/3", - "created_at": "2026-09-10T00:00:00Z", - "state": "closed", - "labels": [], - }, - repository="MemTensor/memmy-agent", - apply=True, - label_ids={}, - days_to_finish=7, - create_closed=True, - client=MODULE.YunxiaoClient(transport), - ) - self.assertEqual(result, "created") - self.assertEqual(calls[-1][2]["status"], "cancelled") - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/yunxiao_github_sync.py b/scripts/yunxiao_github_sync.py deleted file mode 100644 index a5a3e8a23..000000000 --- a/scripts/yunxiao_github_sync.py +++ /dev/null @@ -1,735 +0,0 @@ -from __future__ import annotations - -import argparse -import json -import os -import sys -import time - -from datetime import datetime, timedelta, timezone -from typing import Any -from urllib.error import HTTPError, URLError -from urllib.parse import quote, urlencode -from urllib.request import Request, urlopen - - -SOURCE_LABELS = {"issue": "Issue", "pr": "PR"} -# Yunxiao projects can customize workflow names. Keep the canonical names used -# by the sync state machine, then resolve them to the first matching name in -# the target project's workflow. -STATUS_ALIASES = { - "待处理": ("待处理", "未开始"), - "已完成": ("已完成", "开发完成", "完成"), - "已取消": ("已取消", "已关闭", "关闭"), -} -DEFAULT_PRIORITY_NAME = "中" -DEFAULT_WORKITEM_CATEGORY = "Req" -DEFAULT_WORKITEM_TYPE_NAME = "需求" -DEFAULT_DAYS_TO_FINISH = 7 -REQUIRED_ENV = ( - "YUNXIAO_PROJECT_ID", - "YUNXIAO_PROJECT_NAME", -) - - -class PreflightError(ValueError): - pass - - -class YunxiaoApiError(RuntimeError): - pass - - -def required_env(name: str) -> str: - value = os.environ.get(name, "").strip() - if not value: - raise PreflightError(f"缺少必需的环境变量：{name}（请在仓库 Variables 中配置）") - return value - - -class UrllibTransport: - def __init__( - self, - token: str, - *, - base_url: str = "https://openapi-rdc.aliyuncs.com", - timeout_seconds: int = 30, - ) -> None: - self._token = token - self._base_url = base_url.rstrip("/") - self._timeout_seconds = timeout_seconds - - def __call__(self, method: str, path: str, body: dict[str, Any] | None = None) -> Any: - data = json.dumps(body, ensure_ascii=False).encode() if body is not None else None - request = Request( - f"{self._base_url}{path}", - data=data, - method=method, - headers={ - "Accept": "application/json", - "Content-Type": "application/json", - "x-yunxiao-token": self._token, - }, - ) - try: - with urlopen(request, timeout=self._timeout_seconds) as response: - payload = response.read().decode() - if not payload.strip(): - return None - return json.loads(payload) - except HTTPError as error: - raise YunxiaoApiError( - f"云效 API HTTP {error.code}: {error.read().decode(errors='replace')[:300]}" - ) from error - except URLError as error: - raise YunxiaoApiError(f"无法连接云效 API：{error.reason}") from error - - -def repository_name(repository: str | None = None) -> str: - value = (repository or os.environ.get("GITHUB_REPOSITORY", "")).strip() - if not value or "/" not in value: - raise PreflightError("缺少有效的 GitHub 仓库标识，要求格式为 owner/repository") - return value - - -def build_source_key( - item_type: str, - item: dict[str, Any], - repository: str | None = None, -) -> str: - return f"[GitHub {repository_name(repository)} {SOURCE_LABELS[item_type]} #{item['number']}]" - - -def build_title( - item_type: str, - item: dict[str, Any], - repository: str | None = None, -) -> str: - return f"{build_source_key(item_type, item, repository)} {item['title']}" - - -def iso_after_days(value: str, days: int) -> str: - created_at = datetime.fromisoformat(value.replace("Z", "+00:00")) - return ( - (created_at + timedelta(days=days)) - .astimezone(timezone.utc) - .isoformat() - .replace("+00:00", "Z") - ) - - -def source_status(item_type: str, item: dict[str, Any]) -> str: - if item.get("state") == "open": - return "待处理" - if item_type == "pr" and (item.get("merged") or item.get("merged_at")): - return "已完成" - return "已取消" - - -def item_id(item: dict[str, Any]) -> str: - for key in ("id", "identifier", "organizationId", "userId", "statusId", "value"): - value = item.get(key) - if value: - return str(value) - raise PreflightError(f"云效对象缺少 id：{item}") - - -def item_name(item: dict[str, Any]) -> str | None: - for key in ("name", "displayName", "userName", "displayValue", "statusName"): - value = item.get(key) - if isinstance(value, str): - return value - return None - - -def list_or_extract(payload: Any, keys: tuple[str, ...]) -> list[dict[str, Any]]: - if isinstance(payload, dict): - for key in (*keys, "items"): - value = payload.get(key) - if isinstance(value, list) and all(isinstance(x, dict) for x in value): - return value - if isinstance(payload, list) and all(isinstance(x, dict) for x in payload): - return payload - return [] - - -def find_by_name(items: list[dict[str, Any]], name: str, label: str) -> dict[str, Any]: - for item in items: - if item_name(item) == name: - return item - raise PreflightError(f"未在云效中找到{label}：{name}") - - -def resolve_statuses(workflow: list[dict[str, Any]]) -> dict[str, str]: - names = { - name: item - for item in workflow - if (name := item_name(item)) - } - statuses: dict[str, str] = {} - for canonical, aliases in STATUS_ALIASES.items(): - matched_name = next((name for name in aliases if name in names), None) - if matched_name is None: - supported = "、".join(sorted(names)) or "（空）" - raise PreflightError( - f"未在云效中找到工作流状态：{canonical}；" - f"支持的别名：{'、'.join(aliases)}；当前状态：{supported}" - ) - statuses[canonical] = item_id(names[matched_name]) - return statuses - - -class YunxiaoClient: - def __init__(self, transport: Any) -> None: - self.transport = transport - - def get(self, path: str) -> Any: - return self.transport("GET", path) - - -def preflight( - project_id: str, - project_name: str, - assignee_name: str | None, - client: YunxiaoClient, - *, - workitem_category: str = DEFAULT_WORKITEM_CATEGORY, - workitem_type_name: str = DEFAULT_WORKITEM_TYPE_NAME, - priority_name: str = DEFAULT_PRIORITY_NAME, - parent_id: str | None = None, -) -> dict[str, Any]: - organizations = list_or_extract(client.get("/oapi/v1/platform/organizations"), ()) - if not organizations: - raise PreflightError("PAT 无法访问任何云效组织") - - for organization in organizations: - organization_id = item_id(organization) - project = client.get( - f"/oapi/v1/projex/organizations/{organization_id}/projects/{project_id}" - ) - if isinstance(project, dict) and project.get("name") == project_name: - org = organization_id - break - else: - raise PreflightError(f"未找到项目：{project_name}（{project_id}）") - - encoded_category = quote(workitem_category, safe="") - types = list_or_extract( - client.get( - f"/oapi/v1/projex/organizations/{org}/projects/{project_id}" - f"/workitemTypes?category={encoded_category}" - ), - ("workitemTypes",), - ) - workitem_type = next( - (value for value in types if item_name(value) == workitem_type_name), - None, - ) - if workitem_type is None: - workitem_type = next( - ( - value - for value in types - if value.get("categoryId", value.get("category")) == workitem_category - ), - None, - ) - if not workitem_type: - raise PreflightError( - f"目标项目没有工作项类型：{workitem_type_name}（category={workitem_category}）" - ) - type_id = item_id(workitem_type) - - assignee_id = "" - if assignee_name: - members = list_or_extract( - client.get(f"/oapi/v1/projex/organizations/{org}/projects/{project_id}/members"), - ("members",), - ) - assignee_id = item_id(find_by_name(members, assignee_name, "默认负责人")) - - workflow = list_or_extract( - client.get( - f"/oapi/v1/projex/organizations/{org}/projects/{project_id}" - f"/workitemTypes/{type_id}/workflows" - ), - ("statuses", "workflowStatuses"), - ) - statuses = resolve_statuses(workflow) - - fields = list_or_extract( - client.get( - f"/oapi/v1/projex/organizations/{org}/projects/{project_id}" - f"/workitemTypes/{type_id}/fields" - ), - ("fields", "fieldConfigs"), - ) - priority_field = next( - ( - field - for field in fields - if field.get("id") == "priority" - or field.get("identifier") == "priority" - or field.get("fieldIdentifier") == "priority" - ), - None, - ) - if not priority_field: - raise PreflightError("工作项类型缺少优先级字段") - options = list_or_extract( - priority_field.get("options") or priority_field.get("values") or {}, - (), - ) - if not options: - options = priority_field.get("options") or priority_field.get("values") or [] - priority = find_by_name(options, priority_name, "默认优先级") - - return { - "org": org, - "project_id": project_id, - "workitem_category": workitem_category, - "type_id": type_id, - "type_name": item_name(workitem_type) or workitem_type_name, - "assignee_id": assignee_id, - "priority_id": item_id(priority), - "statuses": statuses, - "parent_id": parent_id or "", - } - - -def sync_labels( - org: str, - project_id: str, - wanted: set[str], - client: YunxiaoClient, -) -> dict[str, str]: - existing_raw = client.get( - f"/oapi/v1/projex/organizations/{org}/projects/{project_id}/labels" - ) - existing_names: set[str] = set() - name_to_id: dict[str, str] = {} - for label in list_or_extract(existing_raw, ()): - name = item_name(label) or label.get("name", "") - if name: - existing_names.add(name) - name_to_id[name] = item_id(label) - - colors = [ - "#e91e63", - "#9c27b0", - "#673ab7", - "#3f51b5", - "#2196f3", - "#00bcd4", - "#009688", - "#4caf50", - "#8bc34a", - "#cddc39", - "#ffc107", - "#ff9800", - "#ff5722", - ] - for index, name in enumerate(sorted(wanted - existing_names)): - try: - response = client.transport( - "POST", - f"/oapi/v1/projex/organizations/{org}/projects/{project_id}/labels", - {"name": name, "color": colors[index % len(colors)]}, - ) - name_to_id[name] = item_id(response) - time.sleep(0.1) - except YunxiaoApiError: - # A concurrent workflow may have created the same label. - pass - return name_to_id - - -def find_workitem( - org: str, - project_id: str, - prefix: str, - client: YunxiaoClient, - *, - category: str = DEFAULT_WORKITEM_CATEGORY, -) -> str | None: - conditions = json.dumps( - { - "conditionGroups": [ - [ - { - "fieldIdentifier": "subject", - "operator": "CONTAINS", - "value": [prefix], - "className": "string", - "format": "input", - } - ] - ] - }, - ensure_ascii=False, - ) - response = client.transport( - "POST", - f"/oapi/v1/projex/organizations/{org}/workitems:search", - {"spaceId": project_id, "category": category, "conditions": conditions}, - ) - data = ( - response - if isinstance(response, list) - else (response or {}).get("data", response) - if isinstance(response, dict) - else response - ) - items = ( - (data.get("workitems", data.get("items", [])) if isinstance(data, dict) else data) - if data - else [] - ) - return item_id(items[0]) if items else None - - -def legacy_source_key(item_type: str, item: dict[str, Any]) -> str: - return f"[GitHub {SOURCE_LABELS[item_type]} #{item['number']}]" - - -def find_existing_workitem( - org: str, - project_id: str, - item_type: str, - item: dict[str, Any], - repository: str, - client: YunxiaoClient, - *, - category: str, -) -> str | None: - current_key = build_source_key(item_type, item, repository) - existing_id = find_workitem( - org, - project_id, - current_key, - client, - category=category, - ) - if existing_id: - return existing_id - # MemOS PR #2159 used a repository-less key. Keep this fallback during - # migration so a new workflow cannot duplicate those existing work items. - return find_workitem( - org, - project_id, - legacy_source_key(item_type, item), - client, - category=category, - ) - - -def item_description(item: dict[str, Any]) -> str: - url = str(item.get("html_url", "")).strip() - body = str(item.get("body") or "").strip() - if body: - return f"GitHub: {url}\n\n{body}" - return f"GitHub: {url}" - - -def sync_one( - org: str, - cfg: dict[str, Any], - item_type: str, - item: dict[str, Any], - *, - repository: str, - apply: bool, - label_ids: dict[str, str] | None, - days_to_finish: int, - create_closed: bool, - client: YunxiaoClient, -) -> str: - existing_id = find_existing_workitem( - org, - cfg["project_id"], - item_type, - item, - repository, - client, - category=cfg["workitem_category"], - ) - - if existing_id: - if not apply: - return "dry-run-status" - status_name = source_status(item_type, item) - client.transport( - "PUT", - f"/oapi/v1/projex/organizations/{org}/workitems/{existing_id}", - {"status": cfg["statuses"][status_name]}, - ) - return "updated-status" - - if item.get("state") != "open" and not create_closed: - return "skipped-closed" - if not apply: - return "dry-run-create" - - payload: dict[str, Any] = { - "spaceId": cfg["project_id"], - "workitemTypeId": cfg["type_id"], - "subject": build_title(item_type, item, repository), - "description": item_description(item), - "status": cfg["statuses"][source_status(item_type, item)], - "assignedTo": cfg["assignee_id"], - "priority": cfg["priority_id"], - "planStartTime": item["created_at"], - "planFinishTime": iso_after_days(item["created_at"], days_to_finish), - } - if not cfg.get("assignee_id"): - payload.pop("assignedTo") - if cfg.get("parent_id"): - payload["parentId"] = cfg["parent_id"] - - github_labels = [ - label["name"] - for label in item.get("labels", []) - if isinstance(label, dict) and label.get("name") - ] - labels = [label_ids[name] for name in github_labels if label_ids and name in label_ids] - if labels: - payload["labels"] = labels - - client.transport( - "POST", - f"/oapi/v1/projex/organizations/{org}/workitems", - payload, - ) - return "created" - - -def configuration_from_environment(client: YunxiaoClient) -> dict[str, Any]: - return preflight( - required_env("YUNXIAO_PROJECT_ID"), - required_env("YUNXIAO_PROJECT_NAME"), - os.environ.get("YUNXIAO_DEFAULT_ASSIGNEE_NAME", "").strip() or None, - client, - workitem_category=os.environ.get( - "YUNXIAO_WORKITEM_CATEGORY", - DEFAULT_WORKITEM_CATEGORY, - ).strip() - or DEFAULT_WORKITEM_CATEGORY, - workitem_type_name=os.environ.get( - "YUNXIAO_WORKITEM_TYPE_NAME", - DEFAULT_WORKITEM_TYPE_NAME, - ).strip() - or DEFAULT_WORKITEM_TYPE_NAME, - priority_name=os.environ.get( - "YUNXIAO_PRIORITY_NAME", - DEFAULT_PRIORITY_NAME, - ).strip() - or DEFAULT_PRIORITY_NAME, - parent_id=os.environ.get("YUNXIAO_PARENT_ID", "").strip() or None, - ) - - -def days_to_finish() -> int: - raw = os.environ.get("YUNXIAO_DAYS_TO_FINISH", "").strip() - if not raw: - return DEFAULT_DAYS_TO_FINISH - try: - value = int(raw) - except ValueError as error: - raise PreflightError("YUNXIAO_DAYS_TO_FINISH 必须是正整数") from error - if value < 1: - raise PreflightError("YUNXIAO_DAYS_TO_FINISH 必须是正整数") - return value - - -def handle_event(client: YunxiaoClient) -> int: - event_path = os.environ.get("GITHUB_EVENT_PATH") - if not event_path: - print("缺少 GITHUB_EVENT_PATH", file=sys.stderr) - return 2 - with open(event_path, encoding="utf-8") as event_file: - event = json.load(event_file) - - action = event.get("action", "") - if action not in ("opened", "closed", "reopened"): - print(f"忽略事件 action={action}", file=sys.stderr) - return 0 - - item = event.get("issue") or event.get("pull_request") - if not item: - print("事件中缺少 issue/pull_request", file=sys.stderr) - return 1 - - if event.get("issue") and "pull_request" in event["issue"]: - print("issue 事件包含 PR 元数据，跳过", file=sys.stderr) - return 0 - - item_type = "pr" if "pull_request" in event else "issue" - if item_type == "pr" and action == "closed": - item["merged"] = item.get("merged") or event.get("pull_request", {}).get("merged", False) - - repository = repository_name( - event.get("repository", {}).get("full_name") or os.environ.get("GITHUB_REPOSITORY") - ) - cfg = configuration_from_environment(client) - all_labels = { - label["name"] - for label in item.get("labels", []) - if isinstance(label, dict) and label.get("name") - } - label_ids = sync_labels(cfg["org"], cfg["project_id"], all_labels, client) if all_labels else {} - result = sync_one( - cfg["org"], - cfg, - item_type, - item, - repository=repository, - apply=True, - label_ids=label_ids, - days_to_finish=days_to_finish(), - create_closed=False, - client=client, - ) - print(f"{repository} {item_type} #{item['number']}: {result}") - return 0 - - -def github_collection( - repository: str, - path: str, - *, - state: str, - token: str, -) -> list[dict[str, Any]]: - result: list[dict[str, Any]] = [] - page = 1 - headers = { - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {token}", - } - while True: - query = urlencode({"state": state, "per_page": 100, "page": page}) - request = Request( - f"https://api.github.com/repos/{repository}/{path}?{query}", - headers=headers, - ) - with urlopen(request, timeout=30) as response: - batch = json.loads(response.read().decode()) - if not isinstance(batch, list) or not batch: - return result - result.extend(item for item in batch if isinstance(item, dict)) - if len(batch) < 100: - return result - page += 1 - - -def backfill(client: YunxiaoClient, *, apply: bool, state: str) -> int: - github_token = os.environ.get("GH_TOKEN", "").strip() - repository = repository_name(os.environ.get("GITHUB_REPOSITORY")) - issues_raw = github_collection(repository, "issues", state=state, token=github_token) - prs = github_collection(repository, "pulls", state=state, token=github_token) - issues = [item for item in issues_raw if "pull_request" not in item] - print(f"GitHub: {len(issues)} {state} issues, {len(prs)} {state} PRs", file=sys.stderr) - - cfg = configuration_from_environment(client) - all_labels: set[str] = set() - for item in issues + prs: - all_labels.update( - label["name"] - for label in item.get("labels", []) - if isinstance(label, dict) and label.get("name") - ) - label_ids = sync_labels(cfg["org"], cfg["project_id"], all_labels, client) if all_labels else {} - - summary: dict[str, Any] = {} - for kind, items, bucket in (("issue", issues, "issues"), ("pr", prs, "prs")): - stats = {"source": len(items), "created": 0, "skipped": 0, "updated": 0, "failed": 0} - for item in items: - try: - result = sync_one( - cfg["org"], - cfg, - kind, - item, - repository=repository, - apply=apply, - label_ids=label_ids, - days_to_finish=days_to_finish(), - create_closed=state == "all", - client=client, - ) - if result.startswith("dry-run"): - stats["skipped"] += 1 - elif result == "created": - stats["created"] += 1 - elif result == "updated-status": - stats["updated"] += 1 - else: - stats["skipped"] += 1 - except (YunxiaoApiError, PreflightError, OSError) as error: - stats["failed"] += 1 - print(f"failed {kind} #{item.get('number')}: {error}", file=sys.stderr) - time.sleep(0.12) - summary[bucket] = stats - - print(json.dumps(summary, ensure_ascii=False)) - return 0 if all(stats["failed"] == 0 for stats in summary.values()) else 1 - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="GitHub → 云效同步") - parser.add_argument("--mode", choices=("preflight", "event", "backfill"), default="preflight") - parser.add_argument("--state", choices=("open", "all"), default="open") - parser.add_argument("--apply", action="store_true") - parser.add_argument("--dry-run", action="store_true") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - token = os.environ.get("YUNXIAO_TOKEN", "").strip() - if not token: - print("缺少 YUNXIAO_TOKEN", file=sys.stderr) - return 2 - - client = YunxiaoClient( - UrllibTransport( - token, - base_url=( - os.environ.get("YUNXIAO_API_BASE_URL", "").strip() - or "https://openapi-rdc.aliyuncs.com" - ), - ) - ) - - try: - if args.mode == "event": - return handle_event(client) - if args.mode == "backfill": - return backfill(client, apply=args.apply and not args.dry_run, state=args.state) - - result = configuration_from_environment(client) - print( - json.dumps( - { - "org": result["org"], - "project_id": result["project_id"], - "parent_id": result["parent_id"], - "workitem_category": result["workitem_category"], - "type_id": result["type_id"], - "type_name": result["type_name"], - "assignee_id": result["assignee_id"], - "priority_id": result["priority_id"], - "statuses": result["statuses"], - }, - ensure_ascii=False, - sort_keys=True, - ) - ) - return 0 - except (PreflightError, YunxiaoApiError, OSError) as error: - print(str(error), file=sys.stderr) - return 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/packaged-runtime-config.test.mjs b/tests/packaged-runtime-config.test.mjs index bc6ad879c..b972d926f 100644 --- a/tests/packaged-runtime-config.test.mjs +++ b/tests/packaged-runtime-config.test.mjs @@ -1,17 +1,14 @@ import { existsSync, - lstatSync, mkdtempSync, mkdirSync, readFileSync, - realpathSync, - renameSync, rmSync, symlinkSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { delimiter, dirname, join, resolve } from "node:path"; +import { dirname, join } from "node:path"; import { spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; import { createPackage } from "@electron/asar"; @@ -23,86 +20,12 @@ import { import { pruneRuntimeEnvFiles } from "../scripts/internal/shared/prune-runtime-env-files-lib.mjs"; const roots = []; -const macBuildScriptSource = readFileSync(new URL("../scripts/internal/mac/build-dmg.sh", import.meta.url), "utf8"); afterEach(() => { while (roots.length) rmSync(roots.pop(), { recursive: true, force: true }); }); describe("packaged desktop runtime configuration", () => { - it("generates a macOS Memory manifest with local workspace dependencies and retained external locks", () => { - const fixture = macMemoryManifestFixture({ externalDependency: true }); - const result = generateMacMemoryManifest(fixture); - expect(result.status, result.stderr).toBe(0); - const manifest = JSON.parse(readFileSync(join(fixture.runtime, "package.json"), "utf8")); - const lock = JSON.parse(readFileSync(join(fixture.runtime, "package-lock.json"), "utf8")); - expect(manifest.dependencies).toEqual({ - "@memmy/agent-source-core": "file:../../../../../../AgentSourceCore", - "fixture-public": "1.0.0", - }); - expect(resolve(fixture.runtime, manifest.dependencies["@memmy/agent-source-core"].slice(5))).toBe(fixture.core); - expect(lock.packages[""].dependencies).toEqual(manifest.dependencies); - expect(lock.packages["node_modules/@memmy/agent-source-core"]).toBeUndefined(); - expect(lock.packages["node_modules/fixture-public"]).toEqual({ - version: "1.0.0", dependencies: { "fixture-transitive": "1.0.0" }, - }); - expect(lock.packages["node_modules/fixture-transitive"]).toEqual({ version: "1.0.0" }); - expect(JSON.parse(readFileSync(join(fixture.runtime, "memory-runtime.json"), "utf8"))).toMatchObject({ - version: "2.1.2", target: "darwin-arm64", entrypoint: "dist/src/server/index.js", - }); - }); - - it("keeps macOS Memory workspace imports working after staging is moved away from the repository", () => { - const fixture = macMemoryManifestFixture(); - const generated = generateMacMemoryManifest(fixture); - expect(generated.status, generated.stderr).toBe(0); - const commands = macBuildScriptSource.split(/\r?\n/).filter((line) => - /^npm (install|ci) --prefix "\$RUNTIME_DIR\/memory"/.test(line)); - expect(commands).toHaveLength(2); - const npmConfig = join(fixture.root, "empty.npmrc"); - const npmGlobalConfig = join(fixture.root, "empty-global.npmrc"); - writeFileSync(npmConfig, ""); - writeFileSync(npmGlobalConfig, ""); - const installed = spawnSync("bash", ["-c", [ - "set -euo pipefail", - 'RUNTIME_DIR="$1"', - 'fixture_node="$2"', - 'fixture_npm_cli="$3"', - 'TARGET_CPU="arm64"', - 'npm() { "$fixture_node" "$fixture_npm_cli" "$@"; }', - ...commands, - ].join("\n"), "mac-memory-fixture", dirname(fixture.runtime), process.execPath, findNpmCli()], { - cwd: fixture.root, - encoding: "utf8", - timeout: 30_000, - env: { - PATH: process.env.PATH, - ...(process.env.SystemRoot ? { SystemRoot: process.env.SystemRoot } : {}), - npm_config_userconfig: npmConfig, - npm_config_globalconfig: npmGlobalConfig, - npm_config_cache: join(fixture.root, "npm-cache"), - npm_config_offline: "true", - npm_config_ignore_scripts: "true", - npm_config_audit: "false", - npm_config_fund: "false", - }, - }); - expect(installed.status, [ - installed.stderr || installed.error?.message, - readFileSync(join(fixture.runtime, "package-lock.json"), "utf8"), - ].join("\n")).toBe(0); - const installedCore = join(fixture.runtime, "node_modules", "@memmy", "agent-source-core"); - expect(lstatSync(installedCore).isSymbolicLink()).toBe(false); - const relocated = join(fixture.root, "relocated-memory"); - renameSync(fixture.runtime, relocated); - rmSync(fixture.core, { recursive: true, force: true }); - const imported = spawnSync(process.execPath, ["--input-type=module", "-e", - 'import { fixtureValue } from "@memmy/agent-source-core"; console.log(fixtureValue);', - ], { cwd: relocated, encoding: "utf8" }); - expect(imported.status, imported.stderr).toBe(0); - expect(imported.stdout.trim()).toBe("local-core-ready"); - }); - it("writes exactly the public allowlist and never serializes env decoys", async () => { const root = fixtureRoot(); const envFile = join(root, ".env"); @@ -682,61 +605,3 @@ function fixtureRoot() { roots.push(root); return root; } - -function macMemoryManifestFixture({ externalDependency = false } = {}) { - // npm compares real package paths; macOS /var is an alias for /private/var. - const root = realpathSync(fixtureRoot()); - const repository = join(root, "repository with spaces"); - const core = join(repository, "AgentSourceCore"); - const memory = join(repository, "Memory"); - const runtime = join(repository, "App", "shell", "desktop", "dist", "runtime", "memory"); - writeFixtureJson(join(core, "package.json"), { - name: "@memmy/agent-source-core", version: "0.0.0", private: true, type: "module", main: "dist/src/index.js", - }); - mkdirSync(join(core, "dist", "src"), { recursive: true }); - writeFileSync(join(core, "dist", "src", "index.js"), 'export const fixtureValue = "local-core-ready";\n'); - writeFixtureJson(join(memory, "package.json"), { - name: "@memmy/memory", version: "2.1.2", - dependencies: { "@memmy/agent-source-core": "0.0.0", ...(externalDependency ? { "fixture-public": "1.0.0" } : {}) }, - }); - writeFixtureJson(join(repository, "package-lock.json"), { - lockfileVersion: 3, - requires: true, - packages: { - "": { name: "fixture-repository", version: "1.1.4" }, - AgentSourceCore: { name: "@memmy/agent-source-core", version: "0.0.0" }, - "node_modules/@memmy/agent-source-core": { resolved: "AgentSourceCore", link: true }, - ...(externalDependency ? { - "node_modules/fixture-public": { version: "1.0.0", dependencies: { "fixture-transitive": "1.0.0" } }, - "node_modules/fixture-transitive": { version: "1.0.0" }, - } : {}), - }, - }); - return { root, repository, core, memory, runtime }; -} - -function generateMacMemoryManifest(fixture) { - // Execute only the manifest generator, never the build script or its credential setup. - const generator = /create_memory_runtime_manifest\(\) \{[\s\S]*?node --input-type=module <<'NODE'\r?\n([\s\S]*?)\r?\nNODE\r?\n\}/.exec(macBuildScriptSource)?.[1]; - expect(generator).toBeTypeOf("string"); - return spawnSync(process.execPath, ["--input-type=module"], { - input: generator, - cwd: fixture.root, - encoding: "utf8", - env: { ROOT_DIR: fixture.repository, MEMORY_DIR: fixture.memory, MEMORY_RUNTIME_DIR: fixture.runtime, TARGET_CPU: "arm64" }, - }); -} - -function findNpmCli() { - if (process.env.npm_execpath?.endsWith("npm-cli.js")) return process.env.npm_execpath; - for (const directory of (process.env.PATH ?? "").split(delimiter)) { - const windowsCli = join(directory, "node_modules", "npm", "bin", "npm-cli.js"); - if (existsSync(windowsCli)) return windowsCli; - const executable = join(directory, "npm"); - if (existsSync(executable)) { - const resolved = realpathSync(executable); - if (resolved.endsWith("npm-cli.js")) return resolved; - } - } - throw new Error("An installed npm CLI is required for the offline package fixture"); -} diff --git a/tests/release-workflow.test.ts b/tests/release-workflow.test.ts index aed6670d3..b28aecd3f 100644 --- a/tests/release-workflow.test.ts +++ b/tests/release-workflow.test.ts @@ -9,7 +9,6 @@ const repoRoot = resolve(import.meta.dirname, ".."); const legacyWorkflowPath = resolve(repoRoot, ".github/workflows/github-release.yml"); const draftWorkflowPath = resolve(repoRoot, ".github/workflows/github-draft-release-v2.yml"); const releaseCompareScriptPath = resolve(repoRoot, "scripts/build-release-compare.mjs"); -const releaseNotesSanitizerPath = resolve(repoRoot, "scripts/sanitize-release-notes.mjs"); const ossIntegrityScriptPath = resolve(repoRoot, "scripts/internal/shared/oss-object-integrity.mjs"); const draftSource = readFileSync(draftWorkflowPath, "utf8"); const releaseCompareSource = readFileSync(releaseCompareScriptPath, "utf8"); @@ -58,186 +57,6 @@ function readJson(relativePath: string): { return JSON.parse(readFileSync(resolve(repoRoot, relativePath), "utf8")); } -function runReleaseNotesSanitizer(markdown: string, publicLanguage?: "en") { - const tempDir = mkdtempSync(resolve(tmpdir(), "memmy-release-notes-sanitizer-")); - const inputPath = resolve(tempDir, "input.md"); - const outputPath = resolve(tempDir, "output.md"); - writeFileSync(inputPath, markdown); - - const result = spawnSync( - "node", - [ - releaseNotesSanitizerPath, - inputPath, - outputPath, - ...(publicLanguage ? ["--language", publicLanguage] : []), - ], - { cwd: repoRoot, encoding: "utf8" }, - ); - - return { - result, - output: existsSync(outputPath) ? readFileSync(outputPath, "utf8") : "", - }; -} - -describe("public release notes sanitizer", () => { - it("removes reserved audit comments but preserves public and fenced content", () => { - const { result, output } = runReleaseNotesSanitizer(` -# Memmy v1.1.3 - -Public release notes. - - - - - - - -\`\`\`markdown - - -\`\`\` - -~~~text - -~~~ -`); - - expect(result.status, result.stderr).toBe(0); - expect(output).toContain("# Memmy v1.1.3"); - expect(output).toContain("Public release notes."); - expect(output).toContain(""); - expect(output).toContain(""); - expect(output).toContain("inside: backtick-fence"); - expect(output).toContain("inside: tilde-fence"); - expect(output).not.toContain("memmy-official-changelog-v2"); - expect(output).not.toContain("target_sha: abc123"); - expect(output.endsWith("\n")).toBe(true); - }); - - it("fails closed for unterminated or inline reserved metadata", () => { - const unterminated = runReleaseNotesSanitizer(` -# Memmy - -\n", - ); - expect(inline.result.status).not.toBe(0); - expect(inline.result.stderr).toContain("must occupy complete lines"); - expect(inline.output).toBe(""); - - const afterOrdinaryComment = runReleaseNotesSanitizer( - " \n", - ); - expect(afterOrdinaryComment.result.status).not.toBe(0); - expect(afterOrdinaryComment.result.stderr).toContain("must occupy complete lines"); - expect(afterOrdinaryComment.output).toBe(""); - - const metadataOnly = runReleaseNotesSanitizer( - "\n", - ); - expect(metadataOnly.result.status).not.toBe(0); - expect(metadataOnly.result.stderr).toContain("no public content"); - expect(metadataOnly.output).toBe(""); - }); - - it("keeps only English sections when a reviewed source contains parallel Chinese sections", () => { - const { result, output } = runReleaseNotesSanitizer( - `# Memmy v1.1.3 - -## Fixes - -- Fixed packaged Memory startup. - -## 修复 - -- 修复随包 Memory 的启动问题。 - -## Upgrade notes - -- Memory remains independently versioned. - -## 升级说明 - -- Memory 继续独立发版。 -`, - "en", - ); - - expect(result.status, result.stderr).toBe(0); - expect(output).toContain("## Fixes"); - expect(output).toContain("Fixed packaged Memory startup."); - expect(output).toContain("## Upgrade notes"); - expect(output).not.toMatch(/[\u3040-\u30ff\u3400-\u9fff\uf900-\ufaff]/); - expect(output.match(/^## Fixes$/gm)).toHaveLength(1); - }); - - it("rejects Chinese prose that remains inside an English section", () => { - const { result, output } = runReleaseNotesSanitizer( - `# Memmy v1.1.3 - -## Fixes - -- 修复随包 Memory 的启动问题。 -`, - "en", - ); - - expect(result.status).not.toBe(0); - expect(result.stderr).toContain("English public release notes contain visible CJK text"); - expect(output).toBe(""); - }); - - it("allows CJK characters in code spans while validating English prose", () => { - const { result, output } = runReleaseNotesSanitizer( - `# Memmy v1.1.3 - -## Fixes - -- Fixed startup when the configured path is \`C:\\\\用户\\\\Memmy\`. -`, - "en", - ); - - expect(result.status, result.stderr).toBe(0); - expect(output).toContain("`C:\\\\用户\\\\Memmy`"); - }); - - it("normalizes the real v1.1.3 reviewed notes to a single English public body", () => { - const notes = readFileSync( - resolve(repoRoot, ".github/release-notes/v1.1.3.md"), - "utf8", - ); - const { result, output } = runReleaseNotesSanitizer(notes, "en"); - - expect(result.status, result.stderr).toBe(0); - expect(output).toContain("# Memmy v1.1.3"); - expect(output).toContain("## Fixes"); - expect(output).toContain("## Upgrade notes"); - expect(output).not.toMatch(/[\u3040-\u30ff\u3400-\u9fff\uf900-\ufaff]/); - expect(output.match(/^## Fixes$/gm)).toHaveLength(1); - expect(output.match(/^## Upgrade notes$/gm)).toHaveLength(1); - }); -}); - describe("Memmy release workflow metadata", () => { it("keeps Memmy metadata aligned while preserving the independent Memory version", () => { const version = readJson("package.json").version; @@ -498,7 +317,6 @@ describe("GitHub Draft Release v2 workflow", () => { (step) => step.name === "Preflight Doc Agent draft endpoint", ); expect(preflight).toBeDefined(); - expect(preflight?.id).toBe("doc_agent"); expect(preflight?.if).toBeUndefined(); const script = draftScript("Preflight Doc Agent draft endpoint"); @@ -515,10 +333,6 @@ describe("GitHub Draft Release v2 workflow", () => { expect(script).toContain("Doc Agent draft endpoint unavailable"); expect(script).toContain("Doc Agent smoke response contract mismatch"); expect(script).toContain("LLM generation: not invoked by smoke"); - expect(script).toContain('echo "available=false" >> "$GITHUB_OUTPUT"'); - expect(script).toContain('echo "available=true" >> "$GITHUB_OUTPUT"'); - expect(script).toContain("safe needs-review Draft"); - expect(script).not.toContain("::error title=Doc Agent"); }); it("reuses a pre-existing tag only when it points at the target commit", () => { @@ -625,35 +439,18 @@ describe("GitHub Draft Release v2 workflow", () => { expect(releaseNotes).toContain("DOC_AGENT_RELEASE_NOTES_REQUEST.json"); expect(releaseNotes).toContain("MEMMY_RELEASE_STYLE_EXAMPLES.json"); expect(releaseNotes).toContain("candidate_count: 3"); - expect(releaseNotes).toContain('public_release_language: "en"'); - expect(releaseNotes).toContain("reviewed_release_notes"); - expect(releaseNotes).toContain("REVIEWED_RELEASE_NOTES.md"); - expect(releaseNotes).toContain("manual-reviewed-by-doc-agent"); - expect(releaseNotes).toContain("manual-repaired-by-doc-agent"); - expect(releaseNotes).toContain("doc-agent-manual-regeneration"); expect(releaseNotes).toContain(".release_notes_md // .release_notes_markdown"); - expect(releaseNotes).toContain("write_safe_fallback_body"); - expect(releaseNotes).toContain("safe-needs-review-fallback"); - expect(releaseNotes).toContain("manual-needs-review-fallback"); - expect(releaseNotes).toContain("safe_needs_review_draft"); - expect(releaseNotes).toContain("will not publish it automatically"); - expect(releaseNotes).toContain("exhausted automatic wording repair"); + expect(releaseNotes).toContain("Doc Agent draft configuration missing"); + expect(releaseNotes).toContain("Doc Agent draft generation failed"); + expect(releaseNotes).toContain("do not fall back silently"); + expect(releaseNotes).toContain("Doc Agent returned invalid release notes"); + expect(releaseNotes).toContain("Doc Agent quality report missing"); + expect(releaseNotes).toContain("Doc Agent candidate selection missing"); + expect(releaseNotes).toContain("Doc Agent release notes need review"); expect(releaseNotes).toContain("requested_candidate_count"); - expect(releaseNotes).toContain("Release notes body was empty"); + expect(releaseNotes).toContain("Release notes generation produced an empty body"); expect(releaseNotes).toContain("RELEASE_NOTES_SOURCE.json"); expect(releaseNotes).toContain("QUALITY_REPORT.json"); - expect(existsSync(releaseNotesSanitizerPath)).toBe(true); - expect(releaseNotes).toContain( - 'node scripts/sanitize-release-notes.mjs "$notes" "$sanitized_notes" --language en', - ); - expect(releaseNotes).toContain('mv "$sanitized_notes" "$notes"'); - expect(releaseNotes).toContain("Release notes sanitization repaired with safe fallback"); - expect(releaseNotes).toContain("Safe release-notes fallback failed"); - expect(releaseNotes).toContain('public_release_language: "en"'); - expect(releaseNotes).toContain("language_validation"); - expect(releaseNotes).not.toContain("