Repository navigation
Expand file tree
/
Copy pathindex.ts
More file actions
1848 lines (1735 loc) · 67.5 KB
/
Copy pathindex.ts
File metadata and controls
1848 lines (1735 loc) · 67.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env node
// The memory server: a knowledge graph of entities, relations and
// observations, persisted as JSONL and served over MCP as nine tools and one
// subscribable resource. createServer() builds a server over a graph file so
// tests can run it in-process; main() serves it on stdio through the SDK's
// serveStdio entry, which answers 2026-07-28 clients (server/discover, change
// notifications on subscriptions/listen) and 2025-era clients (initialize,
// resources/subscribe) from the same factory (#4852). run() reads --transport
// and starts stdio (the default) or Streamable HTTP (http.ts, #576). It runs
// only when this file is the process entry point, so importing it starts
// nothing.
import {
serveStdio,
type StdioServerHandle,
} from "@modelcontextprotocol/server/stdio";
import { McpServer } from "@modelcontextprotocol/server";
import type { Transport } from "@modelcontextprotocol/server";
import { z } from "zod";
import { promises as fs, readFileSync, realpathSync, unlinkSync } from "fs";
import type { FileHandle } from "fs/promises";
import path from "path";
import os from "os";
import { randomBytes } from "crypto";
import { fileURLToPath } from "url";
import { SERVER_VERSION } from "./version.js";
// The package directory: where the default graph file lives, and what a
// relative MEMORY_FILE_PATH resolves against.
const defaultMemoryDir = path.dirname(fileURLToPath(import.meta.url));
// Define memory file path using environment variable with fallback
export const defaultMemoryPath = path.join(defaultMemoryDir, "memory.jsonl");
// Expand a leading "~" to the user's home directory. MCP clients pass
// MEMORY_FILE_PATH from JSON config, where no shell performs tilde expansion,
// so an unexpanded "~" would otherwise be treated as a relative path and
// joined onto the package directory. Mirrors the helper of the same name in
// the filesystem server (src/filesystem/path-utils.ts).
export function expandHome(filepath: string): string {
if (filepath.startsWith("~/") || filepath === "~") {
return path.join(os.homedir(), filepath.slice(1));
}
return filepath;
}
// Handle backward compatibility: migrate memory.json to memory.jsonl if needed.
// baseDir is the directory the default files live in. The server never passes
// it; it exists so tests can run the migration in a temporary directory rather
// than in the package directory, which every importer of this module shares.
export async function ensureMemoryFilePath(
baseDir: string = defaultMemoryDir,
): Promise<string> {
if (process.env.MEMORY_FILE_PATH) {
// Custom path provided. Expand a leading "~" first, then resolve relative
// paths against the package directory (absolute paths are used as-is).
const customPath = expandHome(process.env.MEMORY_FILE_PATH);
return path.isAbsolute(customPath)
? customPath
: path.join(baseDir, customPath);
}
// No custom path set, check for backward compatibility migration
const oldMemoryPath = path.join(baseDir, "memory.json");
const newMemoryPath = path.join(baseDir, "memory.jsonl");
// Only a missing legacy file means there is nothing to migrate. Any other
// failure, from the existence checks or the rename itself, propagates and
// stops startup: carrying on would serve an empty memory.jsonl while the
// user's graph sat unread in memory.json (#5203).
if (!(await pathExists(oldMemoryPath))) {
return newMemoryPath;
}
if (await pathExists(newMemoryPath)) {
// Both files exist, use new one (no migration needed)
return newMemoryPath;
}
// Old file exists, new file doesn't - migrate
console.error(
"DETECTED: Found legacy memory.json file, migrating to memory.jsonl for JSONL format compatibility",
);
try {
await fs.rename(oldMemoryPath, newMemoryPath);
} catch (error) {
const reason = error instanceof Error ? error.message : String(error);
throw new Error(
`Failed to migrate legacy memory file ${oldMemoryPath} to ${newMemoryPath}: ${reason}. ` +
"The server will not start on an empty graph; fix the cause and restart, " +
"or set MEMORY_FILE_PATH to choose a file explicitly.",
{ cause: error },
);
}
console.error("COMPLETED: Successfully migrated memory.json to memory.jsonl");
return newMemoryPath;
}
// True when filePath exists, false when it does not (ENOENT). Any other error,
// such as a permission failure, is rethrown rather than read as "missing".
async function pathExists(filePath: string): Promise<boolean> {
try {
await fs.access(filePath);
return true;
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
return false;
}
throw error;
}
}
// The startup warning for a memory file inside a node_modules tree, or
// undefined for any other path. npx and npm install the server under
// node_modules, so the default memory.jsonl beside dist/index.js lands there,
// and npm treats that tree as its own: a reinstall, an update or a cleared npx
// cache can delete the file and the graph in it (#4117). The default is not
// moved, since that would strand every existing graph; the warning says how to
// choose a durable path instead. Only node_modules is matched, not the server's
// directory as such: the Docker setup mounts a volume over /app/dist, and a
// source checkout's dist survives a rebuild, so there the default is durable.
export function memoryFileLocationWarning(
memoryFilePath: string,
): string | undefined {
// Both separators, so a Windows path is matched on any platform.
const segments = path.resolve(memoryFilePath).split(/[\\/]/);
if (!segments.includes("node_modules")) {
return undefined;
}
return (
`WARNING: the memory file ${memoryFilePath} is inside a node_modules directory, ` +
"which npm manages: reinstalling or updating the server, or clearing the npx cache, " +
"can delete it and the knowledge graph it holds. Set MEMORY_FILE_PATH to an absolute " +
"path outside node_modules, in a directory that exists (for example ~/memory.jsonl), " +
"and move this file there to keep the graph."
);
}
// We are storing our memory using entities, relations, and observations in a graph structure
export interface Entity {
name: string;
entityType: string;
observations: string[];
}
export interface Relation {
from: string;
to: string;
relationType: string;
}
export interface KnowledgeGraph {
entities: Entity[];
relations: Relation[];
}
// Names of requested entities that createEntities did not create: the name
// already existed, or repeated earlier in the same batch. createEntities
// returns the created entities themselves, so identity tells them apart.
export function skippedEntityNames(
requested: Entity[],
created: Entity[],
): string[] {
return requested.filter((e) => !created.includes(e)).map((e) => e.name);
}
// Text telling the agent which entities create_entities skipped.
export function skippedEntitiesNotice(skipped: string[]): string {
const one = skipped.length === 1;
return (
`Skipped ${skipped.length} ${one ? "entity that already exists" : "entities that already exist"}: ` +
`${skipped.join(", ")}. ${one ? "Its" : "Their"} observations were not added; ` +
"use add_observations for existing entities."
);
}
// Opt-in ceilings on how large the write tools may grow the graph (#5525).
// Every mutation reads the whole graph file and writes it back under the
// cross-process lock, so its cost, and how long it holds the lock, grow
// linearly with the file. These let an operator bound that growth. Each is
// unset by default, which leaves the store unbounded, as it always was.
export interface GraphLimits {
// Most entities the graph may hold.
maxEntities?: number;
// Most relations the graph may hold.
maxRelations?: number;
// Most bytes the saved graph file may hold.
maxFileBytes?: number;
}
// The environment variable that sets each limit.
export const GRAPH_LIMIT_ENV: Readonly<Record<keyof GraphLimits, string>> = {
maxEntities: "MEMORY_MAX_ENTITIES",
maxRelations: "MEMORY_MAX_RELATIONS",
maxFileBytes: "MEMORY_MAX_FILE_BYTES",
};
// Read the limits from the environment. Only an unset variable leaves its
// limit off; any other value that is not a positive whole number, an empty
// one included, stops startup, since a limit silently ignored would leave the
// store unbounded while the operator believes it capped.
export function graphLimitsFromEnv(
env: NodeJS.ProcessEnv = process.env,
): GraphLimits {
const limits: GraphLimits = {};
for (const key of Object.keys(GRAPH_LIMIT_ENV) as (keyof GraphLimits)[]) {
const name = GRAPH_LIMIT_ENV[key];
if (env[name] === undefined) {
continue;
}
const raw = env[name].trim();
const value = Number(raw);
if (!/^\d+$/.test(raw) || !Number.isSafeInteger(value) || value < 1) {
throw new Error(
`${name} must be a positive whole number, got "${env[name]}". ` +
"Unset it to leave that limit off.",
);
}
limits[key] = value;
}
return limits;
}
// How large a graph is, on each dimension GraphLimits caps.
interface GraphSize {
entities: number;
relations: number;
bytes: number;
}
// Timings for the cross-process lock around each mutation. Exposed so tests
// can shorten them; the server always uses the defaults.
export interface FileLockOptions {
// A lock not refreshed for this long is presumed abandoned and is broken.
staleMs?: number;
// Give up, failing the mutation, after waiting this long for the lock.
timeoutMs?: number;
// Wait between attempts, plus up to the same again of random jitter.
retryMs?: number;
}
// For best-effort lock cleanup, whose failure must not mask the real outcome.
function ignoreError(): void {}
// Whether opening a lock file failed because the file is held right now. On
// Windows, opening a file that another process is still deleting fails with
// EPERM rather than EEXIST until the deletion completes, so there EPERM means
// "held" too.
function isLockBusy(error: unknown): boolean {
const code = (error as NodeJS.ErrnoException).code;
return (
code === "EEXIST" || (process.platform === "win32" && code === "EPERM")
);
}
// Whether the process that wrote a lock file is known to be gone. Only a
// process on this host can be checked; for any other host, or a lock whose
// contents cannot be read, the lock's age decides instead.
function isLockOwnerDead(contents: string): boolean {
let owner: unknown;
try {
owner = JSON.parse(contents);
} catch {
return false;
}
const { pid, hostname } = (owner ?? {}) as Record<string, unknown>;
if (typeof pid !== "number" || hostname !== os.hostname()) {
return false;
}
try {
process.kill(pid, 0);
return false;
} catch (error) {
// EPERM means the process exists but belongs to another user.
return (error as NodeJS.ErrnoException).code === "ESRCH";
}
}
// What a waiter learns from one look at a lock file: whether it is abandoned,
// and which file it looked at. The device and inode numbers identify the file
// itself rather than its path, so a later check can tell that file from one
// that has replaced it at the same path. They are read as bigints because a
// Windows file id does not fit in a double.
interface LockJudgement {
stale: boolean;
dev: bigint;
ino: bigint;
}
// Judge the lock file open at handle. Its owner and its mtime are read
// through that one handle, so both describe the same lock file even if it is
// replaced meanwhile.
async function judgeLock(
handle: FileHandle,
staleMs: number,
): Promise<LockJudgement> {
const [contents, stats] = await Promise.all([
handle.readFile("utf-8"),
handle.stat({ bigint: true }),
]);
return {
stale:
isLockOwnerDead(contents) || Date.now() - Number(stats.mtimeMs) > staleMs,
dev: stats.dev,
ino: stats.ino,
};
}
// Whether the lock at lockPath is abandoned.
async function isLockStale(
lockPath: string,
staleMs: number,
): Promise<boolean> {
const handle = await fs.open(lockPath, "r");
try {
return (await judgeLock(handle, staleMs)).stale;
} finally {
await handle.close();
}
}
// Remove an abandoned lock, returning whether it did. Breaking is itself
// guarded by a second O_EXCL file, so only one waiter breaks at a time and
// re-checks the lock under that guard: two waiters can never both judge the
// same abandoned lock stale and one then remove the lock the other has just
// taken in its place.
//
// The guard orders breakers only. The lock's own holder can still release it,
// and a breaker that stalls past staleMs loses its guard to another, so the
// file at lockPath can be replaced by a successor's live lock after the
// re-check has judged it stale. Removing by path would then delete that live
// lock and let a third writer in beside its holder (#5560). So the lock is
// never deleted where it stands. It is renamed aside to a name nobody else
// uses, which is atomic and takes whichever file is at lockPath at that
// instant, and what was taken is then compared with the file that was judged:
// the judged file is deleted, and anything else is a live lock and is renamed
// straight back. The handle the judgement was read through stays open until
// then, so the judged file's inode number cannot be reused by a successor's
// file and the comparison cannot be fooled.
//
// What remains is the moment a live lock spends aside before it is put back:
// a writer that creates the lock in exactly that moment is overwritten by the
// restored one and runs beside its holder. Reaching it takes a replacement
// between the judgement and the rename, then a create between the two
// renames; a lock file cannot close that without the kernel's help.
async function breakStaleLock(
lockPath: string,
staleMs: number,
): Promise<boolean> {
const breakerPath = `${lockPath}.break`;
let breaker: FileHandle;
try {
breaker = await fs.open(breakerPath, "wx");
} catch (error) {
if (!isLockBusy(error)) {
throw error;
}
// Another waiter is breaking the lock. Its guard is held for a moment
// only, so one that outlives staleMs was left by a crash: clear it.
const stats = await fs.stat(breakerPath).catch(() => null);
if (stats && Date.now() - stats.mtimeMs > staleMs) {
await fs.unlink(breakerPath).catch(ignoreError);
}
return false;
}
try {
await breaker.close();
const judged = await fs.open(lockPath, "r");
try {
const { stale, dev, ino } = await judgeLock(judged, staleMs);
if (!stale) {
return false;
}
const asidePath = `${lockPath}.${randomBytes(8).toString("hex")}.stale`;
await fs.rename(lockPath, asidePath);
const taken = await fs.stat(asidePath, { bigint: true });
if (taken.dev !== dev || taken.ino !== ino) {
// Not the lock that was judged: a successor's live one. Put it back.
await fs.rename(asidePath, lockPath);
return false;
}
console.error(`Breaking stale memory file lock ${lockPath}`);
await fs.unlink(asidePath);
return true;
} finally {
await judged.close();
}
} finally {
await fs.unlink(breakerPath).catch(ignoreError);
}
}
// The lock files this process holds right now, each mapped to the token it
// writes into it and whether that write has finished. withFileLock releases its lock in a `finally`, which never
// runs when a signal ends the process, so the exit hook releaseHeldLocks()
// removes them instead (#5485). Without it, a lock left by a stopped server
// stalls the next writer until it goes stale, which takes the full staleMs
// when that writer is on another host (another container) and cannot see
// that this process has gone.
interface HeldLock {
token: string;
written: boolean;
}
const heldLocks = new Map<string, HeldLock>();
// The lock paths this process holds right now, for tests.
export function heldLockPaths(): string[] {
return [...heldLocks.keys()];
}
// Synchronously remove every lock this process still holds, as the process
// exits: an `exit` listener cannot wait on a promise. As in withFileLock's own
// release, a lock is removed only while it still holds this process's token,
// so a lock broken as stale and re-taken by another process stays in place.
// A lock is tracked from the moment its exclusive create returns, before its
// token is written. Until that write finishes, a file holding only the start
// of the token (or none of it) is ours too: the exit interrupted the write.
// Nothing else can have replaced it by then, since a lock that new, with a
// live owner, is never judged stale. Once the token is written only an exact
// match is ours, so another writer's replacement, empty while its own token
// is being written, is left alone.
export function releaseHeldLocks(): void {
for (const [lockPath, { token, written }] of heldLocks) {
try {
const contents = readFileSync(lockPath, "utf-8");
if (written ? contents === token : token.startsWith(contents)) {
unlinkSync(lockPath);
}
} catch {
// Already gone, or unreadable: there is nothing of ours to remove.
}
}
heldLocks.clear();
}
// The exit codes a shell reports for a process its signal ended: 128 + the
// signal's number.
const SIGNAL_EXIT_CODES = { SIGINT: 130, SIGTERM: 143 } as const;
let exitLockCleanupInstalled = false;
let lockCleanupInstalled = false;
// Release held locks when the process exits normally or through
// process.exit(), by the `exit` event. Installed once, however the server is
// started. It covers a signal only when something turns that signal into a
// process.exit(): installLockCleanup() does for stdio, and run() does for
// Streamable HTTP, whose signals first close the listener (#5595).
export function installExitLockCleanup(): void {
if (exitLockCleanupInstalled) {
return;
}
exitLockCleanupInstalled = true;
process.on("exit", releaseHeldLocks);
}
// Release held locks however the server process ends (#5485): on a normal
// exit or process.exit() through the `exit` event, and on SIGINT or SIGTERM,
// whose default action would end the process without emitting `exit`. A
// signal handler replaces that default, so it exits itself, with the status
// the default would have given. Installed once, by main(): only the server
// process should take over its signals, not a test that imports this module.
export function installLockCleanup(): void {
if (lockCleanupInstalled) {
return;
}
lockCleanupInstalled = true;
installExitLockCleanup();
for (const [signal, code] of Object.entries(SIGNAL_EXIT_CODES)) {
process.on(signal, () => process.exit(code));
}
}
// Run operation while holding an exclusive lock file at lockPath, so that
// mutations from separate server processes sharing one graph file take turns
// and each one loads the graph the previous one saved (#4797). The lock is a
// file created with O_EXCL, which is atomic on local filesystems: exactly one
// creator wins. The holder refreshes the lock's mtime every staleMs / 2 while
// it runs. A lock left behind by a crashed process is broken once its owner
// is known to be dead, or once it has gone unrefreshed for staleMs.
export async function withFileLock<T>(
lockPath: string,
operation: () => Promise<T>,
{ staleMs = 30_000, timeoutMs = 60_000, retryMs = 10 }: FileLockOptions = {},
): Promise<T> {
const token = JSON.stringify({
pid: process.pid,
hostname: os.hostname(),
token: randomBytes(16).toString("hex"),
});
const deadline = Date.now() + timeoutMs;
for (;;) {
let handle: FileHandle | undefined;
try {
handle = await fs.open(lockPath, "wx");
} catch (error) {
if (!isLockBusy(error)) {
throw error;
}
}
if (handle) {
// Track the lock before anything else is awaited: a signal is handled
// only between tasks, never between the open's completion and this
// line, so from here on the exit hook can remove it (#5485).
const held: HeldLock = { token, written: false };
heldLocks.set(lockPath, held);
try {
await handle.writeFile(token);
held.written = true;
await handle.close();
} catch (error) {
// The lock was created but not written: remove it rather than leave
// every other writer waiting for it to go stale.
await handle.close().catch(ignoreError);
await fs.unlink(lockPath).catch(ignoreError);
heldLocks.delete(lockPath);
throw error;
}
break;
}
// Someone else holds the lock. Break it if it is abandoned; otherwise
// wait and try again.
try {
if (
(await isLockStale(lockPath, staleMs)) &&
(await breakStaleLock(lockPath, staleMs))
) {
continue;
}
} catch (error) {
// The holder released the lock between our attempts: try again now.
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
continue;
}
// On Windows, the lock is still being deleted: wait, then try again.
if (!isLockBusy(error)) {
throw error;
}
}
if (Date.now() >= deadline) {
throw new Error(
`Timed out after ${timeoutMs} ms waiting for the memory file lock ${lockPath}`,
);
}
await new Promise((resolve) =>
setTimeout(resolve, retryMs + Math.random() * retryMs),
);
}
// Keep the lock fresh while the operation runs, so a mutation that outlasts
// staleMs (a large graph, a slow disk) is not mistaken for an abandoned one.
// Refresh only our own lock, as the release below does: once it is broken
// as stale and re-taken, the file belongs to another process, and keeping
// it fresh would stop a crashed new owner's lock from ever going stale.
const heartbeat = setInterval(() => {
const now = new Date();
// Not awaited: an interval callback has no caller to await it, and every
// failure (the lock already gone) is caught and ignored here.
void fs
.readFile(lockPath, "utf-8")
.then((current) => {
if (current !== token) {
// The lock changed hands: stop refreshing it, for good.
clearInterval(heartbeat);
return;
}
return fs.utimes(lockPath, now, now);
})
.catch(ignoreError);
}, staleMs / 2);
heartbeat.unref();
try {
return await operation();
} finally {
clearInterval(heartbeat);
// Release only our own lock: if it was broken as stale while we held it,
// the file may now belong to another process.
const current = await fs.readFile(lockPath, "utf-8").catch(() => null);
if (current === token) {
await fs.unlink(lockPath).catch(ignoreError);
}
// Stop tracking the lock only once the release is done, so an exit during
// it still removes the lock. A lock this same process broke as stale and
// re-took in another withFileLock call carries that call's token, and
// stays tracked for it.
if (heldLocks.get(lockPath)?.token === token) {
heldLocks.delete(lockPath);
}
}
}
// The KnowledgeGraphManager class contains all operations to interact with the knowledge graph
export class KnowledgeGraphManager {
constructor(
private memoryFilePath: string,
private lockOptions: FileLockOptions = {},
private limits: GraphLimits = {},
) {}
// Serializes all read-modify-write graph mutations behind a single queue.
// Without this, concurrent tool calls (e.g. multiple mutations dispatched
// from one LLM turn) each independently load the graph, mutate their own
// copy, and write it back — so whichever write lands last silently
// overwrites the other's changes, and interleaved writes to the same file
// can corrupt it outright. See #1819.
// The queue only orders calls within this process, so each queued mutation
// also holds a lock file next to the graph file while it runs, which orders
// it against other server processes sharing the same file. See #4797.
private mutationQueue: Promise<unknown> = Promise.resolve();
// Members of a stored entity or relation record that the schema does not
// model (a createdAt written by an older version or another client, say),
// keyed by the parsed object loadGraphWithUnreadable made from that record.
// saveGraph writes them back, so a write keeps every record it did not
// delete exactly as it found it: that is what makes create_entities,
// create_relations and add_observations additive, as their
// destructiveHint: false declares (#5058). A WeakMap rather than a property
// on the object, so the extras never reach a tool result or read_graph,
// whose output schemas do not allow them, and are collected with the graph
// they came from. A deleted record is dropped from the graph and takes its
// extras with it; one changed in place (add_observations,
// delete_observations) keeps its object, so keeps them too.
private extraFields = new WeakMap<object, Record<string, unknown>>();
private async withLock<T>(operation: () => Promise<T>): Promise<T> {
const locked = () =>
withFileLock(`${this.memoryFilePath}.lock`, operation, this.lockOptions);
const result = this.mutationQueue.then(locked, locked);
// Always resolve the queue itself, even if this operation failed, so a
// single failed mutation doesn't permanently wedge every call after it.
// The failure still propagates normally to whoever awaited `result`.
this.mutationQueue = result.then(
() => undefined,
() => undefined,
);
return result;
}
private async loadGraph(): Promise<KnowledgeGraph> {
return (await this.loadGraphWithUnreadable()).graph;
}
// Lines that can't be read (malformed JSON, entries that fail validation,
// unknown record types) stay out of the in-memory graph, but mutations pass
// them back to saveGraph so they are written out unchanged. Otherwise the
// next unrelated write would silently delete them from disk, together with
// every valid observation of an entity that has one bad field.
// size is the graph as loaded, for the write tools' limit check (see
// checkLimits).
private async loadGraphWithUnreadable(): Promise<{
graph: KnowledgeGraph;
unreadable: string[];
size: GraphSize;
}> {
try {
const data = await fs.readFile(this.memoryFilePath, "utf-8");
const lines = data.split("\n").filter((line) => line.trim() !== "");
const graph: KnowledgeGraph = { entities: [], relations: [] };
const unreadable: string[] = [];
for (const line of lines) {
let item: unknown;
try {
item = JSON.parse(line);
} catch {
console.error("Skipping malformed line in memory file");
unreadable.push(line);
continue;
}
if (typeof item !== "object" || item === null) {
console.error("Skipping non-object line in memory file");
unreadable.push(line);
continue;
}
const record = item as Record<string, unknown>;
if (record.type === "entity") {
const parsed = EntitySchema.safeParse(item);
if (parsed.success) {
this.keepExtraFields(parsed.data, record, EntitySchema.shape);
graph.entities.push(parsed.data);
} else {
unreadable.push(line);
console.error(
"Skipping invalid entity in memory file:",
parsed.error.issues
.map((issue) => `${issue.path.join(".")}: ${issue.message}`)
.join(", "),
);
}
} else if (record.type === "relation") {
const parsed = RelationSchema.safeParse(item);
if (parsed.success) {
this.keepExtraFields(parsed.data, record, RelationSchema.shape);
graph.relations.push(parsed.data);
} else {
unreadable.push(line);
console.error(
"Skipping invalid relation in memory file:",
parsed.error.issues
.map((issue) => `${issue.path.join(".")}: ${issue.message}`)
.join(", "),
);
}
} else {
unreadable.push(line);
}
}
const size = {
entities: graph.entities.length,
relations: graph.relations.length,
bytes: Buffer.byteLength(data, "utf-8"),
};
return { graph, unreadable, size };
} catch (error) {
if (
error instanceof Error &&
"code" in error &&
error.code === "ENOENT"
) {
return {
graph: { entities: [], relations: [] },
unreadable: [],
size: { entities: 0, relations: 0, bytes: 0 },
};
}
throw error;
}
}
// Record the members of a valid stored record that are neither its type nor
// a field of its schema, for saveGraph to write back. Object.fromEntries
// defines each key as an own property, so even a "__proto__" member stays
// data rather than setting the object's prototype.
private keepExtraFields(
parsed: object,
record: Record<string, unknown>,
shape: Record<string, unknown>,
): void {
const extras = Object.entries(record).filter(
([key]) => key !== "type" && !Object.hasOwn(shape, key),
);
if (extras.length > 0) {
this.extraFields.set(parsed, Object.fromEntries(extras));
}
}
// Refuse a write that would leave the graph over a limit and larger than it
// was (`before`), on any one dimension. Comparing against `before` keeps a
// graph already over a lowered limit usable: a call that adds nothing still
// succeeds, and the delete tools, which never pass `before`, can always
// shrink it. The refusal is thrown before anything is written, so the call
// changes nothing.
private checkLimits(
graph: KnowledgeGraph,
content: string,
before: GraphSize,
): void {
const after: GraphSize = {
entities: graph.entities.length,
relations: graph.relations.length,
bytes: Buffer.byteLength(content, "utf-8"),
};
const dimensions: [keyof GraphSize, keyof GraphLimits, string][] = [
["entities", "maxEntities", "entities"],
["relations", "maxRelations", "relations"],
["bytes", "maxFileBytes", "bytes in the memory file"],
];
for (const [dimension, key, unit] of dimensions) {
const limit = this.limits[key];
if (
limit !== undefined &&
after[dimension] > limit &&
after[dimension] > before[dimension]
) {
throw new Error(
`Refused: this change would bring the knowledge graph to ${after[dimension]} ${unit}, ` +
`over the limit of ${limit} set by ${GRAPH_LIMIT_ENV[key]}. Nothing was saved. ` +
"Delete entities, relations or observations that are no longer needed to make room, " +
`or ask the operator to raise ${GRAPH_LIMIT_ENV[key]}.`,
);
}
}
}
// before is the graph's size as loaded; the write tools that add to the
// graph pass it, so the save is refused if it breaks a limit (checkLimits).
private async saveGraph(
graph: KnowledgeGraph,
unreadable: string[],
before?: GraphSize,
): Promise<void> {
// The modelled fields come first and always hold the current values; a
// record's extras (see extraFields) follow unchanged. A record created by
// this write has none and is written exactly as before.
const lines = [
...graph.entities.map((e) =>
JSON.stringify({
type: "entity",
name: e.name,
entityType: e.entityType,
observations: e.observations,
...this.extraFields.get(e),
}),
),
...graph.relations.map((r) =>
JSON.stringify({
type: "relation",
from: r.from,
to: r.to,
relationType: r.relationType,
...this.extraFields.get(r),
}),
),
...unreadable,
];
const content = lines.join("\n") + "\n";
if (before) {
this.checkLimits(graph, content, before);
}
// Write to a temporary file in the same directory, then rename it over
// the target. fs.writeFile would truncate the memory file before writing,
// so an interruption (SIGKILL, container stop, OOM, power loss) would
// leave the only copy of the graph truncated and unrecoverable.
// rename(2) is atomic on POSIX filesystems: readers see either the
// complete old file or the complete new one, never a partial state.
// The temp file is kept in the same directory so the rename stays on one
// filesystem — renaming across mount points fails with EXDEV.
//
// The rename also replaces the graph file's inode, and a new file takes
// the process umask, so the file's own mode would be lost (#4827): an
// operator's 0600 would come back 0644, and a read-only file would be
// silently replaced, since rename(2) needs only a writable directory.
// So refuse a graph file this process may not write, as fs.writeFile
// would (EACCES), and give the temp file the existing file's permission
// bits before the rename, as src/filesystem/lib.ts preserves them for
// the same write pattern. A graph file that does not exist yet has
// nothing to preserve.
let mode: number | undefined;
try {
await fs.access(this.memoryFilePath, fs.constants.W_OK);
mode = (await fs.stat(this.memoryFilePath)).mode & 0o777;
} catch (error) {
if (
!(error instanceof Error && "code" in error && error.code === "ENOENT")
) {
throw error;
}
}
const directory = path.dirname(this.memoryFilePath);
const tempFilePath = path.join(
directory,
`${path.basename(this.memoryFilePath)}.${randomBytes(16).toString("hex")}.tmp`,
);
try {
// Create the temp file with the existing mode, so the graph is never
// readable through it more widely than through the file it replaces;
// the umask can only narrow that. The chmod then restores any bits the
// umask removed.
await fs.writeFile(
tempFilePath,
content,
mode === undefined ? undefined : { mode },
);
if (mode !== undefined) {
await fs.chmod(tempFilePath, mode);
}
await fs.rename(tempFilePath, this.memoryFilePath);
} catch (error) {
// Never leave a stray temp file behind on failure.
await fs.unlink(tempFilePath).catch(() => {});
throw error;
}
}
async createEntities(entities: Entity[]): Promise<Entity[]> {
return this.withLock(async () => {
const { graph, unreadable, size } = await this.loadGraphWithUnreadable();
const newEntities = entities.filter(
(e, index) =>
!graph.entities.some(
(existingEntity) => existingEntity.name === e.name,
) &&
// Also skip duplicates appearing earlier in this same batch
!entities.slice(0, index).some((earlier) => earlier.name === e.name),
);
graph.entities.push(...newEntities);
// Only a call that adds something is held to the limits: re-saving can
// grow an existing file that was not written in this exact form.
await this.saveGraph(
graph,
unreadable,
newEntities.length > 0 ? size : undefined,
);
return newEntities;
});
}
async createRelations(relations: Relation[]): Promise<Relation[]> {
return this.withLock(async () => {
const { graph, unreadable, size } = await this.loadGraphWithUnreadable();
const entityNames = new Set(graph.entities.map((e) => e.name));
relations.forEach((r) => {
if (!entityNames.has(r.from)) {
throw new Error(`Entity with name ${r.from} not found`);
}
if (!entityNames.has(r.to)) {
throw new Error(`Entity with name ${r.to} not found`);
}
});
const isSameRelation = (a: Relation, b: Relation) =>
a.from === b.from && a.to === b.to && a.relationType === b.relationType;
const newRelations = relations.filter(
(r, index) =>
!graph.relations.some((existingRelation) =>
isSameRelation(existingRelation, r),
) &&
// Also skip duplicates appearing earlier in this same batch
!relations
.slice(0, index)
.some((earlier) => isSameRelation(earlier, r)),
);
graph.relations.push(...newRelations);
// Held to the limits only when it adds something (see createEntities).
await this.saveGraph(
graph,
unreadable,
newRelations.length > 0 ? size : undefined,
);
return newRelations;
});
}
async addObservations(
observations: { entityName: string; contents: string[] }[],
): Promise<{ entityName: string; addedObservations: string[] }[]> {
return this.withLock(async () => {
const { graph, unreadable, size } = await this.loadGraphWithUnreadable();
const results = observations.map((o) => {
const entity = graph.entities.find((e) => e.name === o.entityName);
if (!entity) {
throw new Error(`Entity with name ${o.entityName} not found`);
}
// A Set keeps the first occurrence of each string, so a repeat within
// one call is stored and reported once (#5201).
const newObservations = [...new Set(o.contents)].filter(
(content) => !entity.observations.includes(content),
);
entity.observations.push(...newObservations);
return { entityName: o.entityName, addedObservations: newObservations };
});
// Held to the limits only when it adds something (see createEntities).
const added = results.some((r) => r.addedObservations.length > 0);
await this.saveGraph(graph, unreadable, added ? size : undefined);
return results;
});
}
async deleteEntities(
entityNames: string[],
): Promise<{ deleted: string[]; notFound: string[] }> {
return this.withLock(async () => {
const { graph, unreadable } = await this.loadGraphWithUnreadable();
const present = new Set(graph.entities.map((e) => e.name));
const deleted = entityNames.filter((name) => present.has(name));
const notFound = entityNames.filter((name) => !present.has(name));
graph.entities = graph.entities.filter(
(e) => !entityNames.includes(e.name),
);
graph.relations = graph.relations.filter(
(r) => !entityNames.includes(r.from) && !entityNames.includes(r.to),
);
await this.saveGraph(graph, unreadable);
return { deleted, notFound };
});
}
async deleteObservations(
deletions: { entityName: string; observations: string[] }[],
): Promise<{ deletedCount: number; missingEntities: string[] }> {
return this.withLock(async () => {
const { graph, unreadable } = await this.loadGraphWithUnreadable();
let deletedCount = 0;
const missingEntities: string[] = [];
deletions.forEach((d) => {
const entity = graph.entities.find((e) => e.name === d.entityName);
if (entity) {
const before = entity.observations.length;
entity.observations = entity.observations.filter(
(o) => !d.observations.includes(o),
);