Repository navigation
Expand file tree
/
Copy pathprobe.py
More file actions
77 lines (59 loc) · 2.91 KB
/
Copy pathprobe.py
File metadata and controls
77 lines (59 loc) · 2.91 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
"""Direct reproduction: Memory MCP accepts referentially-inconsistent relations.
Runs the published server (`npx -y @modelcontextprotocol/server-memory`) over
stdio and shows that `create_relations` silently accepts relations between
entities that were never created, leaving a dangling relation in the graph —
while `add_observations` correctly rejects the same missing entity.
python3 probe.py # prints evidence, exits 0 if the bug reproduces
This is the server-side ground truth behind the llmcontract finding: the
documented "entities before relations" workflow is not enforced.
"""
from __future__ import annotations
import asyncio
import json
import os
import sys
from mcp import ClientSession, StdioServerParameters
from mcp.client.stdio import stdio_client
def _text(res) -> str:
return "\n".join(getattr(b, "text", "") or "" for b in (res.content or []))
async def _run() -> bool:
params = StdioServerParameters(
command="npx",
args=["-y", "@modelcontextprotocol/server-memory"],
env={"MEMORY_FILE_PATH": "./memory_probe.json", "PATH": os.environ["PATH"]},
)
async with stdio_client(params) as (r, w):
async with ClientSession(r, w) as s:
await s.initialize()
rel = await s.call_tool("create_relations", {"relations": [
{"from": "Ghost_A", "to": "Ghost_B", "relationType": "knows"}]})
print("create_relations(nonexistent) -> isError:", rel.isError)
print(" returned:", _text(rel).replace("\n", " ")[:200])
obs = await s.call_tool("add_observations", {"observations": [
{"entityName": "Ghost_C", "contents": ["floats"]}]})
print("add_observations(nonexistent) -> isError:", obs.isError)
print(" returned:", _text(obs).replace("\n", " ")[:200])
graph = json.loads(_text(await s.call_tool("read_graph", {})))
print("read_graph -> entities:", len(graph.get("entities", [])),
"| relations:", len(graph.get("relations", [])))
dangling = [
rr for rr in graph.get("relations", [])
if rr.get("from") not in {e.get("name") for e in graph.get("entities", [])}
or rr.get("to") not in {e.get("name") for e in graph.get("entities", [])}
]
bug = (not rel.isError) and bool(dangling) and bool(obs.isError)
print()
print("BUG REPRODUCED:" if bug else "not reproduced:",
f"relations accepted={not rel.isError}, dangling in graph={bool(dangling)}, "
f"observations correctly rejected={bool(obs.isError)}")
return bug
def main() -> int:
# Clean any prior state so the run is self-contained.
try:
os.remove("./memory_probe.json")
except FileNotFoundError:
pass
ok = asyncio.run(_run())
return 0 if ok else 1
if __name__ == "__main__":
sys.exit(main())