Skip to content

Commit 1d7866f

Browse files
authored
feat: use json-repair for LLM output parsing (MemMachine#954)
* feat: use json-repair for malformed LLM JSON output recovery - Add json-repair library to handle common LLM JSON formatting errors - Update language model implementations (OpenAI, Bedrock) to use json-repair - Fix unit tests to validate JSON repair instead of expecting errors - Use real MemMachine examples in tests (project creation, memory search) - Remove unused json imports after json-repair integration This change improves robustness when handling LLM outputs with malformed JSON (unquoted keys, single quotes, trailing commas, etc.) * fix: resolve ruff linting issues - Fix import sorting (I001) in 4 files - Sort imports alphabetically and place json_repair correctly - evaluation/locomo/episodic_agent/llm_judge.py - evaluation/locomo/episodic_memory/llm_judge.py - src/memmachine/common/language_model/openai_chat_completions_language_model.py - src/memmachine/common/language_model/openai_responses_language_model.py - Fix code style issues in test files - Remove whitespace from blank lines (W293) - Replace single quotes with double quotes in multiline strings (Q001) - Rename unused unpacked variables to _content (RUF059) - tests/memmachine/common/language_model/test_openai_chat_completions_language_model.py - tests/memmachine/common/language_model/test_openai_responses_language_model.py All ruff checks now pass successfully. * chore: add json-repair dependency to uv.lock Add json-repair v0.55.0 to lock file for LLM output parsing.
1 parent 34d75ab commit 1d7866f

10 files changed

Lines changed: 97 additions & 28 deletions

File tree

‎evaluation/locomo/episodic_agent/llm_judge.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from pathlib import Path
99
from typing import Any
1010

11+
import json_repair
1112
import numpy as np
1213
from dotenv import load_dotenv
1314
from openai import OpenAI
@@ -60,7 +61,7 @@ def evaluate_llm_judge(question: str, gold_answer: str, generated_answer: str) -
6061
response_format={"type": "json_object"},
6162
temperature=0.0,
6263
)
63-
label = json.loads(response.choices[0].message.content)["label"]
64+
label = json_repair.loads(response.choices[0].message.content)["label"]
6465
return 1 if label == "CORRECT" else 0
6566

6667

‎evaluation/locomo/episodic_memory/llm_judge.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import json
66
from collections import defaultdict
77

8+
import json_repair
89
import numpy as np
910
from dotenv import load_dotenv
1011
from openai import OpenAI
@@ -56,7 +57,7 @@ def evaluate_llm_judge(question, gold_answer, generated_answer) -> int:
5657
response_format={"type": "json_object"},
5758
temperature=0.0,
5859
)
59-
label = json.loads(response.choices[0].message.content)["label"]
60+
label = json_repair.loads(response.choices[0].message.content)["label"]
6061
return 1 if label == "CORRECT" else 0
6162

6263

‎packages/server/pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ dependencies = [
1515
"fastapi>=0.116.1",
1616
"fastmcp>=2.14.0",
1717
"instructor[bedrock]>=1.12.0",
18+
"json-repair>=0.54",
1819
"langchain-aws>=0.2.33",
1920
"neo4j>=5.28.2",
2021
"nltk>=3.9.1",

‎pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ dependencies = [
2424
"fastapi>=0.116.1",
2525
"fastmcp>=2.14.0",
2626
"instructor[bedrock]>=1.12.0",
27+
"json-repair>=0.54",
2728
"langchain-aws>=0.2.33",
2829
"neo4j>=5.28.2",
2930
"nltk>=3.9.1",

‎src/memmachine/common/language_model/amazon_bedrock_language_model.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
"""Amazon Bedrock-based language model implementation."""
22

33
import asyncio
4-
import json
54
import logging
65
import re
76
import time
@@ -10,6 +9,7 @@
109
from uuid import uuid4
1110

1211
import instructor
12+
import json_repair
1313
from pydantic import BaseModel, Field, InstanceOf, TypeAdapter
1414

1515
from memmachine.common.data_types import ExternalServiceAPIError
@@ -493,7 +493,7 @@ def _try_parse_bedrock_completion(
493493
return None
494494

495495
try:
496-
parsed_json = json.loads(json_match.group(0))
496+
parsed_json = json_repair.loads(json_match.group(0))
497497
return TypeAdapter(output_format).validate_python(parsed_json)
498498
except Exception:
499499
return None

‎src/memmachine/common/language_model/openai_chat_completions_language_model.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
"""OpenAI-completions API based language model implementation."""
22

33
import asyncio
4-
import json
54
import logging
65
import time
76
from typing import Any, TypeVar
87
from uuid import uuid4
98

9+
import json_repair
1010
import openai
1111
from openai.types.chat import ChatCompletion, ChatCompletionMessageFunctionToolCall
1212
from pydantic import BaseModel, Field, InstanceOf, TypeAdapter
@@ -259,7 +259,7 @@ async def generate_response( # noqa: C901
259259
"call_id": tool_call.id,
260260
"function": {
261261
"name": tool_call.function.name,
262-
"arguments": json.loads(
262+
"arguments": json_repair.loads(
263263
tool_call.function.arguments,
264264
),
265265
},
@@ -270,8 +270,10 @@ async def generate_response( # noqa: C901
270270
"Unsupported tool call type: %s",
271271
type(tool_call).__name__,
272272
)
273-
except json.JSONDecodeError as e:
274-
raise ValueError("JSON decode error") from e
273+
except (TypeError, ValueError) as e:
274+
raise ValueError(
275+
"Failed to repair or parse JSON from function call arguments"
276+
) from e
275277

276278
return (
277279
response.choices[0].message.content or "",

‎src/memmachine/common/language_model/openai_responses_language_model.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
"""OpenAI-based language model implementation."""
22

33
import asyncio
4-
import json
54
import logging
65
import time
76
from typing import Any, TypeVar
87
from uuid import uuid4
98

9+
import json_repair
1010
import openai
1111
from openai.types.responses import Response
1212
from pydantic import BaseModel, Field, InstanceOf
@@ -278,14 +278,16 @@ async def generate_response(
278278
"call_id": output.call_id,
279279
"function": {
280280
"name": output.name,
281-
"arguments": json.loads(output.arguments),
281+
"arguments": json_repair.loads(output.arguments),
282282
},
283283
}
284284
for output in response.output
285285
if output.type == "function_call"
286286
]
287-
except json.JSONDecodeError as e:
288-
raise ValueError("JSON decode error") from e
287+
except (TypeError, ValueError) as e:
288+
raise ValueError(
289+
"Failed to repair or parse JSON from function call arguments"
290+
) from e
289291

290292
return (
291293
response.output_text,

‎tests/memmachine/common/language_model/test_openai_chat_completions_language_model.py‎

Lines changed: 36 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -261,16 +261,33 @@ async def test_generate_response_with_tool_calls(
261261

262262

263263
@pytest.mark.asyncio
264-
async def test_generate_response_tool_call_json_error(
264+
async def test_generate_response_tool_call_json_repair(
265265
mock_async_openai,
266266
minimal_config,
267267
mock_tool_call_impl,
268268
):
269-
"""Test handling of invalid JSON in tool call arguments."""
269+
"""Test that json-repair successfully fixes invalid JSON in tool call arguments.
270+
271+
This test uses a MemMachine memory search example with multiple JSON errors
272+
that commonly occur in LLM outputs to verify json-repair functionality.
273+
"""
270274
mock_tool_call = mock_tool_call_impl()
271-
mock_tool_call.uuid = "call_123"
272-
mock_tool_call.function.name = "get_weather"
273-
mock_tool_call.function.arguments = '{"location": "Boston",}'
275+
mock_tool_call.id = "call_123" # Use id instead of uuid
276+
mock_tool_call.function.name = "search_memory"
277+
# Invalid JSON with multiple common LLM errors:
278+
# - Missing quotes around keys
279+
# - Trailing comma in array
280+
# - Comments in JSON (not standard)
281+
# - Missing closing brace (will be fixed)
282+
mock_tool_call.function.arguments = """{
283+
query: "user preferences and history",
284+
"top_k": 10,
285+
"types": ["episodic", "semantic",],
286+
"filter": {
287+
user_id: "user_123",
288+
"session_active": true,
289+
},
290+
}"""
274291

275292
mock_response = MagicMock()
276293
mock_response.choices[0].message.tool_calls = [mock_tool_call]
@@ -280,8 +297,20 @@ async def test_generate_response_tool_call_json_error(
280297
mock_client.chat.completions.create.return_value = mock_response
281298

282299
lm = OpenAIChatCompletionsLanguageModel(minimal_config)
283-
with pytest.raises(ValueError, match="JSON decode error"):
284-
await lm.generate_response()
300+
# json-repair should automatically fix the invalid JSON
301+
_content, tool_calls = await lm.generate_response()
302+
303+
# Verify the repaired JSON is parsed correctly
304+
assert len(tool_calls) == 1
305+
assert tool_calls[0]["call_id"] == "call_123"
306+
assert tool_calls[0]["function"]["name"] == "search_memory"
307+
# Verify the complex JSON structure was repaired and parsed correctly
308+
args = tool_calls[0]["function"]["arguments"]
309+
assert args["query"] == "user preferences and history"
310+
assert args["top_k"] == 10
311+
assert args["types"] == ["episodic", "semantic"]
312+
assert args["filter"]["user_id"] == "user_123"
313+
assert args["filter"]["session_active"] is True
285314

286315

287316
@pytest.mark.asyncio

‎tests/memmachine/common/language_model/test_openai_responses_language_model.py‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -236,16 +236,23 @@ async def test_generate_response_with_tool_calls(mock_async_openai, minimal_conf
236236

237237

238238
@pytest.mark.asyncio
239-
async def test_generate_response_tool_call_json_error(
239+
async def test_generate_response_tool_call_json_repair(
240240
mock_async_openai,
241241
minimal_config,
242242
):
243-
"""Test handling of invalid JSON in tool call arguments."""
243+
"""Test that json-repair successfully fixes invalid JSON in tool call arguments."""
244244
mock_tool_call = MagicMock()
245245
mock_tool_call.type = "function_call"
246246
mock_tool_call.call_id = "call_123"
247-
mock_tool_call.name = "get_weather"
248-
mock_tool_call.arguments = '{"location": "Boston",}' # Invalid JSON
247+
mock_tool_call.name = "create_memory_project"
248+
mock_tool_call.arguments = """{
249+
project: "MemMachine",
250+
'description': 'Open-source memory layer for AI agents',
251+
"memory_types": ["episodic", "semantic",],
252+
"features": {
253+
"persistent": true,
254+
"cross_session": true,
255+
"""
249256

250257
mock_response = MagicMock()
251258
mock_response.output_text = None
@@ -256,11 +263,23 @@ async def test_generate_response_tool_call_json_error(
256263
mock_client.responses.create.return_value = mock_response
257264

258265
lm = OpenAIResponsesLanguageModel(minimal_config)
259-
with pytest.raises(ValueError, match="JSON decode error"):
260-
await lm.generate_response(
261-
system_prompt="System prompt",
262-
user_prompt="User prompt",
263-
)
266+
# json-repair should automatically fix the invalid JSON
267+
_content, tool_calls = await lm.generate_response(
268+
system_prompt="System prompt",
269+
user_prompt="User prompt",
270+
)
271+
272+
# Verify the repaired JSON is parsed correctly
273+
assert len(tool_calls) == 1
274+
assert tool_calls[0]["call_id"] == "call_123"
275+
assert tool_calls[0]["function"]["name"] == "create_memory_project"
276+
# Verify the complex JSON structure was repaired and parsed correctly
277+
args = tool_calls[0]["function"]["arguments"]
278+
assert args["project"] == "MemMachine"
279+
assert args["description"] == "Open-source memory layer for AI agents"
280+
assert args["memory_types"] == ["episodic", "semantic"]
281+
assert args["features"]["persistent"] is True
282+
assert args["features"]["cross_session"] is True
264283

265284

266285
@pytest.mark.asyncio

‎uv.lock‎

Lines changed: 13 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)