Skip to content

Commit b17c618

Browse files
nik13claude
andauthored
fix(parallel): protect Responses privacy and interrupted workflows (#562)
OpenAI Responses: preserve MCP results in mixed text+MCP outputs (streaming and non-streaming), with the Parallel framework repair. Independently verified earlier; combined #571+#562 integration proof QUALIFIED at this exact tree (6503256, 4,740 passed, 0 failed, source core 1.2.3). Missing GitHub approval overridden by the repository admin's authorization; current-head CI and source gates enforced. No package publication implied. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent d7e5555 commit b17c618

29 files changed

Lines changed: 3737 additions & 342 deletions

‎.github/workflows/parallel-python-tests.yml‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ jobs:
3737
distribution: wheel
3838
- python: '3.13'
3939
parallel: '1.3.7'
40-
openai: '3.42.0'
40+
openai: '3.45.0'
4141
distribution: sdist
4242
env:
4343
FI_BASE_URL: http://127.0.0.1:9
@@ -66,7 +66,13 @@ jobs:
6666
packages=(python/dist/*.tar.gz python/frameworks/{parallel,openai}/dist/*.tar.gz)
6767
fi
6868
.parallel-test-venv/bin/pip install "${packages[@]}" \
69-
"parallel-web==$PARALLEL_VERSION" "openai==$OPENAI_VERSION" pytest pytest-asyncio
69+
"parallel-web==$PARALLEL_VERSION" "openai==$OPENAI_VERSION" \
70+
./python/frameworks/{deepseek,fireworks,vllm} pytest pytest-asyncio
71+
- name: Prevent source-tree imports from shadowing installed distributions
72+
shell: bash
73+
run: |
74+
mv python/frameworks/openai/traceai_openai "$RUNNER_TEMP/openai-source"
75+
mv python/frameworks/parallel/traceai_parallel "$RUNNER_TEMP/parallel-source"
7076
- name: Test with only loopback networking
7177
shell: bash
7278
run: |

‎python/examples/perplexity/tests/test_perplexity_recipe.py‎

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,19 @@
3737
== Path(CORE_DISTRIBUTION.locate_file("fi_instrumentation/__init__.py")).resolve()
3838
)
3939

40+
# The recipe CI deliberately retains the old published SDK as a compatibility
41+
# control. Only that exact installed distribution exports echoed instructions;
42+
# current source must not reintroduce them to satisfy legacy expectations.
43+
try:
44+
OPENAI_DISTRIBUTION = distribution("traceAI-openai")
45+
except PackageNotFoundError:
46+
LEGACY_OUTPUT = False
47+
else:
48+
LEGACY_OUTPUT = OPENAI_DISTRIBUTION.version == "0.1.10" and (
49+
Path(traceai_openai.__file__).resolve()
50+
== Path(OPENAI_DISTRIBUTION.locate_file("traceai_openai/__init__.py")).resolve()
51+
)
52+
4053
TEST_DIR = Path(__file__).resolve().parent
4154
RECIPE_DIR = TEST_DIR.parent
4255
VENDOR_KEY = "placeholder-perplexity-key"
@@ -254,7 +267,10 @@ def test_responses_at_documented_host(tracing):
254267
assert raw["usage"]["input_tokens_details"][key] == value
255268
assert raw["usage"]["output_tokens_details"] == fake.USAGE["output_tokens_details"]
256269
assert raw["usage"]["tool_calls_details"] == fake.USAGE["tool_calls_details"]
257-
assert fake.INSTRUCTIONS_MARKER in raw["instructions"]
270+
if LEGACY_OUTPUT:
271+
assert fake.INSTRUCTIONS_MARKER in raw["instructions"]
272+
else:
273+
assert "instructions" not in raw
258274
assert fake.SEARCH_URL in attrs["output.value"]
259275
assert fake.SEARCH_SNIPPET in attrs["output.value"]
260276

@@ -349,9 +365,11 @@ def handler(request):
349365
assert _call(handler, prompt=marker) == fake.ANSWER
350366
span, _ = _assert_export(receiver, provider, project, hide_inputs=hidden)
351367
assert (marker in json.dumps(span)) is (not hidden)
352-
# Output echoes remain visible even when the input itself is hidden.
353-
for output_marker in (fake.QUERY_MARKER, fake.INSTRUCTIONS_MARKER):
354-
assert output_marker in _attributes(span)["output.value"]
368+
# Search queries are response-owned output; instructions are input.
369+
output = _attributes(span)["output.value"]
370+
assert fake.QUERY_MARKER in output
371+
assert (fake.INSTRUCTIONS_MARKER in output) is LEGACY_OUTPUT
372+
assert fake.ANSWER in output
355373

356374

357375
@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"])
@@ -363,8 +381,11 @@ def test_hide_outputs_with_control(tracing, monkeypatch, stream):
363381
span, attrs = _assert_export(receiver, provider, project)
364382
_assert_usage(attrs)
365383
assert (fake.ANSWER in json.dumps(span)) is (not hidden)
366-
for marker in (fake.SEARCH_URL, fake.SEARCH_SNIPPET, fake.QUERY_MARKER, fake.INSTRUCTIONS_MARKER):
384+
for marker in (fake.SEARCH_URL, fake.SEARCH_SNIPPET, fake.QUERY_MARKER):
367385
assert (marker in json.dumps(span)) is (not hidden and not stream)
386+
assert (fake.INSTRUCTIONS_MARKER in json.dumps(span)) is (
387+
LEGACY_OUTPUT and not hidden and not stream
388+
)
368389

369390

370391
@pytest.mark.parametrize("hidden", [False, True], ids=["visible", "hidden"])
@@ -381,8 +402,9 @@ def test_empty_text_stream_raw_fallback(tracing, monkeypatch, hidden):
381402
list(client.responses.create(model=fake.REQUEST_MODEL, input="empty stream", stream=True))
382403
span, attrs = _assert_export(receiver, provider, project)
383404
_assert_usage(attrs)
384-
for marker in (fake.SEARCH_URL, fake.QUERY_MARKER, fake.INSTRUCTIONS_MARKER):
405+
for marker in (fake.SEARCH_URL, fake.QUERY_MARKER):
385406
assert (marker in json.dumps(span)) is (not hidden)
407+
assert (fake.INSTRUCTIONS_MARKER in json.dumps(span)) is (LEGACY_OUTPUT and not hidden)
386408

387409

388410
def test_cache_details_require_output_details(tracing):
@@ -457,7 +479,7 @@ def test_subprocess_app(tmp_path, stream, cli_model):
457479
assert (fake.SEARCH_URL in json.dumps(attrs)) is (not stream)
458480
assert (fake.SEARCH_SNIPPET in json.dumps(attrs)) is (not stream)
459481
assert (fake.QUERY_MARKER in json.dumps(attrs)) is (not stream)
460-
assert (fake.INSTRUCTIONS_MARKER in json.dumps(attrs)) is (not stream)
482+
assert (fake.INSTRUCTIONS_MARKER in json.dumps(attrs)) is (LEGACY_OUTPUT and not stream)
461483
[export] = receiver.requests()
462484
assert export["path"] == "/tracer/v1/traces"
463485
assert export["headers"]["x-api-key"] == FI_KEY
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
"""Synthetic fixture to verify declared core compatibility and hidden error details."""
2+
import asyncio
3+
import httpx
4+
import openai
5+
import pytest
6+
from fi_instrumentation import TraceConfig, register
7+
from fi_instrumentation.fi_types import ProjectType
8+
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
9+
from traceai_openai import OpenAIInstrumentor
10+
11+
MESSAGE = 'synthetic-private-upstream-error-483129'
12+
13+
@pytest.mark.parametrize('async_', [False, True])
14+
def test_hidden_failed_response(async_, monkeypatch):
15+
monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', 'true')
16+
exporter = InMemorySpanExporter()
17+
provider = register(project_name='core-floor-probe', project_type=ProjectType.OBSERVE,
18+
span_exporter=exporter, batch=False, verbose=False)
19+
instrumentor = OpenAIInstrumentor()
20+
instrumentor.instrument(tracer_provider=provider, config=TraceConfig(hide_inputs=True))
21+
body = {'id': 'resp_probe', 'object': 'response', 'created_at': 1, 'model': 'synthetic',
22+
'status': 'failed', 'output': [], 'error': {'code': 'server_error', 'message': MESSAGE},
23+
'usage': {'input_tokens': 1, 'output_tokens': 0, 'total_tokens': 1}}
24+
def handler(req): return httpx.Response(200, json=body)
25+
def make_client():
26+
http = (httpx.AsyncClient if async_ else httpx.Client)(transport=httpx.MockTransport(handler), trust_env=False)
27+
return (openai.AsyncOpenAI if async_ else openai.OpenAI)(api_key='synthetic-key', base_url='https://api.parallel.ai/v1', max_retries=0, http_client=http)
28+
try:
29+
if async_:
30+
async def run():
31+
async with make_client() as client:
32+
assert (await client.responses.create(model='synthetic', input='private input')).status == 'failed'
33+
asyncio.run(run())
34+
else:
35+
with make_client() as client:
36+
assert client.responses.create(model='synthetic', input='private input').status == 'failed'
37+
spans = exporter.get_finished_spans()
38+
assert len(spans) == 1
39+
assert MESSAGE not in spans[0].to_json()
40+
finally:
41+
instrumentor.uninstrument()
42+
provider.shutdown()

‎python/frameworks/openai/tests/test_deepseek_raw_response.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -134,9 +134,9 @@ async def run():
134134
assert b"PRIVATE_STREAM_ERROR" not in spans[0].SerializeToString()
135135

136136

137-
@pytest.mark.parametrize("asynchronous", [False, True])
137+
@pytest.mark.parametrize("asynchronous,close_method", [(False, "close"), (True, "close"), (True, "aclose")])
138138
@pytest.mark.parametrize("parse_first", [False, True])
139-
def test_raw_stream_close_before_exhaustion(wire, asynchronous, parse_first):
139+
def test_raw_stream_close_before_exhaustion(wire, asynchronous, parse_first, close_method):
140140
spans = wire("shared")
141141
params = dict(
142142
model="deepseek-reasoner",
@@ -155,8 +155,8 @@ async def run():
155155
if parse_first:
156156
stream = raw.parse()
157157
await stream.__anext__()
158-
await raw.aclose()
159-
await raw.aclose()
158+
await getattr(raw, close_method)()
159+
await getattr(raw, close_method)()
160160

161161
if asynchronous:
162162
asyncio.run(run())

‎python/frameworks/openai/tests/test_deepseek_route.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -555,7 +555,8 @@ def respond(request):
555555
if streaming:
556556
assert a["output.value"] == ANSWER
557557
assert a["gen_ai.output.messages.0.message.content"] == ANSWER
558-
assert spans[0].status.code == (2 if event_type == "response.failed" else 1)
558+
# A terminal incomplete response is not a successful generation either.
559+
assert spans[0].status.code == (1 if event_type == "response.completed" else 2)
559560
assert b"PRIVATE_VENDOR_ERROR" not in spans[0].SerializeToString()
560561

561562

0 commit comments

Comments
 (0)