Skip to content

Commit 5d175af

Browse files
committed
test(examples): keep placeholder keys out of TanStack AI spans and output
Review asked for proof that FI_API_KEY, FI_SECRET_KEY and OPENAI_API_KEY never reach an exported span, stdout or stderr. The contract test now checks every run for the three placeholder keys: the shipped route, the failed model call (which prints the provider error), the refused and the silent collector (which print exporter errors) and the README snippet. A positive control first shows the keys were in use: x-secret-key on every export and "Bearer placeholder-openai-key" on every model call (the fake now records Authorization; the fixture keeps it). The recipe already keeps the keys out, so these pass on c6001eb by design. Mutations show they are load-bearing, each reverted with git checkout: the enricher copying FI_SECRET_KEY onto spans fails 2 tests ('placeholder-fi-secret-key', 'spans'), and the CLI error line printing OPENAI_API_KEY fails test_failed_model_call_exports_error_spans ('placeholder-openai-key', 'stderr'). approved: Nikhil 2026-10-03 blanket Refs: TH-8238
1 parent 98f7444 commit 5d175af

1 file changed

Lines changed: 37 additions & 3 deletions

File tree

‎typescript/examples/tanstack-ai/tests/test_tanstack_ai_contract.py‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@
5151

5252
API_KEY = "placeholder-fi-api-key"
5353
SECRET_KEY = "placeholder-fi-secret-key"
54+
OPENAI_KEY = "placeholder-openai-key"
55+
PLACEHOLDER_KEYS = (API_KEY, SECRET_KEY, OPENAI_KEY)
5456
PROJECT = "th-8238-contract"
5557
MODEL = "gpt-4o-mini"
5658

@@ -76,7 +78,7 @@ def _env(fi_base_url: str, openai_base_url: str) -> dict[str, str]:
7678
"FI_API_KEY": API_KEY,
7779
"FI_SECRET_KEY": SECRET_KEY,
7880
"FI_PROJECT_NAME": PROJECT,
79-
"OPENAI_API_KEY": "placeholder-openai-key",
81+
"OPENAI_API_KEY": OPENAI_KEY,
8082
"OPENAI_BASE_URL": openai_base_url,
8183
"OPENAI_MODEL": MODEL,
8284
}
@@ -121,6 +123,18 @@ def _is_error(status: dict[str, Any]) -> bool:
121123
return status.get("code") in (2, "STATUS_CODE_ERROR")
122124

123125

126+
def _assert_no_placeholder_keys(spans: list[dict[str, Any]], result: Any) -> None:
127+
"""No FI or OpenAI key reaches an exported span, stdout or stderr."""
128+
surfaces = {
129+
"spans": json.dumps(spans),
130+
"stdout": result.stdout.decode("utf-8", "replace"),
131+
"stderr": result.stderr.decode("utf-8", "replace"),
132+
}
133+
for surface, text in surfaces.items():
134+
for key in PLACEHOLDER_KEYS:
135+
assert key not in text, (key, surface)
136+
137+
124138
@pytest.fixture(scope="module")
125139
def default_run():
126140
"""One run of the example exactly as shipped (captureContent unset),
@@ -134,6 +148,7 @@ def default_run():
134148
spans=receiver.spans(),
135149
exports=receiver.requests(),
136150
model_requests=list(fake.requests),
151+
model_authorizations=list(fake.authorizations),
137152
)
138153

139154

@@ -249,6 +264,18 @@ def test_no_prompt_or_response_content_by_default(default_run):
249264
assert not span.get("events"), span["name"]
250265

251266

267+
def test_placeholder_keys_stay_out_of_spans_and_output(default_run):
268+
# Positive control: the keys were in use, as the export headers and as
269+
# the model call's bearer token.
270+
assert default_run.exports
271+
for export in default_run.exports:
272+
assert export["headers"]["x-secret-key"] == SECRET_KEY
273+
assert default_run.model_authorizations
274+
for authorization in default_run.model_authorizations:
275+
assert authorization == "Bearer " + OPENAI_KEY
276+
_assert_no_placeholder_keys(default_run.spans, default_run.result)
277+
278+
252279
def test_export_has_auth_headers_project_resource_and_collector_path(default_run):
253280
exports = default_run.exports
254281
assert exports
@@ -293,6 +320,7 @@ def test_unreachable_collector_does_not_fail_chat():
293320
assert result.returncode == 0, result.stderr.decode()
294321
assert ANSWER in result.stdout.decode()
295322
assert "[futureagi] span export failed" in result.stderr.decode()
323+
_assert_no_placeholder_keys([], result)
296324

297325

298326
@contextlib.contextmanager
@@ -331,6 +359,7 @@ def test_silent_collector_does_not_hold_the_response():
331359
route_ms = int(re.search(r"routeMs=(\d+)", stdout).group(1))
332360
assert route_ms < 3000, route_ms
333361
assert "[futureagi] span export still pending" in result.stderr.decode()
362+
_assert_no_placeholder_keys([], result)
334363

335364

336365
def test_failed_model_call_exports_error_spans():
@@ -342,7 +371,8 @@ def test_failed_model_call_exports_error_spans():
342371
"""
343372
with Receiver() as receiver, FakeOpenAI(CITY, ANSWER, fail_status=500) as fake:
344373
result = _node(EXAMPLE / "src" / "chat.mjs", receiver.origin, fake)
345-
spans = _by_name(receiver.spans())
374+
raw_spans = receiver.spans()
375+
spans = _by_name(raw_spans)
346376
exports = receiver.requests()
347377
model_requests = list(fake.requests)
348378

@@ -365,6 +395,8 @@ def test_failed_model_call_exports_error_spans():
365395
# No thread id was passed, so no session (chat() generated its own id).
366396
for name in (ROOT, ITERATION_0):
367397
assert "session.id" not in _attributes(spans[name]), name
398+
# The provider error is printed; the credentials are not.
399+
_assert_no_placeholder_keys(raw_spans, result)
368400

369401

370402
def test_abort_mid_stream_ends_spans_as_cancelled():
@@ -421,7 +453,8 @@ def test_readme_recipe_runs_and_exports_the_contract_spans():
421453
try:
422454
with Receiver() as receiver, FakeOpenAI(CITY, ANSWER) as fake:
423455
result = _node(script, receiver.origin, fake, THREAD_ID)
424-
spans = _by_name(receiver.spans())
456+
raw_spans = receiver.spans()
457+
spans = _by_name(raw_spans)
425458
finally:
426459
script.unlink()
427460
assert result.returncode == 0, result.stderr.decode()
@@ -438,3 +471,4 @@ def test_readme_recipe_runs_and_exports_the_contract_spans():
438471
for name, span in spans.items():
439472
assert _attributes(span)["session.id"] == THREAD_ID, name
440473
assert PROMPT not in json.dumps(list(spans.values()))
474+
_assert_no_placeholder_keys(raw_spans, result)

0 commit comments

Comments
 (0)