Skip to content

Commit f61079f

Browse files
Daksha1611shuchu
andauthored
fix: Fall back to the serialized body when UDF source misses globals (feast-dev#6816)
Source-first rehydration execs only the UDF's own source into a namespace seeded with pandas/numpy. A UDF that reads a helper, constant, or aliased import from its defining module has no way to resolve those names, but the exec still succeeds, because a function body's free variables are only looked up when it is called. resolve_udf therefore treated rehydration as successful and never fell back to the dill body, even when that body carried the captured globals and would have run. The failure surfaced later as a NameError during retrieval, so an on-demand feature view that served correctly before started raising once it was loaded from the registry. Validate the rebuilt callable before accepting it: walk its LOAD_GLOBAL operands, and the operands of nested code objects, and return None when a name resolves in neither the exec namespace nor builtins. resolve_udf then falls back as it did before. Only LOAD_GLOBAL is inspected. co_names also holds attribute names, so checking it would flag working UDFs and push them onto the dill path for no reason. Self-contained UDFs are unaffected and still take the source path, keeping the Spark and cross-Python-version benefits intact. Signed-off-by: Daksha1611 <[email protected]> Co-authored-by: Shuchu Han <[email protected]>
1 parent f216396 commit f61079f

2 files changed

Lines changed: 127 additions & 3 deletions

File tree

‎sdk/python/feast/transformation/udf_rehydrate.py‎

Lines changed: 64 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,11 @@
99

1010
from __future__ import annotations
1111

12+
import builtins
13+
import dis
1214
import logging
13-
from typing import Callable, Optional
15+
from types import CodeType
16+
from typing import Callable, Optional, Set
1417

1518
import dill
1619

@@ -119,6 +122,46 @@ def _exec_namespace() -> dict:
119122
return ns
120123

121124

125+
def _unresolved_global_names(func: Callable, namespace: dict) -> Set[str]:
126+
"""Global names *func* loads that neither *namespace* nor builtins provide.
127+
128+
``exec``-ing a function definition only binds the function; the globals its
129+
body reads are looked up when it is *called*. Source text alone therefore
130+
cannot tell us whether the callable actually works — a UDF referencing a
131+
helper or constant from its defining module execs cleanly and then raises
132+
``NameError`` mid-retrieval.
133+
134+
Only ``LOAD_GLOBAL`` operands count. ``co_names`` also holds attribute names
135+
(``df.columns``), which would flag working UDFs and push them onto the dill
136+
path for no reason. Nested code objects (inner functions, comprehensions)
137+
are walked too.
138+
"""
139+
missing: Set[str] = set()
140+
code = getattr(func, "__code__", None)
141+
if code is None:
142+
return missing
143+
144+
seen: Set[int] = set()
145+
pending = [code]
146+
while pending:
147+
current = pending.pop()
148+
if id(current) in seen:
149+
continue
150+
seen.add(id(current))
151+
for instruction in dis.get_instructions(current):
152+
if instruction.opname != "LOAD_GLOBAL":
153+
continue
154+
name = instruction.argval
155+
if not isinstance(name, str):
156+
continue
157+
if name not in namespace and not hasattr(builtins, name):
158+
missing.add(name)
159+
for const in current.co_consts:
160+
if isinstance(const, CodeType):
161+
pending.append(const)
162+
return missing
163+
164+
122165
def rehydrate_udf_from_source(
123166
udf_string: str,
124167
*,
@@ -141,7 +184,7 @@ def rehydrate_udf_from_source(
141184
return None
142185

143186
if preferred_name and preferred_name in ns and callable(ns[preferred_name]):
144-
return ns[preferred_name]
187+
return _accept_if_self_contained(ns[preferred_name], ns)
145188

146189
for value in ns.values():
147190
if not callable(value):
@@ -152,11 +195,29 @@ def rehydrate_udf_from_source(
152195
# Skip imported modules / classes we seeded
153196
if name in ("DataFrame",):
154197
continue
155-
return value
198+
return _accept_if_self_contained(value, ns)
156199

157200
return None
158201

159202

203+
def _accept_if_self_contained(func: Callable, ns: dict) -> Optional[Callable]:
204+
"""Return *func* only if every global it reads is available, else ``None``.
205+
206+
Returning ``None`` lets :func:`resolve_udf` fall back to the dill body, which
207+
carries the UDF's captured globals and can still run.
208+
"""
209+
missing = _unresolved_global_names(func, ns)
210+
if missing:
211+
logger.debug(
212+
"udf source rehydrate skipped for %s: unresolved global names %s; "
213+
"falling back to the serialized body",
214+
getattr(func, "__name__", func),
215+
sorted(missing),
216+
)
217+
return None
218+
return func
219+
220+
160221
def resolve_udf(
161222
*,
162223
udf_string: str = "",

‎sdk/python/tests/unit/transformation/test_udf_rehydrate.py‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,3 +92,66 @@ def test_strip_leading_decorators_adversarial_at_spam_is_linear():
9292
# rehydrate should fail closed so dill fallback can run.
9393
assert rehydrate_udf_from_source(spam) is None
9494
assert isinstance(stripped, str)
95+
96+
97+
_SCALE = 10
98+
99+
100+
def _scale(value):
101+
return value * _SCALE
102+
103+
104+
def _udf_using_module_globals(df: pd.DataFrame) -> pd.DataFrame:
105+
out = pd.DataFrame()
106+
out["scaled"] = _scale(df["x"])
107+
return out
108+
109+
110+
_MODULE_GLOBALS_SRC = """def _udf_using_module_globals(df):
111+
out = __import__("pandas").DataFrame()
112+
out["scaled"] = _scale(df["x"])
113+
return out
114+
"""
115+
116+
117+
def test_rehydrate_rejects_udf_with_unresolved_globals():
118+
"""Source alone cannot supply names the UDF reads from its defining module."""
119+
assert (
120+
rehydrate_udf_from_source(
121+
_MODULE_GLOBALS_SRC, preferred_name="_udf_using_module_globals"
122+
)
123+
is None
124+
)
125+
126+
127+
def test_resolve_udf_falls_back_to_dill_when_source_misses_globals():
128+
"""The serialized body carries the captured globals, so it must win here."""
129+
udf = resolve_udf(
130+
udf_string=_MODULE_GLOBALS_SRC,
131+
body=dill.dumps(_udf_using_module_globals, recurse=True),
132+
preferred_name="_udf_using_module_globals",
133+
)
134+
result = udf(pd.DataFrame({"x": [1, 2]}))
135+
assert list(result["scaled"]) == [10, 20]
136+
137+
138+
def test_attribute_access_is_not_mistaken_for_a_missing_global():
139+
"""Attribute names live in co_names too; flagging them would strand good UDFs."""
140+
src = """def _attr_udf(df):
141+
out = __import__("pandas").DataFrame()
142+
out["cols"] = len(df.columns)
143+
out["upper"] = df["s"].str.upper()
144+
return out
145+
"""
146+
fn = rehydrate_udf_from_source(src, preferred_name="_attr_udf")
147+
assert fn is not None
148+
result = fn(pd.DataFrame({"s": ["a"]}))
149+
assert result["upper"].iloc[0] == "A"
150+
151+
152+
def test_nested_scopes_are_inspected_for_missing_globals():
153+
"""A comprehension or inner function has its own code object."""
154+
src = """def _nested_udf(df):
155+
return [_scale(v) for v in df["x"]]
156+
"""
157+
assert rehydrate_udf_from_source(src, preferred_name="_nested_udf") is None

0 commit comments

Comments
 (0)