Skip to content

Commit 21cb098

Browse files
committed
fix: Redeploy Snowflake UDFs on every apply so runtime version changes take effect
Addresses maintainer feedback from @ntkathole on feast-dev#6608 (feast-dev#6608 (comment)): once materialization UDFs were deployed for a project, SnowflakeComputeEngine.update() short-circuited on a `SHOW USER FUNCTIONS` check, and the SQL template used `CREATE FUNCTION IF NOT EXISTS`. Together, this meant an already-deployed project's UDFs were never redeployed, so changing `python_udf_runtime_version` (e.g. after Snowflake decommissions a runtime, as in feast-dev#6606) silently left the stale runtime version in place. Removes the early-return guard and switches every UDF creation statement in snowflake_python_udfs_creation.sql from `CREATE FUNCTION IF NOT EXISTS` to `CREATE OR REPLACE FUNCTION`, so `update()` always (re)deploys the UDFs using whatever runtime version is currently configured. Adds unit tests covering that update() no longer short-circuits and that a redeploy always uses the current python_udf_runtime_version. Signed-off-by: zerafachris <[email protected]>
1 parent a327346 commit 21cb098

4 files changed

Lines changed: 144 additions & 34 deletions

File tree

‎sdk/python/feast/infra/compute_engines/snowflake/snowflake_engine.py‎

Lines changed: 11 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -144,25 +144,18 @@ def update(
144144
entities_to_delete: Sequence[Entity],
145145
entities_to_keep: Sequence[Entity],
146146
):
147-
stage_context = f'"{self.repo_config.batch_engine.database}"."{self.repo_config.batch_engine.schema_}"'
148-
stage_path = f'{stage_context}."feast_{project}"'
147+
stage_path = f'"{self.repo_config.batch_engine.database}"."{self.repo_config.batch_engine.schema_}"."feast_{project}"'
149148
with GetSnowflakeConnection(self.repo_config.batch_engine) as conn:
150-
query = f"SHOW USER FUNCTIONS LIKE 'FEAST_{project.upper()}%' IN SCHEMA {stage_context}"
151-
cursor = execute_snowflake_statement(conn, query)
152-
function_list = pd.DataFrame(
153-
cursor.fetchall(),
154-
columns=[column.name for column in cursor.description],
155-
)
156-
157-
# if the SHOW FUNCTIONS query returns results,
158-
# assumes that the materialization functions have been deployed
159-
if len(function_list.index) > 0:
160-
click.echo(
161-
f"Materialization functions for {Style.BRIGHT + Fore.GREEN}{project}{Style.RESET_ALL} already detected."
162-
)
163-
click.echo()
164-
return None
165-
149+
# Always (re)deploy the materialization functions, using
150+
# `CREATE OR REPLACE FUNCTION` below, instead of skipping deployment
151+
# when functions already exist. Previously, an early return here
152+
# (triggered by a `SHOW USER FUNCTIONS` check) combined with
153+
# `CREATE FUNCTION IF NOT EXISTS` in the SQL template meant that
154+
# once UDFs were deployed for a project, they were never
155+
# redeployed -- so a config change to `python_udf_runtime_version`
156+
# (e.g. after Snowflake decommissions a runtime) would silently
157+
# keep using the stale, already-deployed runtime version. See
158+
# https://github.com/feast-dev/feast/pull/6608#discussion_r3602461959.
166159
click.echo(
167160
f"Deploying materialization functions for {Style.BRIGHT + Fore.GREEN}{project}{Style.RESET_ALL}"
168161
)

‎sdk/python/feast/infra/utils/snowflake/snowpark/snowflake_python_udfs_creation.sql‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,124 +1,124 @@
1-
CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_binary_to_bytes_proto(df BINARY)
1+
CREATE OR REPLACE FUNCTION feast_PROJECT_NAME_snowflake_binary_to_bytes_proto(df BINARY)
22
RETURNS BINARY
33
LANGUAGE PYTHON
44
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
55
PACKAGES = ('protobuf', 'pandas')
66
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_binary_to_bytes_proto'
77
IMPORTS = ('@STAGE_HOLDER/feast.zip');
88

9-
CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_varchar_to_string_proto(df VARCHAR)
9+
CREATE OR REPLACE FUNCTION feast_PROJECT_NAME_snowflake_varchar_to_string_proto(df VARCHAR)
1010
RETURNS BINARY
1111
LANGUAGE PYTHON
1212
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
1313
PACKAGES = ('protobuf', 'pandas')
1414
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_varchar_to_string_proto'
1515
IMPORTS = ('@STAGE_HOLDER/feast.zip');
1616

17-
CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_bytes_to_list_bytes_proto(df ARRAY)
17+
CREATE OR REPLACE FUNCTION feast_PROJECT_NAME_snowflake_array_bytes_to_list_bytes_proto(df ARRAY)
1818
RETURNS BINARY
1919
LANGUAGE PYTHON
2020
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
2121
PACKAGES = ('protobuf', 'pandas')
2222
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_bytes_to_list_bytes_proto'
2323
IMPORTS = ('@STAGE_HOLDER/feast.zip');
2424

25-
CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_varchar_to_list_string_proto(df ARRAY)
25+
CREATE OR REPLACE FUNCTION feast_PROJECT_NAME_snowflake_array_varchar_to_list_string_proto(df ARRAY)
2626
RETURNS BINARY
2727
LANGUAGE PYTHON
2828
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
2929
PACKAGES = ('protobuf', 'pandas')
3030
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_varchar_to_list_string_proto'
3131
IMPORTS = ('@STAGE_HOLDER/feast.zip');
3232

33-
CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_number_to_list_int32_proto(df ARRAY)
33+
CREATE OR REPLACE FUNCTION feast_PROJECT_NAME_snowflake_array_number_to_list_int32_proto(df ARRAY)
3434
RETURNS BINARY
3535
LANGUAGE PYTHON
3636
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
3737
PACKAGES = ('protobuf', 'pandas')
3838
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_number_to_list_int32_proto'
3939
IMPORTS = ('@STAGE_HOLDER/feast.zip');
4040

41-
CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_number_to_list_int64_proto(df ARRAY)
41+
CREATE OR REPLACE FUNCTION feast_PROJECT_NAME_snowflake_array_number_to_list_int64_proto(df ARRAY)
4242
RETURNS BINARY
4343
LANGUAGE PYTHON
4444
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
4545
PACKAGES = ('protobuf', 'pandas')
4646
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_number_to_list_int64_proto'
4747
IMPORTS = ('@STAGE_HOLDER/feast.zip');
4848

49-
CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_float_to_list_double_proto(df ARRAY)
49+
CREATE OR REPLACE FUNCTION feast_PROJECT_NAME_snowflake_array_float_to_list_double_proto(df ARRAY)
5050
RETURNS BINARY
5151
LANGUAGE PYTHON
5252
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
5353
PACKAGES = ('protobuf', 'pandas')
5454
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_float_to_list_double_proto'
5555
IMPORTS = ('@STAGE_HOLDER/feast.zip');
5656

57-
CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_boolean_to_list_bool_proto(df ARRAY)
57+
CREATE OR REPLACE FUNCTION feast_PROJECT_NAME_snowflake_array_boolean_to_list_bool_proto(df ARRAY)
5858
RETURNS BINARY
5959
LANGUAGE PYTHON
6060
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
6161
PACKAGES = ('protobuf', 'pandas')
6262
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_boolean_to_list_bool_proto'
6363
IMPORTS = ('@STAGE_HOLDER/feast.zip');
6464

65-
CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_timestamp_to_list_unix_timestamp_proto(df ARRAY)
65+
CREATE OR REPLACE FUNCTION feast_PROJECT_NAME_snowflake_array_timestamp_to_list_unix_timestamp_proto(df ARRAY)
6666
RETURNS BINARY
6767
LANGUAGE PYTHON
6868
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
6969
PACKAGES = ('protobuf', 'pandas')
7070
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_timestamp_to_list_unix_timestamp_proto'
7171
IMPORTS = ('@STAGE_HOLDER/feast.zip');
7272

73-
CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_number_to_int32_proto(df NUMBER)
73+
CREATE OR REPLACE FUNCTION feast_PROJECT_NAME_snowflake_number_to_int32_proto(df NUMBER)
7474
RETURNS BINARY
7575
LANGUAGE PYTHON
7676
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
7777
PACKAGES = ('protobuf', 'pandas')
7878
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_number_to_int32_proto'
7979
IMPORTS = ('@STAGE_HOLDER/feast.zip');
8080

81-
CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_number_to_int64_proto(df NUMBER)
81+
CREATE OR REPLACE FUNCTION feast_PROJECT_NAME_snowflake_number_to_int64_proto(df NUMBER)
8282
RETURNS BINARY
8383
LANGUAGE PYTHON
8484
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
8585
PACKAGES = ('protobuf', 'pandas')
8686
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_number_to_int64_proto'
8787
IMPORTS = ('@STAGE_HOLDER/feast.zip');
8888

89-
CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_float_to_double_proto(df DOUBLE)
89+
CREATE OR REPLACE FUNCTION feast_PROJECT_NAME_snowflake_float_to_double_proto(df DOUBLE)
9090
RETURNS BINARY
9191
LANGUAGE PYTHON
9292
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
9393
PACKAGES = ('protobuf', 'pandas')
9494
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_float_to_double_proto'
9595
IMPORTS = ('@STAGE_HOLDER/feast.zip');
9696

97-
CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_boolean_to_bool_proto(df BOOLEAN)
97+
CREATE OR REPLACE FUNCTION feast_PROJECT_NAME_snowflake_boolean_to_bool_proto(df BOOLEAN)
9898
RETURNS BINARY
9999
LANGUAGE PYTHON
100100
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
101101
PACKAGES = ('protobuf', 'pandas')
102102
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_boolean_to_bool_boolean_proto'
103103
IMPORTS = ('@STAGE_HOLDER/feast.zip');
104104

105-
CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_timestamp_to_unix_timestamp_proto(df NUMBER)
105+
CREATE OR REPLACE FUNCTION feast_PROJECT_NAME_snowflake_timestamp_to_unix_timestamp_proto(df NUMBER)
106106
RETURNS BINARY
107107
LANGUAGE PYTHON
108108
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
109109
PACKAGES = ('protobuf', 'pandas')
110110
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_timestamp_to_unix_timestamp_proto'
111111
IMPORTS = ('@STAGE_HOLDER/feast.zip');
112112

113-
CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_serialize_entity_keys(names ARRAY, data ARRAY, types ARRAY)
113+
CREATE OR REPLACE FUNCTION feast_PROJECT_NAME_serialize_entity_keys(names ARRAY, data ARRAY, types ARRAY)
114114
RETURNS BINARY
115115
LANGUAGE PYTHON
116116
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'
117117
PACKAGES = ('protobuf', 'pandas')
118118
HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_serialize_entity_keys'
119119
IMPORTS = ('@STAGE_HOLDER/feast.zip');
120120

121-
CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_entity_key_proto_to_string(names ARRAY, data ARRAY, types ARRAY)
121+
CREATE OR REPLACE FUNCTION feast_PROJECT_NAME_entity_key_proto_to_string(names ARRAY, data ARRAY, types ARRAY)
122122
RETURNS BINARY
123123
LANGUAGE PYTHON
124124
RUNTIME_VERSION = 'RUNTIME_VERSION_HOLDER'

‎sdk/python/tests/unit/infra/compute_engines/snowflake/__init__.py‎

Whitespace-only changes.
Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
"""Regression tests for SnowflakeComputeEngine.update().
2+
3+
See https://github.com/feast-dev/feast/pull/6608#discussion_r3602461959: a
4+
maintainer pointed out that once materialization UDFs were deployed for a
5+
project, a `SHOW USER FUNCTIONS` check made `update()` return early on every
6+
subsequent call, and the SQL template used `CREATE FUNCTION IF NOT EXISTS`.
7+
Together, this meant a user changing `python_udf_runtime_version` (e.g. after
8+
Snowflake decommissions a runtime) would never actually get the new runtime
9+
version deployed -- the stale, already-deployed UDFs were left untouched.
10+
11+
The fix removes the early return and switches the SQL template to
12+
`CREATE OR REPLACE FUNCTION`, so `update()` always (re)deploys the UDFs using
13+
the currently configured runtime version.
14+
"""
15+
16+
import shutil
17+
import tempfile
18+
from unittest.mock import MagicMock, patch
19+
20+
from feast.infra.compute_engines.snowflake.snowflake_engine import (
21+
SnowflakeComputeEngine,
22+
SnowflakeComputeEngineConfig,
23+
)
24+
25+
MODULE = "feast.infra.compute_engines.snowflake.snowflake_engine"
26+
27+
28+
def _make_engine(python_udf_runtime_version: str = "3.10") -> SnowflakeComputeEngine:
29+
repo_config = MagicMock()
30+
repo_config.offline_store.type = "snowflake.offline"
31+
repo_config.batch_engine = SnowflakeComputeEngineConfig(
32+
database="FEAST",
33+
schema="PUBLIC",
34+
python_udf_runtime_version=python_udf_runtime_version,
35+
)
36+
37+
return SnowflakeComputeEngine(
38+
repo_config=repo_config,
39+
offline_store=MagicMock(),
40+
online_store=MagicMock(),
41+
)
42+
43+
44+
def _run_update(engine: SnowflakeComputeEngine, mock_execute: MagicMock) -> None:
45+
mock_conn_cm = MagicMock()
46+
mock_conn_cm.__enter__ = MagicMock(return_value=MagicMock())
47+
mock_conn_cm.__exit__ = MagicMock(return_value=False)
48+
49+
def _fake_package_snowpark_zip(project_name):
50+
copy_path = tempfile.mkdtemp()
51+
return copy_path, f"{copy_path}/feast.zip"
52+
53+
with (
54+
patch(f"{MODULE}.GetSnowflakeConnection", return_value=mock_conn_cm),
55+
patch(f"{MODULE}.execute_snowflake_statement", mock_execute),
56+
patch(f"{MODULE}.package_snowpark_zip", side_effect=_fake_package_snowpark_zip),
57+
patch(f"{MODULE}.shutil.rmtree", side_effect=shutil.rmtree),
58+
):
59+
engine.update(
60+
project="myproject",
61+
views_to_delete=[],
62+
views_to_keep=[],
63+
entities_to_delete=[],
64+
entities_to_keep=[],
65+
)
66+
67+
68+
def test_update_never_short_circuits_on_existing_functions():
69+
"""update() must not query `SHOW USER FUNCTIONS` and skip deployment --
70+
that early-return path is what caused stale runtime versions to stick
71+
around after a config change.
72+
"""
73+
engine = _make_engine()
74+
mock_execute = MagicMock()
75+
76+
_run_update(engine, mock_execute)
77+
78+
executed_queries = [call.args[1] for call in mock_execute.call_args_list]
79+
assert not any("SHOW USER FUNCTIONS" in q for q in executed_queries), (
80+
"update() should no longer perform a SHOW USER FUNCTIONS check that "
81+
"causes it to skip (re)deploying the UDFs"
82+
)
83+
assert any("CREATE OR REPLACE FUNCTION" in q for q in executed_queries), (
84+
"update() should deploy UDFs using CREATE OR REPLACE FUNCTION so "
85+
"redeploys pick up a changed python_udf_runtime_version"
86+
)
87+
88+
89+
def test_update_redeploys_with_current_runtime_version_on_every_call():
90+
"""Calling update() twice (simulating two `feast apply` runs, e.g. before
91+
and after a user bumps `python_udf_runtime_version`) must deploy the UDFs
92+
both times, each time using whatever runtime version is currently
93+
configured.
94+
"""
95+
engine = _make_engine(python_udf_runtime_version="3.11")
96+
mock_execute = MagicMock()
97+
98+
_run_update(engine, mock_execute)
99+
first_call_count = mock_execute.call_count
100+
assert first_call_count > 0
101+
102+
_run_update(engine, mock_execute)
103+
second_call_count = mock_execute.call_count
104+
assert second_call_count == 2 * first_call_count, (
105+
"the second update() call should re-run the full deployment (no "
106+
"early return), just like the first"
107+
)
108+
109+
executed_queries = [call.args[1] for call in mock_execute.call_args_list]
110+
create_function_queries = [
111+
q for q in executed_queries if "CREATE OR REPLACE FUNCTION" in q
112+
]
113+
assert create_function_queries, "expected CREATE OR REPLACE FUNCTION queries"
114+
assert all("RUNTIME_VERSION = '3.11'" in q for q in create_function_queries), (
115+
"every deployed UDF should use the currently configured "
116+
"python_udf_runtime_version ('3.11'), including on redeploy"
117+
)

0 commit comments

Comments
 (0)