You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
{{ message }}
Repository navigation
Commit 01605a8
Browse filesBrowse the repository at this point in the historyBrowse files
# Build CASE expressions to create structs only for non-NULL values with non-empty keys
219
219
struct_list=", ".join([
220
-
f"CASE WHEN {col} IS NOT NULL THEN "
220
+
f"CASE WHEN {col} IS NOT NULL AND {col} IS NOT NaN THEN "
221
221
f"{{key: '{key}', value: CAST({col} AS DOUBLE)}} "
222
222
f"END"
223
223
forcol, keyinvalid_pairs
224
224
])
225
225
226
-
# Use list_filter to remove NULL struct entries, then map_from_entries
227
-
rates_map_expr=f"map_from_entries(list_filter([{struct_list}], x -> x IS NOT NULL)) AS rates"
226
+
# Use list_filter to remove NULL struct entries AND validate keys are not NULL/empty
227
+
# COALESCE with empty MAP to handle edge cases on subsequent runs
228
+
# This prevents Arrow validation errors from NULL keys
229
+
rates_map_expr=f"COALESCE(map_from_entries(list_filter([{struct_list}], x -> x IS NOT NULL AND x.key IS NOT NULL AND x.key != '')), MAP([], [])::MAP(VARCHAR, DOUBLE)) AS rates"
0 commit comments