Skip to content

Commit 01605a8

Browse files
Manos PalaiologosManos Palaiologos
authored andcommitted
bypass known bug of pyiceberg and pyarrow with MAP (apache/arrow#38553)
1 parent 6bff357 commit 01605a8

3 files changed

Lines changed: 74 additions & 15 deletions

File tree

‎kestra/flows/main_makerates.rates_daily.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
id: rates_daily2
1+
id: rates_daily
22
namespace: makerates
33

44
description: |

‎src/iceberg_loader.py‎

Lines changed: 71 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -217,14 +217,16 @@ def get_arrow_batch(self, source_pattern: str) -> Optional[pa.Table]:
217217

218218
# Build CASE expressions to create structs only for non-NULL values with non-empty keys
219219
struct_list = ", ".join([
220-
f"CASE WHEN {col} IS NOT NULL THEN "
220+
f"CASE WHEN {col} IS NOT NULL AND {col} IS NOT NaN THEN "
221221
f"{{key: '{key}', value: CAST({col} AS DOUBLE)}} "
222222
f"END"
223223
for col, key in valid_pairs
224224
])
225225

226-
# Use list_filter to remove NULL struct entries, then map_from_entries
227-
rates_map_expr = f"map_from_entries(list_filter([{struct_list}], x -> x IS NOT NULL)) AS rates"
226+
# Use list_filter to remove NULL struct entries AND validate keys are not NULL/empty
227+
# COALESCE with empty MAP to handle edge cases on subsequent runs
228+
# This prevents Arrow validation errors from NULL keys
229+
rates_map_expr = f"COALESCE(map_from_entries(list_filter([{struct_list}], x -> x IS NOT NULL AND x.key IS NOT NULL AND x.key != '')), MAP([], [])::MAP(VARCHAR, DOUBLE)) AS rates"
228230
logger.debug(f"MAP expression (first 200 chars): {rates_map_expr[:200]}...")
229231
else:
230232
# Fallback: empty MAP if no rates columns found
@@ -288,6 +290,32 @@ def get_arrow_batch(self, source_pattern: str) -> Optional[pa.Table]:
288290

289291
logger.info(f"Executing DuckDB Query with Deduplication on keys: {self.pk_list} based on extraction_timestamp")
290292

293+
# CRITICAL: Validate MAP has no NULL keys before Arrow conversion (prevents idempotency issues)
294+
validation_query = f"""
295+
SELECT COUNT(*) as invalid_count
296+
FROM ({final_query}) validated
297+
WHERE EXISTS (
298+
SELECT 1 FROM unnest(map_keys(validated.rates)) AS k(key)
299+
WHERE k.key IS NULL OR k.key = ''
300+
)
301+
"""
302+
try:
303+
invalid_result = self.con.sql(validation_query).fetchone()
304+
if invalid_result and invalid_result[0] > 0:
305+
logger.error(f"❌ CRITICAL: Found {invalid_result[0]} rows with NULL or empty MAP keys!")
306+
logger.error("This will cause Arrow validation errors. Filtering out invalid rows...")
307+
# Add filter to exclude rows with invalid MAP keys
308+
final_query = f"""
309+
SELECT * FROM ({final_query}) validated
310+
WHERE NOT EXISTS (
311+
SELECT 1 FROM unnest(map_keys(validated.rates)) AS k(key)
312+
WHERE k.key IS NULL OR k.key = ''
313+
)
314+
"""
315+
logger.info("✅ Applied filter to exclude invalid MAP entries")
316+
except Exception as e:
317+
logger.warning(f"MAP validation query failed (might be empty result): {e}")
318+
291319
try:
292320
arrow_result = self.con.sql(final_query).arrow()
293321

@@ -403,13 +431,47 @@ def load_to_iceberg(self, arrow_table: pa.Table):
403431
logger.error(f"Schema: {arrow_table.schema}")
404432
raise
405433

406-
logger.info(f"Upserting {arrow_table.num_rows} rows using keys: {self.pk_list}...")
434+
# CRITICAL FIX: PyIceberg's upsert() has a bug with MAP types that causes
435+
# "Map array keys array should have no nulls" on subsequent runs
436+
# https://github.com/apache/arrow/issues/38553 related issue on pyarrow + MAP bug
437+
438+
# We use delete+append strategy for idempotency instead of buggy merge upsert
439+
logger.info(f"Processing {arrow_table.num_rows} rows using delete+append (idempotent, MAP-safe)...")
440+
407441
try:
408-
upsert_result = table.upsert(df=arrow_table, join_cols=self.pk_list)
409-
logger.info("Upsert complete.")
410-
logger.info("\nUpsert operation completed")
411-
logger.info(f"Rows Updated: {upsert_result.rows_updated}")
412-
logger.info(f"Rows Inserted: {upsert_result.rows_inserted}")
442+
# Extract unique combinations of primary key values
443+
pk_values_set = set()
444+
for i in range(arrow_table.num_rows):
445+
pk_tuple = tuple(arrow_table.column(pk)[i].as_py() for pk in self.pk_list)
446+
pk_values_set.add(pk_tuple)
447+
448+
# Build delete predicate: (rate_date='2024-01-01' AND source='x' AND base_currency='USD') OR ...
449+
delete_predicates = []
450+
for pk_values in pk_values_set:
451+
row_predicates = []
452+
for pk, value in zip(self.pk_list, pk_values):
453+
if isinstance(value, str):
454+
row_predicates.append(f"{pk} = '{value}'")
455+
else:
456+
row_predicates.append(f"{pk} = {value}")
457+
delete_predicates.append(f"({' AND '.join(row_predicates)})")
458+
459+
# Delete existing rows with matching keys (idempotency)
460+
if delete_predicates:
461+
combined_predicate = " OR ".join(delete_predicates)
462+
logger.info(f"Deleting {len(pk_values_set)} existing row(s) with matching keys...")
463+
logger.debug(f"Delete predicate: {combined_predicate[:300]}...")
464+
table.delete(combined_predicate)
465+
logger.info("✅ Deleted existing rows")
466+
467+
# Append new data (no merge, avoids MAP bug)
468+
logger.info(f"Appending {arrow_table.num_rows} rows...")
469+
table.append(arrow_table)
470+
logger.info(f"✅ Successfully inserted {arrow_table.num_rows} rows")
471+
472+
except Exception as e:
473+
logger.error(f"Delete+Append strategy failed: {e}")
474+
raise
413475
except Exception as e:
414476
if "Map array keys array should have no nulls" in str(e):
415477
logger.error("═" * 80)

‎src/sync_gold_to_dynamodb.py‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,10 @@
1313
from decimal import Decimal
1414
from typing import Dict, Any
1515
from pathlib import Path
16-
from utils.helpers import load_config
16+
from utils.helpers import load_config,parse_env_vars_config
1717
from utils.dynamodb import DynamoDBClient
18-
from utils.helpers import parse_env_vars_config
18+
from util.logging_config import root_logger as logger
1919

20-
# Setup Logging
21-
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
22-
logger = logging.getLogger(__name__)
2320
# ENV VARS FOR JOB CONFIG
2421
JOB_CONFIG_FILE = "settings.yaml"
2522

0 commit comments

Comments
 (0)