Skip to content

Commit 9ebb273

Browse files
dramaticllyclaude
andcommitted
Spark: Auto-set default name mapping in SparkTableUtil.importSparkTable
After restoring `NameMapping.empty()` as the safe default in `Parquet.ReadBuilder` (commit 8881516), Spark tests that import external Parquet files via `SparkTableUtil.importSparkTable` and then read them through Iceberg fail field resolution: the imported files have no Iceberg field IDs and no `schema.name-mapping.default` property is auto-set, so the strict empty mapping returns null for every projected field. Mirror the established pattern from `AddFilesProcedure.ensureNameMappingPresent` and `BaseTableCreationSparkAction`: when the target Iceberg table does not already have `schema.name-mapping.default` set, auto-derive a name mapping from the target schema before importing. Subsequent reads via Iceberg pick up the property automatically through `BaseReader`. Applied identically to v3.5, v4.0, and v4.1. This honors the deprecation guidance Russell Spitzer documented when deprecating the NETFLIX_UNSAFE positional fallback ("Use name mapping instead") and brings `importSparkTable` in line with `AddFilesProcedure`, which has been doing this for some time. Verified on Spark 4.1 via the full `org.apache.iceberg.spark.source.*` test package (`-DtestParallelism=8`); previously-failing `TestIcebergSourceHadoopTables.testTableWithInt96Timestamp` and `TestIdentityPartitionData.testProjections` now pass with no collateral failures elsewhere. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
1 parent 8da5062 commit 9ebb273

3 files changed

Lines changed: 48 additions & 0 deletions

File tree

‎spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
import org.apache.iceberg.hadoop.Util;
6868
import org.apache.iceberg.io.FileIO;
6969
import org.apache.iceberg.io.OutputFile;
70+
import org.apache.iceberg.mapping.MappingUtil;
7071
import org.apache.iceberg.mapping.NameMapping;
7172
import org.apache.iceberg.mapping.NameMappingParser;
7273
import org.apache.iceberg.relocated.com.google.common.base.Joiner;
@@ -582,6 +583,8 @@ public static void importSparkTable(
582583
"Table %s does not exist", sourceTableIdentWithDB);
583584
}
584585

586+
ensureNameMappingPresent(targetTable);
587+
585588
try {
586589
PartitionSpec spec =
587590
findCompatibleSpec(targetTable, spark, sourceTableIdentWithDB.unquotedString());
@@ -659,6 +662,19 @@ public static void importSparkTable(
659662
spark, sourceTableIdent, targetTable, stagingDir, Collections.emptyMap(), false, 1);
660663
}
661664

665+
/**
666+
* Auto-set the default name mapping on the target table if it is not already set, so that
667+
* imported files without Iceberg field IDs resolve fields by name rather than by unsafe position.
668+
* Mirrors {@code AddFilesProcedure#ensureNameMappingPresent}.
669+
*/
670+
private static void ensureNameMappingPresent(Table table) {
671+
if (table.properties().get(TableProperties.DEFAULT_NAME_MAPPING) == null) {
672+
NameMapping mapping = MappingUtil.create(table.schema());
673+
String mappingJson = NameMappingParser.toJson(mapping);
674+
table.updateProperties().set(TableProperties.DEFAULT_NAME_MAPPING, mappingJson).commit();
675+
}
676+
}
677+
662678
private static void importUnpartitionedSparkTable(
663679
SparkSession spark,
664680
TableIdentifier sourceTableIdent,

‎spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
import org.apache.iceberg.hadoop.Util;
6868
import org.apache.iceberg.io.FileIO;
6969
import org.apache.iceberg.io.OutputFile;
70+
import org.apache.iceberg.mapping.MappingUtil;
7071
import org.apache.iceberg.mapping.NameMapping;
7172
import org.apache.iceberg.mapping.NameMappingParser;
7273
import org.apache.iceberg.relocated.com.google.common.base.Joiner;
@@ -379,6 +380,8 @@ public static void importSparkTable(
379380
"Table %s does not exist", sourceTableIdentWithDB);
380381
}
381382

383+
ensureNameMappingPresent(targetTable);
384+
382385
try {
383386
PartitionSpec spec =
384387
findCompatibleSpec(targetTable, spark, sourceTableIdentWithDB.unquotedString());
@@ -428,6 +431,19 @@ public static void importSparkTable(
428431
spark, sourceTableIdent, targetTable, stagingDir, Collections.emptyMap(), false, 1);
429432
}
430433

434+
/**
435+
* Auto-set the default name mapping on the target table if it is not already set, so that
436+
* imported files without Iceberg field IDs resolve fields by name rather than by unsafe position.
437+
* Mirrors {@code AddFilesProcedure#ensureNameMappingPresent}.
438+
*/
439+
private static void ensureNameMappingPresent(Table table) {
440+
if (table.properties().get(TableProperties.DEFAULT_NAME_MAPPING) == null) {
441+
NameMapping mapping = MappingUtil.create(table.schema());
442+
String mappingJson = NameMappingParser.toJson(mapping);
443+
table.updateProperties().set(TableProperties.DEFAULT_NAME_MAPPING, mappingJson).commit();
444+
}
445+
}
446+
431447
private static void importUnpartitionedSparkTable(
432448
SparkSession spark,
433449
TableIdentifier sourceTableIdent,

‎spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@
6666
import org.apache.iceberg.hadoop.Util;
6767
import org.apache.iceberg.io.FileIO;
6868
import org.apache.iceberg.io.OutputFile;
69+
import org.apache.iceberg.mapping.MappingUtil;
6970
import org.apache.iceberg.mapping.NameMapping;
7071
import org.apache.iceberg.mapping.NameMappingParser;
7172
import org.apache.iceberg.relocated.com.google.common.base.Joiner;
@@ -378,6 +379,8 @@ public static void importSparkTable(
378379
"Table %s does not exist", sourceTableIdentWithDB);
379380
}
380381

382+
ensureNameMappingPresent(targetTable);
383+
381384
try {
382385
PartitionSpec spec =
383386
findCompatibleSpec(targetTable, spark, sourceTableIdentWithDB.unquotedString());
@@ -427,6 +430,19 @@ public static void importSparkTable(
427430
spark, sourceTableIdent, targetTable, stagingDir, Collections.emptyMap(), false, 1);
428431
}
429432

433+
/**
434+
* Auto-set the default name mapping on the target table if it is not already set, so that
435+
* imported files without Iceberg field IDs resolve fields by name rather than by unsafe position.
436+
* Mirrors {@code AddFilesProcedure#ensureNameMappingPresent}.
437+
*/
438+
private static void ensureNameMappingPresent(Table table) {
439+
if (table.properties().get(TableProperties.DEFAULT_NAME_MAPPING) == null) {
440+
NameMapping mapping = MappingUtil.create(table.schema());
441+
String mappingJson = NameMappingParser.toJson(mapping);
442+
table.updateProperties().set(TableProperties.DEFAULT_NAME_MAPPING, mappingJson).commit();
443+
}
444+
}
445+
430446
private static void importUnpartitionedSparkTable(
431447
SparkSession spark,
432448
TableIdentifier sourceTableIdent,

0 commit comments

Comments
 (0)