Skip to content

Commit 773907d

Browse files
authored
Core: Add id tracking to MetricsConfig (apache#17022)
1 parent 34b03b7 commit 773907d

27 files changed

Lines changed: 252 additions & 140 deletions

File tree

‎.palantir/revapi.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -564,6 +564,9 @@ acceptedBreaks:
564564
- code: "java.field.removedWithConstant"
565565
old: "field org.apache.iceberg.PartitionStatsHandler.PARTITION_FIELD_NAME"
566566
justification: "Removed deprecated functionality for partition stats"
567+
- code: "java.method.removed"
568+
old: "method org.apache.iceberg.MetricsConfig org.apache.iceberg.MetricsConfig::forPositionDelete(org.apache.iceberg.Table)"
569+
justification: "Deprecated for removal in 1.12.0"
567570
- code: "java.method.removed"
568571
old: "method org.apache.iceberg.Schema org.apache.iceberg.PartitionStatsHandler::schema(org.apache.iceberg.types.Types.StructType,\
569572
\ int)"

‎core/src/main/java/org/apache/iceberg/MetricsConfig.java‎

Lines changed: 81 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
import javax.annotation.concurrent.Immutable;
3333
import org.apache.iceberg.MetricsModes.MetricsMode;
3434
import org.apache.iceberg.exceptions.ValidationException;
35-
import org.apache.iceberg.relocated.com.google.common.base.Joiner;
35+
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
3636
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
3737
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
3838
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
@@ -49,27 +49,37 @@
4949
public final class MetricsConfig implements Serializable {
5050

5151
private static final Logger LOG = LoggerFactory.getLogger(MetricsConfig.class);
52-
private static final Joiner DOT = Joiner.on('.');
5352

5453
// Disable metrics by default for wide tables to prevent excessive metadata
5554
private static final MetricsMode DEFAULT_MODE =
5655
MetricsModes.fromString(DEFAULT_WRITE_METRICS_MODE_DEFAULT);
57-
private static final MetricsConfig DEFAULT = new MetricsConfig(ImmutableMap.of(), DEFAULT_MODE);
56+
private static final MetricsConfig DEFAULT =
57+
new MetricsConfig(ImmutableMap.of(), DEFAULT_MODE, ImmutableMap.of());
5858
private static final MetricsConfig POSITION_DELETE_MODE =
5959
new MetricsConfig(
6060
ImmutableMap.of(
6161
MetadataColumns.DELETE_FILE_PATH.name(),
6262
MetricsModes.Full.get(),
6363
MetadataColumns.DELETE_FILE_POS.name(),
6464
MetricsModes.Full.get()),
65-
DEFAULT_MODE);
65+
MetricsModes.None.get(),
66+
ImmutableMap.of(
67+
MetadataColumns.DELETE_FILE_PATH.fieldId(),
68+
MetadataColumns.DELETE_FILE_PATH.name(),
69+
MetadataColumns.DELETE_FILE_POS.fieldId(),
70+
MetadataColumns.DELETE_FILE_POS.name()));
6671

6772
private final Map<String, MetricsMode> columnModes;
6873
private final MetricsMode defaultMode;
74+
private final Map<Integer, String> idToName;
6975

70-
private MetricsConfig(Map<String, MetricsMode> columnModes, MetricsMode defaultMode) {
76+
private MetricsConfig(
77+
Map<String, MetricsMode> columnModes,
78+
MetricsMode defaultMode,
79+
Map<Integer, String> idToName) {
7180
this.columnModes = SerializableMap.copyOf(columnModes).immutableMap();
7281
this.defaultMode = defaultMode;
82+
this.idToName = idToName != null ? SerializableMap.copyOf(idToName).immutableMap() : null;
7383
}
7484

7585
public static MetricsConfig getDefault() {
@@ -84,13 +94,23 @@ public static MetricsConfig forPositionDelete() {
8494
* Creates a metrics config from table configuration.
8595
*
8696
* @param props table configuration
87-
* @deprecated use {@link MetricsConfig#forTable(Table)}. Will be removed in 2.0.0
97+
* @deprecated use {@link MetricsConfig#forTable(Table)}. Will be removed in 1.13.0
8898
*/
8999
@Deprecated
90100
public static MetricsConfig fromProperties(Map<String, String> props) {
91101
return from(props, null, null);
92102
}
93103

104+
/**
105+
* Validates metrics config properties with the given schema.
106+
*
107+
* @param props table properties
108+
* @param schema table schema
109+
*/
110+
public static void validate(Map<String, String> props, Schema schema) {
111+
from(props, schema, null).validateReferencedColumns(schema);
112+
}
113+
94114
/**
95115
* Creates a metrics config from a table.
96116
*
@@ -100,32 +120,42 @@ public static MetricsConfig forTable(Table table) {
100120
return from(table.properties(), table.schema(), table.sortOrder());
101121
}
102122

103-
/**
104-
* Creates a metrics config for a position delete file.
105-
*
106-
* @param table an Iceberg table
107-
* @deprecated This method is deprecated as of version 1.11.0 and will be removed in 1.12.0.
108-
* Position deletes that include row data are no longer supported. Use {@link
109-
* #forPositionDelete()} instead.
110-
*/
111-
@Deprecated
112-
public static MetricsConfig forPositionDelete(Table table) {
113-
ImmutableMap.Builder<String, MetricsMode> columnModes = ImmutableMap.builder();
123+
public Iterable<Integer> metricsFieldIds() {
124+
Preconditions.checkState(idToName != null, "Cannot resolve column mode by ID: missing schema");
125+
return idToName.keySet();
126+
}
114127

115-
columnModes.put(MetadataColumns.DELETE_FILE_PATH.name(), MetricsModes.Full.get());
116-
columnModes.put(MetadataColumns.DELETE_FILE_POS.name(), MetricsModes.Full.get());
128+
public MetricsMode columnMode(int id) {
129+
Preconditions.checkState(idToName != null, "Cannot resolve column mode by ID: missing schema");
130+
String name = idToName.get(id);
131+
if (name != null) {
132+
return columnMode(name);
133+
}
117134

118-
MetricsConfig tableConfig = forTable(table);
135+
return defaultMode;
136+
}
119137

120-
MetricsMode defaultMode = tableConfig.defaultMode;
121-
tableConfig.columnModes.forEach(
122-
(columnAlias, mode) -> {
123-
String positionDeleteColumnAlias =
124-
DOT.join(MetadataColumns.DELETE_FILE_ROW_FIELD_NAME, columnAlias);
125-
columnModes.put(positionDeleteColumnAlias, mode);
126-
});
138+
public MetricsMode columnMode(String columnAlias) {
139+
return columnModes.getOrDefault(columnAlias, defaultMode);
140+
}
127141

128-
return new MetricsConfig(columnModes.build(), defaultMode);
142+
public void validateReferencedColumns(Schema schema) {
143+
for (String column : columnModes.keySet()) {
144+
Types.NestedField field = schema.findField(column);
145+
ValidationException.check(
146+
field != null,
147+
"Invalid metrics config, could not find column %s from table prop %s in schema %s",
148+
column,
149+
METRICS_MODE_COLUMN_CONF_PREFIX + column,
150+
schema);
151+
152+
ValidationException.check(
153+
null == idToName || column.equals(idToName.get(field.fieldId())),
154+
"Incorrect field name for id %s: %s (expected %s)",
155+
field.fieldId(),
156+
column,
157+
idToName.get(field.fieldId()));
158+
}
129159
}
130160

131161
static Set<Integer> limitFieldIds(Schema schema, int limit) {
@@ -225,6 +255,7 @@ public Set<Integer> map(
225255
*/
226256
public static MetricsConfig from(Map<String, String> props, Schema schema, SortOrder order) {
227257
int maxInferredDefaultColumns = maxInferredColumnDefaults(props);
258+
Map<Integer, String> idToName = Maps.newHashMap();
228259
Map<String, MetricsMode> columnModes = Maps.newHashMap();
229260

230261
// Handle user override of default mode
@@ -237,13 +268,20 @@ public static MetricsConfig from(Map<String, String> props, Schema schema, SortO
237268
} else if (schema == null) {
238269
defaultMode = DEFAULT_MODE;
239270
} else {
240-
if (TypeUtil.getProjectedIds(schema).size() <= maxInferredDefaultColumns) {
271+
Set<Integer> ids = TypeUtil.getProjectedIds(schema);
272+
if (ids.size() <= maxInferredDefaultColumns) {
273+
for (int id : ids) {
274+
idToName.put(id, schema.findColumnName(id));
275+
}
276+
241277
// there are less than the inferred limit (including structs), so the default is used
242278
// everywhere
243279
defaultMode = DEFAULT_MODE;
244280
} else {
245281
for (Integer id : limitFieldIds(schema, maxInferredDefaultColumns)) {
246-
columnModes.put(schema.findColumnName(id), DEFAULT_MODE);
282+
String name = schema.findColumnName(id);
283+
idToName.put(id, name);
284+
columnModes.put(name, DEFAULT_MODE);
247285
}
248286

249287
// all other columns don't use metrics
@@ -254,18 +292,29 @@ public static MetricsConfig from(Map<String, String> props, Schema schema, SortO
254292
// First set sorted column with sorted column default (can be overridden by user)
255293
MetricsMode sortedColDefaultMode = sortedColumnDefaultMode(defaultMode);
256294
Set<String> sortedCols = SortOrderUtil.orderPreservingSortedColumns(order);
257-
sortedCols.forEach(sc -> columnModes.put(sc, sortedColDefaultMode));
295+
sortedCols.forEach(
296+
name -> {
297+
columnModes.put(name, sortedColDefaultMode);
298+
Types.NestedField field = schema != null ? schema.findField(name) : null;
299+
if (field != null) {
300+
idToName.put(field.fieldId(), name);
301+
}
302+
});
258303

259304
// Handle user overrides of defaults
260305
for (String key : props.keySet()) {
261306
if (key.startsWith(METRICS_MODE_COLUMN_CONF_PREFIX)) {
262307
String columnAlias = key.replaceFirst(METRICS_MODE_COLUMN_CONF_PREFIX, "");
263308
MetricsMode mode = parseMode(props.get(key), defaultMode, "column " + columnAlias);
264309
columnModes.put(columnAlias, mode);
310+
Types.NestedField field = schema != null ? schema.findField(columnAlias) : null;
311+
if (field != null) {
312+
idToName.put(field.fieldId(), columnAlias);
313+
}
265314
}
266315
}
267316

268-
return new MetricsConfig(columnModes, defaultMode);
317+
return new MetricsConfig(columnModes, defaultMode, idToName);
269318
}
270319

271320
/**
@@ -309,19 +358,4 @@ private static MetricsMode parseMode(String modeString, MetricsMode fallback, St
309358
return fallback;
310359
}
311360
}
312-
313-
public void validateReferencedColumns(Schema schema) {
314-
for (String column : columnModes.keySet()) {
315-
ValidationException.check(
316-
schema.findField(column) != null,
317-
"Invalid metrics config, could not find column %s from table prop %s in schema %s",
318-
column,
319-
METRICS_MODE_COLUMN_CONF_PREFIX + column,
320-
schema);
321-
}
322-
}
323-
324-
public MetricsMode columnMode(String columnAlias) {
325-
return columnModes.getOrDefault(columnAlias, defaultMode);
326-
}
327361
}

‎core/src/main/java/org/apache/iceberg/MetricsModes.java‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,12 @@ public static MetricsMode fromString(String mode) {
5959
*
6060
* <p>Implementations must be immutable.
6161
*/
62-
public interface MetricsMode extends Serializable {}
62+
public interface MetricsMode extends Serializable {
63+
default boolean hasBounds() {
64+
throw new UnsupportedOperationException(
65+
"Unexpected implementation of MetricsMode without hasBounds");
66+
}
67+
}
6368

6469
/**
6570
* Under this mode, value_counts, null_value_counts, nan_value_counts, lower_bounds, upper_bounds
@@ -72,6 +77,11 @@ public static None get() {
7277
return INSTANCE;
7378
}
7479

80+
@Override
81+
public boolean hasBounds() {
82+
return false;
83+
}
84+
7585
@Override
7686
public String toString() {
7787
return "none";
@@ -86,6 +96,11 @@ public static Counts get() {
8696
return INSTANCE;
8797
}
8898

99+
@Override
100+
public boolean hasBounds() {
101+
return false;
102+
}
103+
89104
@Override
90105
public String toString() {
91106
return "counts";
@@ -108,6 +123,11 @@ public static Truncate withLength(int length) {
108123
return new Truncate(length);
109124
}
110125

126+
@Override
127+
public boolean hasBounds() {
128+
return true;
129+
}
130+
111131
public int length() {
112132
return length;
113133
}
@@ -145,6 +165,11 @@ public static Full get() {
145165
return INSTANCE;
146166
}
147167

168+
@Override
169+
public boolean hasBounds() {
170+
return true;
171+
}
172+
148173
@Override
149174
public String toString() {
150175
return "full";

‎core/src/main/java/org/apache/iceberg/PropertiesUpdate.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ public Map<String, String> apply() {
9090

9191
// Validate the metrics
9292
if (base != null && base.schema() != null) {
93-
MetricsConfig.fromProperties(newProperties).validateReferencedColumns(base.schema());
93+
MetricsConfig.validate(newProperties, base.schema());
9494
}
9595

9696
return newProperties;

‎core/src/main/java/org/apache/iceberg/TableMetadata.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ static TableMetadata newTableMetadata(
139139

140140
// Validate the metrics configuration. Note: we only do this on new tables to we don't
141141
// break existing tables.
142-
MetricsConfig.fromProperties(properties).validateReferencedColumns(schema);
142+
MetricsConfig.validate(properties, schema);
143143

144144
PropertyUtil.validateCommitProperties(properties);
145145

‎core/src/test/java/org/apache/iceberg/TestFileGenerationUtil.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,8 @@ public void testBoundsWithSpecificValues() {
9292
@SuppressWarnings("deprecation")
9393
void testBoundsForAllMetricsModes(String metricsMode) {
9494
MetricsConfig metricsConfig =
95-
MetricsConfig.fromProperties(
96-
ImmutableMap.of(TableProperties.DEFAULT_WRITE_METRICS_MODE, metricsMode));
95+
MetricsConfig.from(
96+
ImmutableMap.of(TableProperties.DEFAULT_WRITE_METRICS_MODE, metricsMode), SCHEMA, null);
9797
Metrics metrics =
9898
FileGenerationUtil.generateRandomMetrics(
9999
SCHEMA,

‎core/src/test/java/org/apache/iceberg/TestMetrics.java‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,7 @@ public void testMetricsModeForNestedStructFields() throws IOException {
326326
MetricsModes.None.get().toString(),
327327
TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "nestedStructCol.longCol",
328328
MetricsModes.Full.get().toString());
329-
MetricsConfig config = MetricsConfig.fromProperties(properties);
329+
MetricsConfig config = MetricsConfig.from(properties, NESTED_SCHEMA, null);
330330

331331
Metrics metrics = getMetrics(NESTED_SCHEMA, config, buildNestedTestRecord());
332332
MetricsWithStats metricsWithStats =
@@ -590,7 +590,8 @@ public void testNoneMetricsMode() throws IOException {
590590
Metrics metrics =
591591
getMetrics(
592592
NESTED_SCHEMA,
593-
MetricsConfig.fromProperties(ImmutableMap.of("write.metadata.metrics.default", "none")),
593+
MetricsConfig.from(
594+
ImmutableMap.of("write.metadata.metrics.default", "none"), NESTED_SCHEMA, null),
594595
buildNestedTestRecord());
595596
MetricsWithStats metricsWithStats =
596597
new MetricsWithStats(metrics, MetricsUtil.fromMetrics(NESTED_SCHEMA, metrics));
@@ -613,8 +614,8 @@ public void testCountsMetricsMode() throws IOException {
613614
Metrics metrics =
614615
getMetrics(
615616
NESTED_SCHEMA,
616-
MetricsConfig.fromProperties(
617-
ImmutableMap.of("write.metadata.metrics.default", "counts")),
617+
MetricsConfig.from(
618+
ImmutableMap.of("write.metadata.metrics.default", "counts"), NESTED_SCHEMA, null),
618619
buildNestedTestRecord());
619620
MetricsWithStats metricsWithStats =
620621
new MetricsWithStats(metrics, MetricsUtil.fromMetrics(NESTED_SCHEMA, metrics));
@@ -638,7 +639,8 @@ public void testFullMetricsMode() throws IOException {
638639
Metrics metrics =
639640
getMetrics(
640641
NESTED_SCHEMA,
641-
MetricsConfig.fromProperties(ImmutableMap.of("write.metadata.metrics.default", "full")),
642+
MetricsConfig.from(
643+
ImmutableMap.of("write.metadata.metrics.default", "full"), NESTED_SCHEMA, null),
642644
buildNestedTestRecord());
643645
MetricsWithStats metricsWithStats =
644646
new MetricsWithStats(metrics, MetricsUtil.fromMetrics(NESTED_SCHEMA, metrics));
@@ -675,8 +677,10 @@ public void testTruncateStringMetricsMode() throws IOException {
675677
Metrics metrics =
676678
getMetrics(
677679
singleStringColSchema,
678-
MetricsConfig.fromProperties(
679-
ImmutableMap.of("write.metadata.metrics.default", "truncate(10)")),
680+
MetricsConfig.from(
681+
ImmutableMap.of("write.metadata.metrics.default", "truncate(10)"),
682+
singleStringColSchema,
683+
null),
680684
record);
681685
MetricsWithStats metricsWithStats =
682686
new MetricsWithStats(metrics, MetricsUtil.fromMetrics(singleStringColSchema, metrics));
@@ -702,8 +706,10 @@ public void testTruncateBinaryMetricsMode() throws IOException {
702706
Metrics metrics =
703707
getMetrics(
704708
singleBinaryColSchema,
705-
MetricsConfig.fromProperties(
706-
ImmutableMap.of("write.metadata.metrics.default", "truncate(5)")),
709+
MetricsConfig.from(
710+
ImmutableMap.of("write.metadata.metrics.default", "truncate(5)"),
711+
singleBinaryColSchema,
712+
null),
707713
record);
708714
MetricsWithStats metricsWithStats =
709715
new MetricsWithStats(metrics, MetricsUtil.fromMetrics(singleBinaryColSchema, metrics));

0 commit comments

Comments
 (0)