Skip to content

Commit f25203b

Browse files
committed
[SPARK-59778][SQL] Refine display property contracts and helpers
1 parent 7b9429e commit f25203b

13 files changed

Lines changed: 72 additions & 48 deletions

File tree

‎sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Table.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -103,17 +103,20 @@ default Map<String, String> properties() {
103103
* precedence, and reserved keys such as {@link TableCatalog#PROP_EXTERNAL} and
104104
* {@link TableCatalog#PROP_LOCATION} are ignored in this map. Table type and other reserved
105105
* metadata are read from {@code properties()}.
106+
* After a stored property is removed, Spark displays the entry for that key if it remains in
107+
* this map.
106108
*
107-
* <p>{@code ALTER TABLE ... SET/UNSET TBLPROPERTIES} only changes {@code properties()}.
108-
* Display-only entries are controlled by the catalog and cannot be removed with
109-
* {@code UNSET TBLPROPERTIES}.
109+
* <p>Spark passes {@code ALTER TABLE ... SET/UNSET TBLPROPERTIES} to
110+
* {@link TableCatalog#alterTable} without consulting this map. Catalogs should apply these
111+
* changes to {@code properties()} and keep display-only entries unchanged.
110112
*
111113
* <p>For now, {@code SHOW TABLE EXTENDED} omits these entries for the built-in session catalog
112114
* and its {@link CatalogExtension}s.
113115
*
114116
* <p>These entries may be transient and are excluded from {@code SHOW CREATE TABLE}. The
115117
* default implementation returns an empty map.
116118
*
119+
* @return a non-null map with non-null keys and values
117120
* @since 4.4.0
118121
*/
119122
default Map<String, String> displayProperties() {

‎sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableInfo.java‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -93,11 +93,7 @@ public static class Builder extends RelationBuilder<Builder> {
9393
/**
9494
* Sets additional display-only properties.
9595
*
96-
* <p>Spark leaves these properties empty when calling
97-
* {@code TableCatalog.createTable}, {@code TableCatalog.createTableLike},
98-
* {@code StagingTableCatalog.stageCreate}, {@code StagingTableCatalog.stageReplace}, and
99-
* {@code StagingTableCatalog.stageCreateOrReplace}.
100-
*
96+
* @param displayProperties a non-null map with non-null keys and values
10197
* @see Table#displayProperties()
10298
* @since 4.4.0
10399
*/

‎sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -677,10 +677,14 @@ private[sql] object CatalogV2Util {
677677
builder
678678
}
679679

680-
/** Returns non-reserved table properties with additional display-only metadata. */
681-
def tablePropertiesForDisplay(table: Table): Map[String, String] = {
682-
(table.displayProperties().asScala.toMap ++ table.properties().asScala)
680+
/** Returns redacted, non-reserved table properties with additional display-only metadata. */
681+
def tablePropertiesForDisplay(
682+
properties: util.Map[String, String],
683+
displayProperties: util.Map[String, String],
684+
conf: SQLConf): Map[String, String] = {
685+
val merged = (displayProperties.asScala.toMap ++ properties.asScala)
683686
.filterNot { case (key, _) => TABLE_RESERVED_PROPERTIES.contains(key) }
687+
conf.redactOptions(merged)
684688
}
685689

686690
def convertTableProperties(t: TableSpec): Map[String, String] = {

‎sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogSuite.scala‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -97,12 +97,13 @@ class CatalogSuite extends SparkFunSuite {
9797
}
9898

9999
gridTest("display properties survive copies, pinning and alterations")(
100-
Seq("copyOnLoad", "rowLevel")) { mode =>
101-
val catalog = if (mode == "rowLevel") {
100+
Seq("InMemoryTableCatalog", "InMemoryRowLevelOperationTableCatalog")) { catalogName =>
101+
val catalog = if (catalogName == "InMemoryRowLevelOperationTableCatalog") {
102102
new InMemoryRowLevelOperationTableCatalog
103103
} else {
104104
new InMemoryTableCatalog
105105
}
106+
// The row-level catalog always copies on load; enable this for InMemoryTableCatalog too.
106107
catalog.initialize("test", new CaseInsensitiveStringMap(util.Map.of("copyOnLoad", "true")))
107108
val table = catalog.createTable(testIdent, new TableInfo.Builder()
108109
.withColumns(columns)
@@ -121,10 +122,12 @@ class CatalogSuite extends SparkFunSuite {
121122
val altered = catalog.alterTable(testIdent, TableChange.setProperty("persisted", "updated"))
122123
.asInstanceOf[InMemoryBaseTable]
123124
assert(altered.validatedVersion() === "2")
124-
assert(CatalogV2Util.tablePropertiesForDisplay(catalog.loadTable(testIdent)) ===
125-
Map("persisted" -> "updated", "catalog-label" -> "catalog-value"))
126-
assert(CatalogV2Util.tablePropertiesForDisplay(catalog.loadTable(testIdent, "snapshot")) ===
127-
Map("persisted" -> "stored", "catalog-label" -> "catalog-value"))
125+
val current = catalog.loadTable(testIdent)
126+
assert(current.properties() === util.Map.of("persisted", "updated"))
127+
assert(current.displayProperties() === util.Map.of("catalog-label", "catalog-value"))
128+
val snapshot = catalog.loadTable(testIdent, "snapshot")
129+
assert(snapshot.properties() === util.Map.of("persisted", "stored"))
130+
assert(snapshot.displayProperties() === util.Map.of("catalog-label", "catalog-value"))
128131
}
129132

130133
test("listTables") {

‎sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogV2UtilSuite.scala‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ class CatalogV2UtilSuite extends SparkFunSuite {
4242
when(table.properties()).thenReturn(util.Map.of(
4343
"persisted", "stored", TableCatalog.PROP_EXTERNAL, "true"))
4444

45-
assert(CatalogV2Util.tablePropertiesForDisplay(table) === Map("persisted" -> "stored"))
45+
assert(CatalogV2Util.tablePropertiesForDisplay(
46+
table.properties(), table.displayProperties(), new SQLConf) === Map("persisted" -> "stored"))
4647
}
4748

4849
test("tablePropertiesForDisplay ignores reserved display keys and preserves stored values") {
@@ -55,11 +56,28 @@ class CatalogV2UtilSuite extends SparkFunSuite {
5556
.withDisplayProperties(displayProperties)
5657
.build(), "table")
5758

58-
assert(CatalogV2Util.tablePropertiesForDisplay(table) ===
59+
assert(CatalogV2Util.tablePropertiesForDisplay(
60+
table.properties(), table.displayProperties(), new SQLConf) ===
5961
Map("persisted" -> "stored", "catalog-label" -> "catalog-value"))
6062
assert(table.properties() === util.Map.of("persisted", "stored"))
6163
}
6264

65+
test("tablePropertiesForDisplay redacts stored and display-only values") {
66+
val conf = new SQLConf
67+
conf.setConf(SQLConf.SQL_OPTIONS_REDACTION_PATTERN, "(?i)catalog-label".r)
68+
val properties = util.Map.of("password", "stored-password", "persisted", "stored")
69+
val displayProperties = util.Map.of(
70+
"catalog-label", "catalog-value", "secret", "display-secret", "persisted", "display-value")
71+
72+
assert(CatalogV2Util.tablePropertiesForDisplay(properties, displayProperties, conf) === Map(
73+
"password" -> "*********(redacted)",
74+
"catalog-label" -> "*********(redacted)",
75+
"secret" -> "*********(redacted)",
76+
"persisted" -> "stored"))
77+
assert(properties.get("password") === "stored-password")
78+
assert(displayProperties.get("secret") === "display-secret")
79+
}
80+
6381
private def catalogWithStateOptions(keys: java.util.Set[String]): TableCatalog = {
6482
val catalog = mock(classOf[TableCatalog])
6583
when(catalog.tableStateOptionKeys()).thenReturn(keys)

‎sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableCatalog.scala‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -168,8 +168,7 @@ class BasicInMemoryTableCatalog extends TableCatalog {
168168
}
169169

170170
override def createTable(ident: Identifier, tableInfo: TableInfo): Table = {
171-
createTable(
172-
ident, tableInfo.columns(), tableInfo.partitions(), tableInfo.properties(),
171+
createTable(ident, tableInfo.columns(), tableInfo.partitions(), tableInfo.properties(),
173172
Distributions.unspecified(), Array.empty, None, None, tableInfo.constraints())
174173
}
175174

‎sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeTableExec.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,8 @@ case class DescribeTableExec(
203203
}
204204
})
205205
val properties =
206-
conf.redactOptions(CatalogV2Util.tablePropertiesForDisplay(table)).toList
206+
CatalogV2Util.tablePropertiesForDisplay(
207+
tableProperties, table.displayProperties(), conf).toList
207208
.sortBy(_._1).map {
208209
case (key, value) => key + "=" + value
209210
}.mkString("[", ",", "]")

‎sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,8 @@ case class ShowTablePropertiesExec(
3232

3333
override protected def run(): Seq[InternalRow] = {
3434
// The reserved properties are accessible through DESCRIBE
35-
val properties = conf.redactOptions(CatalogV2Util.tablePropertiesForDisplay(catalogTable))
35+
val properties = CatalogV2Util.tablePropertiesForDisplay(
36+
catalogTable.properties(), catalogTable.displayProperties(), conf)
3637
propertyKey match {
3738
case Some(p) =>
3839
val propValue = properties

‎sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExtendedExec.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,8 @@ case class ShowTablesExtendedExec(
9999
})
100100

101101
val displayedTableProperties =
102-
conf.redactOptions(CatalogV2Util.tablePropertiesForDisplay(table)).toList
102+
CatalogV2Util.tablePropertiesForDisplay(
103+
tableProperties, table.displayProperties(), conf).toList
103104
.sortBy(_._1)
104105
.map { case (key, value) => key + "=" + value }
105106
if (displayedTableProperties.nonEmpty) {

‎sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CommandSuiteBase.scala‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,21 @@ trait CommandSuiteBase extends SharedSparkSession {
8080
.asInstanceOf[InMemoryTable]
8181
}
8282

83+
def createTableWithDisplayProperties(
84+
namespace: String,
85+
table: String,
86+
external: Boolean = false,
87+
withLocation: Boolean = false): Unit = {
88+
val location = if (withLocation) "LOCATION 'file:/tmp/display-properties' " else ""
89+
sql(s"CREATE TABLE $catalog.$namespace.$table (id bigint) $defaultUsing " +
90+
s"COMMENT 'table comment' $location" +
91+
"TBLPROPERTIES ('persisted' = 'stored')")
92+
setDisplayProperties(loadTable(catalog, namespace, table), external)
93+
}
94+
95+
/**
96+
* Sets display-only test metadata. When external is true, also marks the stored table external.
97+
*/
8398
def setDisplayProperties(table: InMemoryTable, external: Boolean = false): Unit = {
8499
if (external) {
85100
table.properties.put(TableCatalog.PROP_EXTERNAL, "true")

0 commit comments

Comments
 (0)