Skip to content

Commit 7b9429e

Browse files
committed
[SPARK-59778][SQL] Clarify display property contracts and expand coverage
1 parent 7948d05 commit 7b9429e

16 files changed

Lines changed: 231 additions & 127 deletions

File tree

‎python/pyspark/sql/catalog.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -383,12 +383,6 @@ def getTableProperties(self, tableName: str) -> Dict[str, str]:
383383
384384
.. versionadded:: 4.2.0
385385
386-
Notes
387-
-----
388-
For V2 tables, this also includes the table's display-only properties. These entries
389-
may be transient and unsuitable for copying into ``CREATE TABLE`` or ``ALTER TABLE``.
390-
Use :meth:`getCreateTableString` to obtain the table's DDL.
391-
392386
Parameters
393387
----------
394388
tableName : str
@@ -399,6 +393,12 @@ def getTableProperties(self, tableName: str) -> Dict[str, str]:
399393
dict
400394
Map of property key to value.
401395
396+
Notes
397+
-----
398+
For V2 tables, the returned dict also includes display-only properties, which may be
399+
transient. These entries are excluded from the table DDL returned by
400+
:meth:`getCreateTableString`.
401+
402402
Examples
403403
--------
404404
>>> _ = spark.sql("DROP TABLE IF EXISTS tbl_props_doc")

‎sql/api/src/main/scala/org/apache/spark/sql/catalog/Catalog.scala‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -846,9 +846,8 @@ abstract class Catalog {
846846
/**
847847
* Returns all table properties as a map (same as `SHOW TBLPROPERTIES`).
848848
*
849-
* For V2 tables, this also includes the table's display-only properties. These entries may be
850-
* transient and unsuitable for copying into `CREATE TABLE` or `ALTER TABLE`. Use
851-
* `getCreateTableString` to obtain the table's DDL.
849+
* For V2 tables, the returned map also includes display-only properties, which may be
850+
* transient. These entries are excluded from the table DDL returned by `getCreateTableString`.
852851
*
853852
* @param tableName
854853
* table or view name; may be qualified with catalog and database (namespace).

‎sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Table.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,13 @@ default Map<String, String> properties() {
104104
* {@link TableCatalog#PROP_LOCATION} are ignored in this map. Table type and other reserved
105105
* metadata are read from {@code properties()}.
106106
*
107+
* <p>{@code ALTER TABLE ... SET/UNSET TBLPROPERTIES} only changes {@code properties()}.
108+
* Display-only entries are controlled by the catalog and cannot be removed with
109+
* {@code UNSET TBLPROPERTIES}.
110+
*
111+
* <p>For now, {@code SHOW TABLE EXTENDED} omits these entries for the built-in session catalog
112+
* and its {@link CatalogExtension}s.
113+
*
107114
* <p>These entries may be transient and are excluded from {@code SHOW CREATE TABLE}. The
108115
* default implementation returns an empty map.
109116
*

‎sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableInfo.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,11 @@ public Map<String, String> properties() {
6565
/**
6666
* Returns the additional display-only properties described by {@link Table#displayProperties()}.
6767
*
68+
* <p>These properties describe metadata returned by a catalog. Spark leaves them empty when
69+
* calling {@code TableCatalog.createTable},
70+
* {@code TableCatalog.createTableLike}, {@code StagingTableCatalog.stageCreate},
71+
* {@code StagingTableCatalog.stageReplace}, and {@code StagingTableCatalog.stageCreateOrReplace}.
72+
*
6873
* @since 4.4.0
6974
*/
7075
public Map<String, String> displayProperties() {
@@ -86,7 +91,12 @@ public static class Builder extends RelationBuilder<Builder> {
8691
protected Builder self() { return this; }
8792

8893
/**
89-
* Sets additional display-only properties, taking a defensive copy of the map.
94+
* Sets additional display-only properties.
95+
*
96+
* <p>Spark leaves these properties empty when calling
97+
* {@code TableCatalog.createTable}, {@code TableCatalog.createTableLike},
98+
* {@code StagingTableCatalog.stageCreate}, {@code StagingTableCatalog.stageReplace}, and
99+
* {@code StagingTableCatalog.stageCreateOrReplace}.
90100
*
91101
* @see Table#displayProperties()
92102
* @since 4.4.0

‎sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogSuite.scala‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -107,14 +107,20 @@ class CatalogSuite extends SparkFunSuite {
107107
val table = catalog.createTable(testIdent, new TableInfo.Builder()
108108
.withColumns(columns)
109109
.withProperties(util.Map.of("persisted", "stored"))
110-
.withDisplayProperties(util.Map.of("catalog-label", "catalog-value"))
111-
.build())
112-
val loaded = catalog.loadTable(testIdent)
110+
.build()).asInstanceOf[InMemoryBaseTable]
111+
table.setDisplayProperties(util.Map.of("catalog-label", "catalog-value"))
112+
table.setVersion("3")
113+
table.setValidatedVersion("2")
114+
val loaded = catalog.loadTable(testIdent).asInstanceOf[InMemoryBaseTable]
113115
assert(loaded ne table)
114116
assert(loaded.displayProperties() === util.Map.of("catalog-label", "catalog-value"))
117+
assert(loaded.version() === "3")
118+
assert(loaded.validatedVersion() === "2")
115119
catalog.pinTable(testIdent, "snapshot")
116120

117-
catalog.alterTable(testIdent, TableChange.setProperty("persisted", "updated"))
121+
val altered = catalog.alterTable(testIdent, TableChange.setProperty("persisted", "updated"))
122+
.asInstanceOf[InMemoryBaseTable]
123+
assert(altered.validatedVersion() === "2")
118124
assert(CatalogV2Util.tablePropertiesForDisplay(catalog.loadTable(testIdent)) ===
119125
Map("persisted" -> "updated", "catalog-label" -> "catalog-value"))
120126
assert(CatalogV2Util.tablePropertiesForDisplay(catalog.loadTable(testIdent, "snapshot")) ===

‎sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryBaseTable.scala‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,16 @@ abstract class InMemoryBaseTable(
102102
}
103103

104104
/**
105-
* Copies version, validated version and display-only metadata into a replacement table.
105+
* Copies version, validated version and display-only metadata from another table.
106+
*
107+
* Some test catalogs (e.g. [[NullColumnIdInMemoryTableCatalog]],
108+
* [[NullTableIdAndNullColumnIdInMemoryTableCatalog]]) create a new table object
109+
* that overrides specific behavior (such as nulling out column IDs). The new
110+
* object's version counter starts at 0. Without this call, the version counter
111+
* resets every time the catalog creates such a replacement table, breaking the
112+
* monotonic-version assumption that downstream consumers rely on (e.g.
113+
* [[InMemoryTable]].copy, validated-version propagation, and the join-refresh
114+
* tests in [[DSv2IncrementallyConstructedQueryTests]]).
106115
*/
107116
def copyTableStateFrom(sourceTable: InMemoryBaseTable): Unit = {
108117
setDisplayProperties(sourceTable.displayProperties())

‎sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryRowLevelOperationTableCatalog.scala‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@ class InMemoryRowLevelOperationTableCatalog
7171
val schema = CatalogV2Util.v2ColumnsToStructType(tableInfo.columns)
7272
val table = new InMemoryRowLevelOperationTable(
7373
tableName, schema, tableInfo.partitions, tableInfo.properties, tableInfo.constraints())
74-
table.setDisplayProperties(tableInfo.displayProperties())
7574
tables.put(ident, table)
7675
namespaces.putIfAbsent(ident.namespace.toList, Map())
7776
table

‎sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableCatalog.scala‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -168,11 +168,9 @@ class BasicInMemoryTableCatalog extends TableCatalog {
168168
}
169169

170170
override def createTable(ident: Identifier, tableInfo: TableInfo): Table = {
171-
val table = createTable(
171+
createTable(
172172
ident, tableInfo.columns(), tableInfo.partitions(), tableInfo.properties(),
173173
Distributions.unspecified(), Array.empty, None, None, tableInfo.constraints())
174-
table.asInstanceOf[InMemoryBaseTable].setDisplayProperties(tableInfo.displayProperties())
175-
table
176174
}
177175

178176
// scalastyle:off argcount
@@ -263,7 +261,6 @@ class BasicInMemoryTableCatalog extends TableCatalog {
263261
}
264262

265263
table.increaseVersion()
266-
val currentVersion = table.version()
267264
val columnsWithIds = InMemoryBaseTable.assignMissingIds(
268265
CatalogV2Util.structTypeToV2Columns(schema))
269266
val reconstructedId = Option(table.id()).getOrElse(util.UUID.randomUUID().toString)
@@ -278,8 +275,7 @@ class BasicInMemoryTableCatalog extends TableCatalog {
278275
throw new UnsupportedOperationException(
279276
s"Unsupported InMemoryBaseTable subclass: ${other.getClass.getName}")
280277
}
281-
newTable.setVersion(currentVersion)
282-
newTable.setDisplayProperties(table.displayProperties())
278+
newTable.copyTableStateFrom(table)
283279
changes.foreach {
284280
case a: TableChange.AddConstraint =>
285281
newTable.setValidatedVersion(a.validatedTableVersion())

‎sql/connect/common/src/main/scala/org/apache/spark/sql/connect/Catalog.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -853,6 +853,9 @@ class Catalog(sparkSession: SparkSession) extends catalog.Catalog {
853853
/**
854854
* Returns all table properties as a map (same as `SHOW TBLPROPERTIES`).
855855
*
856+
* For V2 tables, the returned map also includes display-only properties, which may be
857+
* transient. These entries are excluded from the table DDL returned by `getCreateTableString`.
858+
*
856859
* @param tableName
857860
* qualified or unqualified table or view name
858861
* @since 4.2.0

‎sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,12 @@
1717

1818
package org.apache.spark.sql.connector
1919

20+
import java.util
2021
import java.util.Locale
2122

2223
import org.scalatest.BeforeAndAfter
2324

24-
import org.apache.spark.sql.{AnalysisException, DataFrame, SaveMode}
25+
import org.apache.spark.sql.{AnalysisException, DataFrame, Row, SaveMode}
2526
import org.apache.spark.sql.catalyst.TableIdentifier
2627
import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
2728
import org.apache.spark.sql.connector.catalog._
@@ -53,6 +54,25 @@ class DataSourceV2DataFrameSessionCatalogSuite
5354

5455
override protected val catalogAndNamespace: String = ""
5556

57+
test("display properties survive ALTER TABLE in the session catalog") {
58+
val t = "display_properties"
59+
withTable(t) {
60+
sql(s"CREATE TABLE $t (id bigint) USING $v2Format")
61+
val table = sessionCatalog.loadTable(Identifier.of(Array("default"), t))
62+
.asInstanceOf[InMemoryTable]
63+
table.setDisplayProperties(util.Map.of("catalog-label", "catalog-value"))
64+
65+
sql(s"ALTER TABLE $t SET TBLPROPERTIES ('k' = 'v')")
66+
checkAnswer(sql(s"SHOW TBLPROPERTIES $t"),
67+
Seq(Row("catalog-label", "catalog-value"), Row("k", "v")))
68+
checkAnswer(sql(s"DESCRIBE TABLE EXTENDED $t").where("col_name = 'Table Properties'"),
69+
Row("Table Properties", "[catalog-label=catalog-value,k=v]", ""))
70+
71+
sql(s"ALTER TABLE $t UNSET TBLPROPERTIES ('k')")
72+
checkAnswer(sql(s"SHOW TBLPROPERTIES $t"), Row("catalog-label", "catalog-value"))
73+
}
74+
}
75+
5676
test("saveAsTable: Append mode should not fail if the table already exists " +
5777
"and a same-name temp view exist") {
5878
withTable("same_name") {
@@ -177,6 +197,7 @@ class InMemoryTableSessionCatalog extends TestV2SessionCatalogBase[InMemoryTable
177197

178198
val newTable = new InMemoryTable(table.name, schema, table.partitioning, properties)
179199
.alterTableWithData(table.data, schema)
200+
newTable.setDisplayProperties(table.displayProperties())
180201

181202
tables.put(ident, newTable)
182203

0 commit comments

Comments
 (0)