|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, |
| 13 | + * software distributed under the License is distributed on an |
| 14 | + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | + * KIND, either express or implied. See the License for the |
| 16 | + * specific language governing permissions and limitations |
| 17 | + * under the License. |
| 18 | + */ |
| 19 | + |
| 20 | +package org.apache.comet.contrib.delta |
| 21 | + |
| 22 | +import org.apache.spark.sql.functions._ |
| 23 | + |
| 24 | +// Local reproductions for the Delta 4.1 own-suite regression families that are |
| 25 | +// not yet fixed on this branch. One faithful, minimal repro per root cause so |
| 26 | +// each can be diagnosed and fixed (and kept as a regression guard afterward). |
| 27 | +// |
| 28 | +// Status when written (after F1 DPP / F2 time-travel / #198 fixes landed): |
| 29 | +// F3 (row tracking) -- PENDING; dominant family (all RowTracking* suites) |
| 30 | +// F4 (protobuf recursion)-- PENDING |
| 31 | +// F6 (corrupted file) -- PENDING |
| 32 | +// |
| 33 | +// Each test asserts the EXPECTED (correct) behavior and is marked `ignore` so |
| 34 | +// CI stays green while the bug is open. It is a confirmed repro: un-`ignore` it |
| 35 | +// (change `ignore(` -> `test(`) to watch it fail today and pass once the |
| 36 | +// corresponding fix lands. Verified failing at the time of writing (see commit |
| 37 | +// message / docs/08-known-limitations.md). |
| 38 | +class CometDeltaPendingReproSuite extends CometDeltaTestBase { |
| 39 | + |
| 40 | + // === F3: row tracking -- materialized stable row IDs ====================== |
| 41 | + // |
| 42 | + // Mirrors RowTracking{Merge,Compaction,ReadWrite}Suite. When a file is |
| 43 | + // rewritten (OPTIMIZE / z-order / compaction / MERGE), Delta preserves stable |
| 44 | + // row IDs by MATERIALIZING them into real parquet columns |
| 45 | + // `_row-id-col-<uuid>` / `_row-commit-version-col-<uuid>`. The Spark plan then |
| 46 | + // reads `coalesce(_metadata.row_id, base_row_id + row_index)`. The native scan |
| 47 | + // classifies those names as synthetic (`isExtraSyntheticName`) and synthesizes |
| 48 | + // from `base_row_id + row_index` instead of reading the persisted values -- |
| 49 | + // so after a rewrite the row IDs CHANGE (new file => new base+index) instead |
| 50 | + // of staying stable. Expected: row IDs are stable across OPTIMIZE. |
| 51 | + |
| 52 | + ignore("F3: row IDs are stable across OPTIMIZE (materialized row-id columns)") { |
| 53 | + assume(deltaSparkAvailable, "delta-spark not on the test classpath; skipping") |
| 54 | + withDeltaTable("f3_rowid_optimize") { tablePath => |
| 55 | + val ss = spark |
| 56 | + import ss.implicits._ |
| 57 | + // Multiple files so OPTIMIZE actually compacts/rewrites. |
| 58 | + (0 until 100) |
| 59 | + .map(i => (i.toLong, s"v_$i")) |
| 60 | + .toDF("id", "v") |
| 61 | + .repartition(5) |
| 62 | + .write |
| 63 | + .format("delta") |
| 64 | + .option("delta.enableRowTracking", "true") |
| 65 | + .option("delta.minReaderVersion", "3") |
| 66 | + .option("delta.minWriterVersion", "7") |
| 67 | + .save(tablePath) |
| 68 | + |
| 69 | + def rowIdsByValue(): Map[String, Long] = |
| 70 | + spark.read.format("delta").load(tablePath) |
| 71 | + .select(col("v"), col("_metadata.row_id").as("rid")) |
| 72 | + .collect() |
| 73 | + .map(r => r.getString(0) -> r.getLong(1)) |
| 74 | + .toMap |
| 75 | + |
| 76 | + val before = rowIdsByValue() |
| 77 | + // OPTIMIZE compacts the 5 files into 1, rewriting + materializing row IDs. |
| 78 | + spark.sql(s"OPTIMIZE delta.`$tablePath`") |
| 79 | + val after = rowIdsByValue() |
| 80 | + |
| 81 | + assert(before.nonEmpty && before.size == 100, s"setup: ${before.size} rows") |
| 82 | + val changed = before.keys.filter(k => before(k) != after.getOrElse(k, -1L)).toSeq.sorted |
| 83 | + assert( |
| 84 | + changed.isEmpty, |
| 85 | + s"row IDs must be stable across OPTIMIZE; changed for ${changed.size} rows, " + |
| 86 | + s"e.g. ${changed.take(3).map(k => s"$k: ${before(k)} -> ${after.get(k)}").mkString(", ")}") |
| 87 | + } |
| 88 | + } |
| 89 | + |
| 90 | + // === F4: deeply-nested data-skipping expression -> protobuf recursion ===== |
| 91 | + // |
| 92 | + // Mirrors DataSkippingDeltaTests "remove redundant stats column references in |
| 93 | + // data skipping expression". A WHERE with ~101 AND'd conditions builds a very |
| 94 | + // deep boolean expression; serializing it to Comet's native proto exceeds |
| 95 | + // protobuf's default recursion limit (100), throwing |
| 96 | + // "Protocol message had too many levels of nesting" from BinaryExpr.mergeFrom. |
| 97 | + // Expected: the query runs. |
| 98 | + |
| 99 | + ignore("F4: deeply-nested data-skipping filter does not overflow protobuf nesting") { |
| 100 | + assume(deltaSparkAvailable, "delta-spark not on the test classpath; skipping") |
| 101 | + val tbl = "f4_deep_filter" |
| 102 | + withTable(tbl) { |
| 103 | + val colNames = (0 to 100).map(i => s"col_$i") |
| 104 | + spark.sql( |
| 105 | + s"CREATE TABLE $tbl (${colNames.map(_ + " INT").mkString(", ")}) USING delta") |
| 106 | + spark.sql( |
| 107 | + s"INSERT INTO $tbl VALUES (${colNames.map(_ => "0").mkString(", ")})") |
| 108 | + val whereClause = colNames.map(c => s"$c != 1").mkString(" AND ") |
| 109 | + // Must not throw a protobuf recursion error. |
| 110 | + val rows = spark.sql(s"SELECT col_0 FROM $tbl WHERE $whereClause").collect() |
| 111 | + assert(rows.length == 1, s"expected the single all-zero row, got ${rows.length}") |
| 112 | + } |
| 113 | + } |
| 114 | + |
| 115 | + // === F6: corrupted / empty file (SC-8810) ================================= |
| 116 | + // |
| 117 | + // Mirrors DeltaSuite "SC-8810: skipping deleted file still throws on corrupted |
| 118 | + // file". With one data file truncated to 0 bytes, vanilla Spark+Delta throws a |
| 119 | + // `[FAILED_READ_FILE.NO_HINT]` SparkException. Comet's native reader instead |
| 120 | + // throws `CometNativeException: ... Requested range was invalid`. Expected: |
| 121 | + // the error is the Spark-compatible one (so user-facing error handling and the |
| 122 | + // Delta test pass). Repro asserts the message contains the Spark marker. |
| 123 | + |
| 124 | + ignore("F6: reading a corrupted file surfaces a Spark-compatible error (SC-8810)") { |
| 125 | + assume(deltaSparkAvailable, "delta-spark not on the test classpath; skipping") |
| 126 | + withDeltaTable("f6_corrupt") { tablePath => |
| 127 | + val ss = spark |
| 128 | + import ss.implicits._ |
| 129 | + Seq(1).toDF().write.format("delta").mode("append").save(tablePath) |
| 130 | + Seq(2, 2).toDF().write.format("delta").mode("append").save(tablePath) |
| 131 | + Seq(4).toDF().write.format("delta").mode("append").save(tablePath) |
| 132 | + |
| 133 | + // Truncate one data file to 0 bytes to simulate corruption. |
| 134 | + val dir = new java.io.File(tablePath) |
| 135 | + val parquet = dir.listFiles() |
| 136 | + .filter(f => !f.getName.startsWith("_") && f.getName.endsWith(".parquet")) |
| 137 | + .sortBy(_.getName) |
| 138 | + .head |
| 139 | + val ch = new java.io.FileOutputStream(parquet) |
| 140 | + try ch.getChannel.truncate(0) finally ch.close() |
| 141 | + |
| 142 | + val ex = intercept[Exception] { |
| 143 | + spark.read.format("delta").load(tablePath).collect() |
| 144 | + } |
| 145 | + val msg = Option(ex.getMessage).getOrElse("") + |
| 146 | + Option(ex.getCause).map(c => Option(c.getMessage).getOrElse("")).getOrElse("") |
| 147 | + assert( |
| 148 | + msg.contains("FAILED_READ_FILE"), |
| 149 | + s"expected a Spark-compatible FAILED_READ_FILE error, got: $msg") |
| 150 | + } |
| 151 | + } |
| 152 | +} |
0 commit comments