|
14 | 14 | # KIND, either express or implied. See the License for the |
15 | 15 | # specific language governing permissions and limitations |
16 | 16 | # under the License. |
17 | | -import time |
18 | 17 |
|
19 | 18 | from pyspark.sql import SparkSession |
20 | 19 | from pyspark.sql.functions import current_date, date_add, expr |
21 | 20 |
|
22 | 21 | spark = SparkSession.builder.getOrCreate() |
23 | 22 |
|
24 | | -print("Create database") |
25 | | - |
26 | 23 | spark.sql( |
27 | 24 | """ |
28 | 25 | CREATE DATABASE IF NOT EXISTS default; |
|
31 | 28 |
|
32 | 29 | spark.sql( |
33 | 30 | """ |
34 | | - use default; |
35 | | -""" |
36 | | -) |
37 | | - |
38 | | -spark.sql( |
39 | | - """ |
40 | | - DROP TABLE IF EXISTS test_null_nan; |
41 | | -""" |
42 | | -) |
43 | | - |
44 | | -spark.sql( |
45 | | - """ |
46 | | - CREATE TABLE test_null_nan |
| 31 | + CREATE OR REPLACE TABLE default.test_null_nan |
47 | 32 | USING iceberg |
48 | 33 | AS SELECT |
49 | 34 | 1 AS idx, |
|
59 | 44 |
|
60 | 45 | spark.sql( |
61 | 46 | """ |
62 | | - DROP TABLE IF EXISTS test_null_nan_rewritten; |
| 47 | + CREATE OR REPLACE TABLE default.test_null_nan_rewritten |
| 48 | + USING iceberg |
| 49 | + AS SELECT * FROM default.test_null_nan |
63 | 50 | """ |
64 | 51 | ) |
65 | 52 |
|
66 | 53 | spark.sql( |
67 | 54 | """ |
68 | | - CREATE TABLE test_null_nan_rewritten |
69 | | - USING iceberg |
70 | | - AS SELECT * FROM test_null_nan |
| 55 | +CREATE OR REPLACE TABLE default.test_limit as |
| 56 | + SELECT * LATERAL VIEW explode(ARRAY(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)) AS idx; |
71 | 57 | """ |
72 | 58 | ) |
73 | 59 |
|
74 | 60 | spark.sql( |
75 | 61 | """ |
76 | | - DROP TABLE IF EXISTS test_limit; |
| 62 | +CREATE OR REPLACE TABLE default.test_positional_mor_deletes ( |
| 63 | + dt date, |
| 64 | + number integer, |
| 65 | + letter string |
| 66 | +) |
| 67 | +USING iceberg |
| 68 | +TBLPROPERTIES ( |
| 69 | + 'write.delete.mode'='merge-on-read', |
| 70 | + 'write.update.mode'='merge-on-read', |
| 71 | + 'write.merge.mode'='merge-on-read', |
| 72 | + 'format-version'='2' |
| 73 | +); |
77 | 74 | """ |
78 | 75 | ) |
79 | 76 |
|
| 77 | +# Partitioning is not really needed, but there is a bug: |
| 78 | +# https://github.com/apache/iceberg/pull/7685 |
80 | 79 | spark.sql( |
81 | 80 | """ |
82 | | - CREATE TABLE test_limit |
83 | | - USING iceberg |
84 | | - AS SELECT |
85 | | - 1 AS idx |
86 | | - UNION ALL SELECT |
87 | | - 2 AS idx |
88 | | - UNION ALL SELECT |
89 | | - 3 AS idx |
90 | | - UNION ALL SELECT |
91 | | - 4 AS idx |
92 | | - UNION ALL SELECT |
93 | | - 5 AS idx |
94 | | - UNION ALL SELECT |
95 | | - 6 AS idx |
96 | | - UNION ALL SELECT |
97 | | - 7 AS idx |
98 | | - UNION ALL SELECT |
99 | | - 8 AS idx |
100 | | - UNION ALL SELECT |
101 | | - 9 AS idx |
102 | | - UNION ALL SELECT |
103 | | - 10 AS idx |
| 81 | + ALTER TABLE default.test_positional_mor_deletes ADD PARTITION FIELD years(dt) AS dt_years |
| 82 | +""" |
| 83 | +) |
| 84 | + |
| 85 | +spark.sql( |
104 | 86 | """ |
| 87 | +INSERT INTO default.test_positional_mor_deletes |
| 88 | +VALUES |
| 89 | + (CAST('2023-03-01' AS date), 1, 'a'), |
| 90 | + (CAST('2023-03-02' AS date), 2, 'b'), |
| 91 | + (CAST('2023-03-03' AS date), 3, 'c'), |
| 92 | + (CAST('2023-03-04' AS date), 4, 'd'), |
| 93 | + (CAST('2023-03-05' AS date), 5, 'e'), |
| 94 | + (CAST('2023-03-06' AS date), 6, 'f'), |
| 95 | + (CAST('2023-03-07' AS date), 7, 'g'), |
| 96 | + (CAST('2023-03-08' AS date), 8, 'h'), |
| 97 | + (CAST('2023-03-09' AS date), 9, 'i'), |
| 98 | + (CAST('2023-03-10' AS date), 10, 'j'), |
| 99 | + (CAST('2023-03-11' AS date), 11, 'k'), |
| 100 | + (CAST('2023-03-12' AS date), 12, 'l'); |
| 101 | +""" |
105 | 102 | ) |
106 | 103 |
|
107 | 104 | spark.sql( |
108 | 105 | """ |
109 | | - DROP TABLE IF EXISTS test_deletes; |
| 106 | +DELETE FROM default.test_positional_mor_deletes WHERE number = 9 |
110 | 107 | """ |
111 | 108 | ) |
112 | 109 |
|
113 | 110 | spark.sql( |
114 | 111 | """ |
115 | | - CREATE TABLE test_deletes |
| 112 | + CREATE OR REPLACE TABLE default.test_positional_mor_double_deletes ( |
| 113 | + dt date, |
| 114 | + number integer, |
| 115 | + letter string |
| 116 | + ) |
116 | 117 | USING iceberg |
117 | 118 | TBLPROPERTIES ( |
118 | 119 | 'write.delete.mode'='merge-on-read', |
119 | 120 | 'write.update.mode'='merge-on-read', |
120 | | - 'write.merge.mode'='merge-on-read' |
121 | | - ) |
122 | | - AS SELECT |
123 | | - 1 AS idx, |
124 | | - True AS deleted |
125 | | -UNION ALL SELECT |
126 | | - 2 AS idx, |
127 | | - False AS deleted; |
| 121 | + 'write.merge.mode'='merge-on-read', |
| 122 | + 'format-version'='2' |
| 123 | + ); |
| 124 | +""" |
| 125 | +) |
| 126 | + |
| 127 | +# Partitioning is not really needed, but there is a bug: |
| 128 | +# https://github.com/apache/iceberg/pull/7685 |
| 129 | +spark.sql( |
| 130 | + """ |
| 131 | + ALTER TABLE default.test_positional_mor_double_deletes ADD PARTITION FIELD years(dt) AS dt_years |
| 132 | +""" |
| 133 | +) |
| 134 | + |
| 135 | +spark.sql( |
| 136 | + """ |
| 137 | +INSERT INTO default.test_positional_mor_double_deletes |
| 138 | +VALUES |
| 139 | + (CAST('2023-03-01' AS date), 1, 'a'), |
| 140 | + (CAST('2023-03-02' AS date), 2, 'b'), |
| 141 | + (CAST('2023-03-03' AS date), 3, 'c'), |
| 142 | + (CAST('2023-03-04' AS date), 4, 'd'), |
| 143 | + (CAST('2023-03-05' AS date), 5, 'e'), |
| 144 | + (CAST('2023-03-06' AS date), 6, 'f'), |
| 145 | + (CAST('2023-03-07' AS date), 7, 'g'), |
| 146 | + (CAST('2023-03-08' AS date), 8, 'h'), |
| 147 | + (CAST('2023-03-09' AS date), 9, 'i'), |
| 148 | + (CAST('2023-03-10' AS date), 10, 'j'), |
| 149 | + (CAST('2023-03-11' AS date), 11, 'k'), |
| 150 | + (CAST('2023-03-12' AS date), 12, 'l'); |
| 151 | +""" |
| 152 | +) |
| 153 | + |
| 154 | +spark.sql( |
| 155 | + """ |
| 156 | + DELETE FROM default.test_positional_mor_double_deletes WHERE number = 9 |
128 | 157 | """ |
129 | 158 | ) |
130 | 159 |
|
131 | 160 | spark.sql( |
132 | 161 | """ |
133 | | - DELETE FROM test_deletes WHERE deleted = True; |
| 162 | + DELETE FROM default.test_positional_mor_double_deletes WHERE letter == 'f' |
134 | 163 | """ |
135 | 164 | ) |
136 | 165 |
|
|
156 | 185 | all_types_dataframe.writeTo("default.test_all_types").tableProperty("format-version", "2").partitionedBy( |
157 | 186 | "intCol" |
158 | 187 | ).createOrReplace() |
159 | | - |
160 | | -while True: |
161 | | - time.sleep(1) |
|
0 commit comments