Skip to content

Commit 3a584a2

Browse files
authored
Python: Refactor integration tests (#7698)
* Python: Refactor integration tests This splits out running and building the integration tests that enables quick development iterations. I've also added ipython that will give a more meaningful error when something goes wrong with the provisioning of the tests. * Simplify SQL
1 parent 5eb4511 commit 3a584a2

5 files changed

Lines changed: 112 additions & 62 deletions

File tree

‎python/Makefile‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,16 @@ test-s3:
3434

3535
test-integration:
3636
docker-compose -f dev/docker-compose-integration.yml kill
37-
docker-compose -f dev/docker-compose-integration.yml build
37+
docker-compose -f dev/docker-compose-integration.yml rm -f
3838
docker-compose -f dev/docker-compose-integration.yml up -d
39-
sleep 30
40-
poetry run pytest tests/ -m integration ${PYTEST_ARGS}
39+
sleep 10
40+
docker-compose -f dev/docker-compose-integration.yml exec -T spark-iceberg ipython ./provision.py
41+
poetry run pytest tests/ -v -m integration ${PYTEST_ARGS}
42+
43+
test-integration-rebuild:
44+
docker-compose -f dev/docker-compose-integration.yml kill
45+
docker-compose -f dev/docker-compose-integration.yml rm -f
46+
docker-compose -f dev/docker-compose-integration.yml build --no-cache
4147

4248
test-adlfs:
4349
sh ./dev/run-azurite.sh

‎python/dev/Dockerfile‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME} && mkdir -p /home/iceberg/
3737
WORKDIR ${SPARK_HOME}
3838

3939
ENV SPARK_VERSION=3.3.2
40-
ENV ICEBERG_VERSION=1.2.0
40+
ENV ICEBERG_VERSION=1.2.1
4141
ENV AWS_SDK_VERSION=2.20.18
4242

4343
RUN curl -s https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \
@@ -62,6 +62,8 @@ ENV PATH="/opt/spark/sbin:/opt/spark/bin:${PATH}"
6262
RUN chmod u+x /opt/spark/sbin/* && \
6363
chmod u+x /opt/spark/bin/*
6464

65+
RUN pip3 install -q ipython
66+
6567
COPY entrypoint.sh .
6668
COPY provision.py .
6769

‎python/dev/entrypoint.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,4 +22,4 @@ start-master.sh -p 7077
2222
start-worker.sh spark://spark-iceberg:7077
2323
start-history-server.sh
2424

25-
python3 ./provision.py
25+
tail -f /dev/null

‎python/dev/provision.py‎

Lines changed: 83 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,12 @@
1414
# KIND, either express or implied. See the License for the
1515
# specific language governing permissions and limitations
1616
# under the License.
17-
import time
1817

1918
from pyspark.sql import SparkSession
2019
from pyspark.sql.functions import current_date, date_add, expr
2120

2221
spark = SparkSession.builder.getOrCreate()
2322

24-
print("Create database")
25-
2623
spark.sql(
2724
"""
2825
CREATE DATABASE IF NOT EXISTS default;
@@ -31,19 +28,7 @@
3128

3229
spark.sql(
3330
"""
34-
use default;
35-
"""
36-
)
37-
38-
spark.sql(
39-
"""
40-
DROP TABLE IF EXISTS test_null_nan;
41-
"""
42-
)
43-
44-
spark.sql(
45-
"""
46-
CREATE TABLE test_null_nan
31+
CREATE OR REPLACE TABLE default.test_null_nan
4732
USING iceberg
4833
AS SELECT
4934
1 AS idx,
@@ -59,78 +44,122 @@
5944

6045
spark.sql(
6146
"""
62-
DROP TABLE IF EXISTS test_null_nan_rewritten;
47+
CREATE OR REPLACE TABLE default.test_null_nan_rewritten
48+
USING iceberg
49+
AS SELECT * FROM default.test_null_nan
6350
"""
6451
)
6552

6653
spark.sql(
6754
"""
68-
CREATE TABLE test_null_nan_rewritten
69-
USING iceberg
70-
AS SELECT * FROM test_null_nan
55+
CREATE OR REPLACE TABLE default.test_limit as
56+
SELECT * LATERAL VIEW explode(ARRAY(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)) AS idx;
7157
"""
7258
)
7359

7460
spark.sql(
7561
"""
76-
DROP TABLE IF EXISTS test_limit;
62+
CREATE OR REPLACE TABLE default.test_positional_mor_deletes (
63+
dt date,
64+
number integer,
65+
letter string
66+
)
67+
USING iceberg
68+
TBLPROPERTIES (
69+
'write.delete.mode'='merge-on-read',
70+
'write.update.mode'='merge-on-read',
71+
'write.merge.mode'='merge-on-read',
72+
'format-version'='2'
73+
);
7774
"""
7875
)
7976

77+
# Partitioning is not really needed, but there is a bug:
78+
# https://github.com/apache/iceberg/pull/7685
8079
spark.sql(
8180
"""
82-
CREATE TABLE test_limit
83-
USING iceberg
84-
AS SELECT
85-
1 AS idx
86-
UNION ALL SELECT
87-
2 AS idx
88-
UNION ALL SELECT
89-
3 AS idx
90-
UNION ALL SELECT
91-
4 AS idx
92-
UNION ALL SELECT
93-
5 AS idx
94-
UNION ALL SELECT
95-
6 AS idx
96-
UNION ALL SELECT
97-
7 AS idx
98-
UNION ALL SELECT
99-
8 AS idx
100-
UNION ALL SELECT
101-
9 AS idx
102-
UNION ALL SELECT
103-
10 AS idx
81+
ALTER TABLE default.test_positional_mor_deletes ADD PARTITION FIELD years(dt) AS dt_years
82+
"""
83+
)
84+
85+
spark.sql(
10486
"""
87+
INSERT INTO default.test_positional_mor_deletes
88+
VALUES
89+
(CAST('2023-03-01' AS date), 1, 'a'),
90+
(CAST('2023-03-02' AS date), 2, 'b'),
91+
(CAST('2023-03-03' AS date), 3, 'c'),
92+
(CAST('2023-03-04' AS date), 4, 'd'),
93+
(CAST('2023-03-05' AS date), 5, 'e'),
94+
(CAST('2023-03-06' AS date), 6, 'f'),
95+
(CAST('2023-03-07' AS date), 7, 'g'),
96+
(CAST('2023-03-08' AS date), 8, 'h'),
97+
(CAST('2023-03-09' AS date), 9, 'i'),
98+
(CAST('2023-03-10' AS date), 10, 'j'),
99+
(CAST('2023-03-11' AS date), 11, 'k'),
100+
(CAST('2023-03-12' AS date), 12, 'l');
101+
"""
105102
)
106103

107104
spark.sql(
108105
"""
109-
DROP TABLE IF EXISTS test_deletes;
106+
DELETE FROM default.test_positional_mor_deletes WHERE number = 9
110107
"""
111108
)
112109

113110
spark.sql(
114111
"""
115-
CREATE TABLE test_deletes
112+
CREATE OR REPLACE TABLE default.test_positional_mor_double_deletes (
113+
dt date,
114+
number integer,
115+
letter string
116+
)
116117
USING iceberg
117118
TBLPROPERTIES (
118119
'write.delete.mode'='merge-on-read',
119120
'write.update.mode'='merge-on-read',
120-
'write.merge.mode'='merge-on-read'
121-
)
122-
AS SELECT
123-
1 AS idx,
124-
True AS deleted
125-
UNION ALL SELECT
126-
2 AS idx,
127-
False AS deleted;
121+
'write.merge.mode'='merge-on-read',
122+
'format-version'='2'
123+
);
124+
"""
125+
)
126+
127+
# Partitioning is not really needed, but there is a bug:
128+
# https://github.com/apache/iceberg/pull/7685
129+
spark.sql(
130+
"""
131+
ALTER TABLE default.test_positional_mor_double_deletes ADD PARTITION FIELD years(dt) AS dt_years
132+
"""
133+
)
134+
135+
spark.sql(
136+
"""
137+
INSERT INTO default.test_positional_mor_double_deletes
138+
VALUES
139+
(CAST('2023-03-01' AS date), 1, 'a'),
140+
(CAST('2023-03-02' AS date), 2, 'b'),
141+
(CAST('2023-03-03' AS date), 3, 'c'),
142+
(CAST('2023-03-04' AS date), 4, 'd'),
143+
(CAST('2023-03-05' AS date), 5, 'e'),
144+
(CAST('2023-03-06' AS date), 6, 'f'),
145+
(CAST('2023-03-07' AS date), 7, 'g'),
146+
(CAST('2023-03-08' AS date), 8, 'h'),
147+
(CAST('2023-03-09' AS date), 9, 'i'),
148+
(CAST('2023-03-10' AS date), 10, 'j'),
149+
(CAST('2023-03-11' AS date), 11, 'k'),
150+
(CAST('2023-03-12' AS date), 12, 'l');
151+
"""
152+
)
153+
154+
spark.sql(
155+
"""
156+
DELETE FROM default.test_positional_mor_double_deletes WHERE number = 9
128157
"""
129158
)
130159

131160
spark.sql(
132161
"""
133-
DELETE FROM test_deletes WHERE deleted = True;
162+
DELETE FROM default.test_positional_mor_double_deletes WHERE letter == 'f'
134163
"""
135164
)
136165

@@ -156,6 +185,3 @@
156185
all_types_dataframe.writeTo("default.test_all_types").tableProperty("format-version", "2").partitionedBy(
157186
"intCol"
158187
).createOrReplace()
159-
160-
while True:
161-
time.sleep(1)

‎python/mkdocs/docs/contributing.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,22 @@ make test PYTEST_ARGS="--pdb"
107107

108108
To see all available pytest arguments, run `make test PYTEST_ARGS="--help"`.
109109

110+
### Integration tests
111+
112+
PyIceberg has integration tests with Apache Spark. Spark will create a new database and provision some tables that PyIceberg can query against.
113+
114+
```sh
115+
make test-integration
116+
```
117+
118+
This will restart the containers, to get to a clean state, and then run the PyTest suite. In case something changed in the Dockerfile or the provision script, you can run:
119+
120+
```sh
121+
make test-integration-rebuild
122+
```
123+
124+
To rebuild the containers from scratch.
125+
110126
## Code standards
111127

112128
Below are the formalized conventions that we adhere to in the PyIceberg project. The goal of this is to have a common agreement on how to evolve the codebase, but also using it as guidelines for newcomers to the project.

0 commit comments

Comments
 (0)