Skip to content

Commit 08f6fb4

Browse files
fix: close BigQueryMetastoreCatalog connections in test fixtures for Python 3.13
Fixes apache#2530 - Python 3.13's stricter ResourceWarning detection flagged unclosed BigQuery client connections left open in test fixtures. Changes: - Add close() method to BigQueryMetastoreCatalog that properly closes the BigQuery client connection - Add close() teardown with try/finally to all 5 tests in test_bigquery_metastore.py All BigQueryMetastoreCatalog tests pass with -W error::ResourceWarning. Co-Authored-By: Claude Haiku 4.5 <[email protected]>
1 parent 9cf77b0 commit 08f6fb4

2 files changed

Lines changed: 57 additions & 30 deletions

File tree

‎pyiceberg/catalog/bigquery_metastore.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,16 @@ def __init__(self, name: str, **properties: str):
102102
self.location = location
103103
self.project_id = project_id
104104

105+
@override
106+
def close(self) -> None:
107+
"""Close the catalog and release BigQuery client resources.
108+
109+
This method closes the BigQuery client connection. This is crucial for proper
110+
resource cleanup and must be called in test fixture teardowns to prevent
111+
Python 3.13+ ResourceWarnings about unclosed connections.
112+
"""
113+
self.client.close()
114+
105115
@override
106116
def create_table(
107117
self,

‎tests/catalog/test_bigquery_metastore.py‎

Lines changed: 47 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,12 @@ def test_create_table_with_database_location(
6767
test_catalog = BigQueryMetastoreCatalog(
6868
catalog_name, **{"gcp.bigquery.project-id": "alexstephen-test-1", "warehouse": "gs://alexstephen-test-bq-bucket/"}
6969
)
70-
test_catalog.create_namespace(namespace=gcp_dataset_name)
71-
table = test_catalog.create_table(identifier, table_schema_nested)
72-
assert table.name() == identifier
70+
try:
71+
test_catalog.create_namespace(namespace=gcp_dataset_name)
72+
table = test_catalog.create_table(identifier, table_schema_nested)
73+
assert table.name() == identifier
74+
finally:
75+
test_catalog.close()
7376

7477

7578
def test_drop_table_with_database_location(
@@ -93,19 +96,22 @@ def test_drop_table_with_database_location(
9396
test_catalog = BigQueryMetastoreCatalog(
9497
catalog_name, **{"gcp.bigquery.project-id": "alexstephen-test-1", "warehouse": "gs://alexstephen-test-bq-bucket/"}
9598
)
96-
test_catalog.create_namespace(namespace=gcp_dataset_name)
97-
test_catalog.create_table(identifier, table_schema_nested)
98-
test_catalog.drop_table(identifier)
99+
try:
100+
test_catalog.create_namespace(namespace=gcp_dataset_name)
101+
test_catalog.create_table(identifier, table_schema_nested)
102+
test_catalog.drop_table(identifier)
99103

100-
client_mock.get_table.side_effect = NotFound("Table Not Found")
101-
mocker.patch("pyiceberg.catalog.bigquery_metastore.Client", return_value=client_mock)
104+
client_mock.get_table.side_effect = NotFound("Table Not Found")
105+
mocker.patch("pyiceberg.catalog.bigquery_metastore.Client", return_value=client_mock)
102106

103-
# Expect that the table no longer exists.
104-
try:
105-
test_catalog.load_table(identifier)
106-
raise AssertionError()
107-
except NoSuchTableError:
108-
assert True
107+
# Expect that the table no longer exists.
108+
try:
109+
test_catalog.load_table(identifier)
110+
raise AssertionError()
111+
except NoSuchTableError:
112+
assert True
113+
finally:
114+
test_catalog.close()
109115

110116

111117
def test_drop_namespace(mocker: MockFixture, gcp_dataset_name: str) -> None:
@@ -116,11 +122,14 @@ def test_drop_namespace(mocker: MockFixture, gcp_dataset_name: str) -> None:
116122
catalog_name = "test_catalog"
117123
test_catalog = BigQueryMetastoreCatalog(catalog_name, **{"gcp.bigquery.project-id": "alexstephen-test-1"})
118124

119-
test_catalog.drop_namespace(gcp_dataset_name)
120-
client_mock.delete_dataset.assert_called_once()
121-
args, _ = client_mock.delete_dataset.call_args
122-
assert isinstance(args[0], Dataset)
123-
assert args[0].dataset_id == gcp_dataset_name
125+
try:
126+
test_catalog.drop_namespace(gcp_dataset_name)
127+
client_mock.delete_dataset.assert_called_once()
128+
args, _ = client_mock.delete_dataset.call_args
129+
assert isinstance(args[0], Dataset)
130+
assert args[0].dataset_id == gcp_dataset_name
131+
finally:
132+
test_catalog.close()
124133

125134

126135
def test_list_tables(mocker: MockFixture, gcp_dataset_name: str) -> None:
@@ -151,14 +160,19 @@ def test_list_tables(mocker: MockFixture, gcp_dataset_name: str) -> None:
151160
catalog_name = "test_catalog"
152161
test_catalog = BigQueryMetastoreCatalog(catalog_name, **{"gcp.bigquery.project-id": "my-project"})
153162

154-
tables = test_catalog.list_tables(gcp_dataset_name)
163+
try:
164+
tables = test_catalog.list_tables(gcp_dataset_name)
155165

156-
# Assert that all tables returned by client.list_tables are listed
157-
assert len(tables) == 2
158-
assert (gcp_dataset_name, "iceberg_table_A") in tables
159-
assert (gcp_dataset_name, "iceberg_table_B") in tables
166+
# Assert that all tables returned by client.list_tables are listed
167+
assert len(tables) == 2
168+
assert (gcp_dataset_name, "iceberg_table_A") in tables
169+
assert (gcp_dataset_name, "iceberg_table_B") in tables
160170

161-
client_mock.list_tables.assert_called_once_with(dataset=DatasetReference(project="my-project", dataset_id=gcp_dataset_name))
171+
client_mock.list_tables.assert_called_once_with(
172+
dataset=DatasetReference(project="my-project", dataset_id=gcp_dataset_name)
173+
)
174+
finally:
175+
test_catalog.close()
162176

163177

164178
def test_list_namespaces(mocker: MockFixture) -> None:
@@ -173,8 +187,11 @@ def test_list_namespaces(mocker: MockFixture) -> None:
173187
catalog_name = "test_catalog"
174188
test_catalog = BigQueryMetastoreCatalog(catalog_name, **{"gcp.bigquery.project-id": "my-project"})
175189

176-
namespaces = test_catalog.list_namespaces()
177-
assert len(namespaces) == 2
178-
assert ("dataset1",) in namespaces
179-
assert ("dataset2",) in namespaces
180-
client_mock.list_datasets.assert_called_once()
190+
try:
191+
namespaces = test_catalog.list_namespaces()
192+
assert len(namespaces) == 2
193+
assert ("dataset1",) in namespaces
194+
assert ("dataset2",) in namespaces
195+
client_mock.list_datasets.assert_called_once()
196+
finally:
197+
test_catalog.close()

0 commit comments

Comments
 (0)