Skip to content

Commit 76bed66

Browse files
yuseok89potiuk
andauthored
Add GCSToAzureBlobStorageOperator for GCS to Azure Blob transfer (#64966)
* Add GCSToAzureBlobStorageOperator for GCS to Azure Blob transfer * Fix CI * Fix CI error * Replace AirflowException with ValueError * Update providers/microsoft/azure/src/airflow/providers/microsoft/azure/transfers/gcs_to_wasb.py Co-authored-by: Jarek Potiuk <[email protected]> * Update providers/microsoft/azure/src/airflow/providers/microsoft/azure/transfers/gcs_to_wasb.py Co-authored-by: Jarek Potiuk <[email protected]> * Avoid name mangling on _is_match_glob_supported attribute * Remove redundant _transform_file_path filter clause * Fix self.blob_prefix mutation on execute() retry * Fix silent overwrite of colliding blobs when replace=True * Fix Windows backslash in Azure blob names --------- Co-authored-by: Jarek Potiuk <[email protected]>
1 parent 75cba29 commit 76bed66

9 files changed

Lines changed: 624 additions & 3 deletions

File tree

‎dev/breeze/tests/test_selective_checks.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2484,7 +2484,7 @@ def test_expected_output_push(
24842484
),
24852485
{
24862486
"selected-providers-list-as-string": "amazon common.compat common.io common.sql "
2487-
"databricks dbt.cloud ftp google jdbc microsoft.mssql mysql "
2487+
"databricks dbt.cloud ftp google jdbc microsoft.azure microsoft.mssql mysql "
24882488
"openlineage oracle postgres sftp snowflake standard trino",
24892489
"all-python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
24902490
"all-python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
@@ -2503,7 +2503,7 @@ def test_expected_output_push(
25032503
{
25042504
"description": "amazon...standard",
25052505
"test_types": "Providers[amazon] Providers[common.compat,common.io,common.sql,"
2506-
"databricks,dbt.cloud,ftp,jdbc,microsoft.mssql,mysql,openlineage,oracle,"
2506+
"databricks,dbt.cloud,ftp,jdbc,microsoft.azure,microsoft.mssql,mysql,openlineage,oracle,"
25072507
"postgres,sftp,snowflake,trino] Providers[google] Providers[standard]",
25082508
}
25092509
]

‎providers/microsoft/azure/docs/index.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,8 @@ Dependent package
159159
`apache-airflow-providers-amazon <https://airflow.apache.org/docs/apache-airflow-providers-amazon>`_ ``amazon``
160160
`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_ ``common.compat``
161161
`apache-airflow-providers-common-messaging <https://airflow.apache.org/docs/apache-airflow-providers-common-messaging>`_ ``common.messaging``
162+
`apache-airflow-providers-google <https://airflow.apache.org/docs/apache-airflow-providers-google>`_ ``google``
163+
`apache-airflow-providers-openlineage <https://airflow.apache.org/docs/apache-airflow-providers-openlineage>`_ ``openlineage``
162164
`apache-airflow-providers-oracle <https://airflow.apache.org/docs/apache-airflow-providers-oracle>`_ ``oracle``
163165
`apache-airflow-providers-sftp <https://airflow.apache.org/docs/apache-airflow-providers-sftp>`_ ``sftp``
164166
======================================================================================================================== ====================
Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
2+
.. Licensed to the Apache Software Foundation (ASF) under one
3+
or more contributor license agreements. See the NOTICE file
4+
distributed with this work for additional information
5+
regarding copyright ownership. The ASF licenses this file
6+
to you under the Apache License, Version 2.0 (the
7+
"License"); you may not use this file except in compliance
8+
with the License. You may obtain a copy of the License at
9+
10+
.. http://www.apache.org/licenses/LICENSE-2.0
11+
12+
.. Unless required by applicable law or agreed to in writing,
13+
software distributed under the License is distributed on an
14+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
KIND, either express or implied. See the License for the
16+
specific language governing permissions and limitations
17+
under the License.
18+
19+
=====================================================
20+
Google Cloud Storage to Azure Blob Storage transfer
21+
=====================================================
22+
23+
`Google Cloud Storage <https://cloud.google.com/storage/>`__ and
24+
`Azure Blob Storage <https://learn.microsoft.com/en-us/azure/storage/blobs/>`__
25+
are object stores commonly used for data lakes and file exchange.
26+
This guide describes copying objects from GCS into an Azure Blob container.
27+
28+
Install the optional dependency when using this operator:
29+
30+
.. code-block:: bash
31+
32+
pip install 'apache-airflow-providers-microsoft-azure[google]'
33+
34+
Prerequisite Tasks
35+
------------------
36+
37+
.. include:: ../operators/_partials/prerequisite_tasks.rst
38+
39+
.. _howto/operator:GCSToAzureBlobStorageOperator:
40+
41+
Operator
42+
--------
43+
44+
Use :class:`~airflow.providers.microsoft.azure.transfers.gcs_to_wasb.GCSToAzureBlobStorageOperator`
45+
to list objects under a GCS ``prefix`` and upload them to a container using ``blob_prefix`` as the base path.
46+
Use ``keep_directory_structure`` and ``flatten_structure`` the same way as
47+
:class:`~airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSToS3Operator` (``flatten_structure`` wins when both apply).
48+
Object keys ending with ``/`` (GCS console folder markers) are not copied.
49+
50+
Example:
51+
52+
.. code-block:: python
53+
54+
copy_gcs_to_azure = GCSToAzureBlobStorageOperator(
55+
task_id="gcs_to_azure_blob",
56+
gcs_bucket="my-gcs-bucket",
57+
prefix="exports/daily/",
58+
container_name="my-container",
59+
blob_prefix="imports/daily",
60+
gcp_conn_id="google_cloud_default",
61+
wasb_conn_id="wasb_default",
62+
replace=True,
63+
)
64+
65+
Reference
66+
---------
67+
68+
* `Google Cloud Storage Python client <https://cloud.google.com/python/docs/reference/storage/latest>`__
69+
* `Azure Blob Storage client library <https://learn.microsoft.com/en-us/python/api/overview/azure/storage-blob-readme?view=azure-python>`__

‎providers/microsoft/azure/provider.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,10 @@ transfers:
362362
target-integration-name: Microsoft Azure Blob Storage
363363
how-to-guide: /docs/apache-airflow-providers-microsoft-azure/transfer/s3_to_wasb.rst
364364
python-module: airflow.providers.microsoft.azure.transfers.s3_to_wasb
365+
- source-integration-name: Google Cloud Storage (GCS)
366+
target-integration-name: Microsoft Azure Blob Storage
367+
how-to-guide: /docs/apache-airflow-providers-microsoft-azure/transfer/gcs_to_wasb.rst
368+
python-module: airflow.providers.microsoft.azure.transfers.gcs_to_wasb
365369

366370

367371
connection-types:

‎providers/microsoft/azure/pyproject.toml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,12 @@ dependencies = [
113113
"common.messaging" = [
114114
"apache-airflow-providers-common-messaging>=2.0.0"
115115
]
116+
"google" = [
117+
"apache-airflow-providers-google"
118+
]
119+
"openlineage" = [
120+
"apache-airflow-providers-openlineage>=2.3.0"
121+
]
116122

117123
[dependency-groups]
118124
dev = [
@@ -122,6 +128,8 @@ dev = [
122128
"apache-airflow-providers-amazon",
123129
"apache-airflow-providers-common-compat",
124130
"apache-airflow-providers-common-messaging",
131+
"apache-airflow-providers-google",
132+
"apache-airflow-providers-openlineage",
125133
"apache-airflow-providers-oracle",
126134
"apache-airflow-providers-sftp",
127135
# Additional devel dependencies (do not remove this line and add extra development dependencies)

‎providers/microsoft/azure/src/airflow/providers/microsoft/azure/get_provider_info.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -352,6 +352,12 @@ def get_provider_info():
352352
"how-to-guide": "/docs/apache-airflow-providers-microsoft-azure/transfer/s3_to_wasb.rst",
353353
"python-module": "airflow.providers.microsoft.azure.transfers.s3_to_wasb",
354354
},
355+
{
356+
"source-integration-name": "Google Cloud Storage (GCS)",
357+
"target-integration-name": "Microsoft Azure Blob Storage",
358+
"how-to-guide": "/docs/apache-airflow-providers-microsoft-azure/transfer/gcs_to_wasb.rst",
359+
"python-module": "airflow.providers.microsoft.azure.transfers.gcs_to_wasb",
360+
},
355361
],
356362
"connection-types": [
357363
{

0 commit comments

Comments
 (0)