Skip to content

Commit d37a8de

Browse files
mitre88potiuk
authored andcommitted
Resolve GCSFileTransformOperator destination fallbacks after rendering
destination_bucket/destination_object are template fields, so resolving their 'or source_*' fallbacks in __init__ reads the unrendered value. Move the fallback resolution to execute() where the rendered values are available, keep get_openlineage_facets_on_complete resolving fallbacks locally (it can run outside execute), and remove the operator from the validate_operators_init exemptions list. Related: apache#70296
1 parent 471b82e commit d37a8de

2 files changed

Lines changed: 6 additions & 5 deletions

File tree

‎providers/google/src/airflow/providers/google/cloud/operators/gcs.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -692,15 +692,17 @@ def __init__(
692692
super().__init__(**kwargs)
693693
self.source_bucket = source_bucket
694694
self.source_object = source_object
695-
self.destination_bucket = destination_bucket or self.source_bucket
696-
self.destination_object = destination_object or self.source_object
695+
self.destination_bucket = destination_bucket
696+
self.destination_object = destination_object
697697

698698
self.gcp_conn_id = gcp_conn_id
699699
self.transform_script = transform_script
700700
self.output_encoding = sys.getdefaultencoding()
701701
self.impersonation_chain = impersonation_chain
702702

703703
def execute(self, context: Context) -> None:
704+
self.destination_bucket = self.destination_bucket or self.source_bucket
705+
self.destination_object = self.destination_object or self.source_object
704706
hook = GCSHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
705707

706708
with NamedTemporaryFile() as source_file, NamedTemporaryFile() as destination_file:
@@ -747,8 +749,8 @@ def get_openlineage_facets_on_start(self):
747749
name=self.source_object,
748750
)
749751
output_dataset = Dataset(
750-
namespace=f"gs://{self.destination_bucket}",
751-
name=self.destination_object,
752+
namespace=f"gs://{self.destination_bucket or self.source_bucket}",
753+
name=self.destination_object or self.source_object,
752754
)
753755

754756
return OperatorLineage(inputs=[input_dataset], outputs=[output_dataset])

‎scripts/ci/prek/validate_operators_init_exemptions.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,5 @@ providers/google/src/airflow/providers/google/cloud/operators/cloud_storage_tran
1212
providers/google/src/airflow/providers/google/cloud/operators/dataproc.py::DataprocCreateClusterOperator
1313
providers/google/src/airflow/providers/google/cloud/operators/dataproc.py::DataprocSubmitJobOperator
1414
providers/google/src/airflow/providers/google/cloud/operators/functions.py::CloudFunctionDeployFunctionOperator
15-
providers/google/src/airflow/providers/google/cloud/operators/gcs.py::GCSFileTransformOperator
1615
providers/google/src/airflow/providers/google/cloud/sensors/bigquery_dts.py::BigQueryDataTransferServiceTransferRunSensor
1716
providers/google/src/airflow/providers/google/cloud/sensors/cloud_composer.py::CloudComposerExternalTaskSensor

0 commit comments

Comments
 (0)