Skip to content

Commit eea26cd

Browse files
mitre88potiuk
authored andcommitted
Template the Neptune cluster operators' db_cluster_id argument under its own name
NeptuneStartDbClusterOperator and NeptuneStopDbClusterOperator took db_cluster_id but templated and stored it as cluster_id, which the operators-init check flags. Keeping the constructor argument and renaming the attribute and template field leaves existing Dags passing db_cluster_id= working, as the AWS provider maintainers asked. related: apache#70296 Generated-by: Claude Opus 5
1 parent a1f1a49 commit eea26cd

2 files changed

Lines changed: 38 additions & 38 deletions

File tree

‎providers/amazon/src/airflow/providers/amazon/aws/operators/neptune.py‎

Lines changed: 38 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -50,36 +50,36 @@ def handle_waitable_exception(
5050

5151
if code in ("InvalidDBInstanceStateFault", "InvalidDBInstanceState"):
5252
if operator.deferrable:
53-
operator.log.info("Deferring until instances become available: %s", operator.cluster_id)
53+
operator.log.info("Deferring until instances become available: %s", operator.db_cluster_id)
5454
operator.defer(
5555
trigger=NeptuneClusterInstancesAvailableTrigger(
5656
aws_conn_id=operator.aws_conn_id,
57-
db_cluster_id=operator.cluster_id,
57+
db_cluster_id=operator.db_cluster_id,
5858
region_name=operator.region_name,
5959
botocore_config=operator.botocore_config,
6060
verify=operator.verify,
6161
),
6262
method_name="execute",
6363
)
6464
else:
65-
operator.log.info("Need to wait for instances to become available: %s", operator.cluster_id)
66-
operator.hook.wait_for_cluster_instance_availability(cluster_id=operator.cluster_id)
65+
operator.log.info("Need to wait for instances to become available: %s", operator.db_cluster_id)
66+
operator.hook.wait_for_cluster_instance_availability(cluster_id=operator.db_cluster_id)
6767
if code in ["InvalidClusterState", "InvalidDBClusterStateFault"]:
6868
if operator.deferrable:
69-
operator.log.info("Deferring until cluster becomes available: %s", operator.cluster_id)
69+
operator.log.info("Deferring until cluster becomes available: %s", operator.db_cluster_id)
7070
operator.defer(
7171
trigger=NeptuneClusterAvailableTrigger(
7272
aws_conn_id=operator.aws_conn_id,
73-
db_cluster_id=operator.cluster_id,
73+
db_cluster_id=operator.db_cluster_id,
7474
region_name=operator.region_name,
7575
botocore_config=operator.botocore_config,
7676
verify=operator.verify,
7777
),
7878
method_name="execute",
7979
)
8080
else:
81-
operator.log.info("Need to wait for cluster to become available: %s", operator.cluster_id)
82-
operator.hook.wait_for_cluster_availability(operator.cluster_id)
81+
operator.log.info("Need to wait for cluster to become available: %s", operator.db_cluster_id)
82+
operator.hook.wait_for_cluster_availability(operator.db_cluster_id)
8383

8484

8585
class NeptuneStartDbClusterOperator(AwsBaseOperator[NeptuneHook]):
@@ -116,7 +116,7 @@ class NeptuneStartDbClusterOperator(AwsBaseOperator[NeptuneHook]):
116116
"""
117117

118118
aws_hook_class = NeptuneHook
119-
template_fields: Sequence[str] = aws_template_fields("cluster_id")
119+
template_fields: Sequence[str] = aws_template_fields("db_cluster_id")
120120

121121
def __init__(
122122
self,
@@ -128,26 +128,26 @@ def __init__(
128128
**kwargs,
129129
):
130130
super().__init__(**kwargs)
131-
self.cluster_id = db_cluster_id
131+
self.db_cluster_id = db_cluster_id
132132
self.wait_for_completion = wait_for_completion
133133
self.deferrable = deferrable
134134
self.waiter_delay = waiter_delay
135135
self.waiter_max_attempts = waiter_max_attempts
136136

137137
def execute(self, context: Context, event: dict[str, Any] | None = None, **kwargs) -> dict[str, str]:
138-
self.log.info("Starting Neptune cluster: %s", self.cluster_id)
138+
self.log.info("Starting Neptune cluster: %s", self.db_cluster_id)
139139

140140
# Check to make sure the cluster is not already available.
141-
status = self.hook.get_cluster_status(self.cluster_id)
141+
status = self.hook.get_cluster_status(self.db_cluster_id)
142142
if status.lower() in NeptuneHook.AVAILABLE_STATES:
143-
self.log.info("Neptune cluster %s is already available.", self.cluster_id)
144-
return {"db_cluster_id": self.cluster_id}
143+
self.log.info("Neptune cluster %s is already available.", self.db_cluster_id)
144+
return {"db_cluster_id": self.db_cluster_id}
145145
if status.lower() in NeptuneHook.ERROR_STATES:
146146
# some states will not allow you to start the cluster
147147
self.log.error(
148-
"Neptune cluster %s is in error state %s and cannot be started", self.cluster_id, status
148+
"Neptune cluster %s is in error state %s and cannot be started", self.db_cluster_id, status
149149
)
150-
raise AirflowException(f"Neptune cluster {self.cluster_id} is in error state {status}")
150+
raise AirflowException(f"Neptune cluster {self.db_cluster_id} is in error state {status}")
151151

152152
"""
153153
A cluster and its instances must be in a valid state to send the start request.
@@ -157,7 +157,7 @@ def execute(self, context: Context, event: dict[str, Any] | None = None, **kwarg
157157
Let the waiters handle retries and detecting the error states.
158158
"""
159159
try:
160-
self.hook.conn.start_db_cluster(DBClusterIdentifier=self.cluster_id)
160+
self.hook.conn.start_db_cluster(DBClusterIdentifier=self.db_cluster_id)
161161
except ClientError as ex:
162162
code = ex.response["Error"]["Code"]
163163
self.log.warning("Received client error when attempting to start the cluster: %s", code)
@@ -170,12 +170,12 @@ def execute(self, context: Context, event: dict[str, Any] | None = None, **kwarg
170170
raise
171171

172172
if should_defer(deferrable=self.deferrable, wait_for_completion=self.wait_for_completion):
173-
self.log.info("Deferring for cluster start: %s", self.cluster_id)
173+
self.log.info("Deferring for cluster start: %s", self.db_cluster_id)
174174

175175
self.defer(
176176
trigger=NeptuneClusterAvailableTrigger(
177177
aws_conn_id=self.aws_conn_id,
178-
db_cluster_id=self.cluster_id,
178+
db_cluster_id=self.db_cluster_id,
179179
waiter_delay=self.waiter_delay,
180180
waiter_max_attempts=self.waiter_max_attempts,
181181
region_name=self.region_name,
@@ -186,12 +186,12 @@ def execute(self, context: Context, event: dict[str, Any] | None = None, **kwarg
186186
)
187187

188188
elif self.wait_for_completion:
189-
self.log.info("Waiting for Neptune cluster %s to start.", self.cluster_id)
189+
self.log.info("Waiting for Neptune cluster %s to start.", self.db_cluster_id)
190190
self.hook.wait_for_cluster_availability(
191-
self.cluster_id, self.waiter_delay, self.waiter_max_attempts
191+
self.db_cluster_id, self.waiter_delay, self.waiter_max_attempts
192192
)
193193

194-
return {"db_cluster_id": self.cluster_id}
194+
return {"db_cluster_id": self.db_cluster_id}
195195

196196
def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> dict[str, str]:
197197
validated_event = validate_execute_complete_event(event)
@@ -238,7 +238,7 @@ class NeptuneStopDbClusterOperator(AwsBaseOperator[NeptuneHook]):
238238
"""
239239

240240
aws_hook_class = NeptuneHook
241-
template_fields: Sequence[str] = aws_template_fields("cluster_id")
241+
template_fields: Sequence[str] = aws_template_fields("db_cluster_id")
242242

243243
def __init__(
244244
self,
@@ -250,28 +250,28 @@ def __init__(
250250
**kwargs,
251251
):
252252
super().__init__(**kwargs)
253-
self.cluster_id = db_cluster_id
253+
self.db_cluster_id = db_cluster_id
254254
self.wait_for_completion = wait_for_completion
255255
self.deferrable = deferrable
256256
self.waiter_delay = waiter_delay
257257
self.waiter_max_attempts = waiter_max_attempts
258258

259259
def execute(self, context: Context, event: dict[str, Any] | None = None, **kwargs) -> dict[str, str]:
260-
self.log.info("Stopping Neptune cluster: %s", self.cluster_id)
260+
self.log.info("Stopping Neptune cluster: %s", self.db_cluster_id)
261261

262262
# Check to make sure the cluster is not already stopped or that its not in a bad state
263-
status = self.hook.get_cluster_status(self.cluster_id)
263+
status = self.hook.get_cluster_status(self.db_cluster_id)
264264
self.log.info("Current status: %s", status)
265265

266266
if status.lower() in NeptuneHook.STOPPED_STATES:
267-
self.log.info("Neptune cluster %s is already stopped.", self.cluster_id)
268-
return {"db_cluster_id": self.cluster_id}
267+
self.log.info("Neptune cluster %s is already stopped.", self.db_cluster_id)
268+
return {"db_cluster_id": self.db_cluster_id}
269269
if status.lower() in NeptuneHook.ERROR_STATES:
270270
# some states will not allow you to stop the cluster
271271
self.log.error(
272-
"Neptune cluster %s is in error state %s and cannot be stopped", self.cluster_id, status
272+
"Neptune cluster %s is in error state %s and cannot be stopped", self.db_cluster_id, status
273273
)
274-
raise AirflowException(f"Neptune cluster {self.cluster_id} is in error state {status}")
274+
raise AirflowException(f"Neptune cluster {self.db_cluster_id} is in error state {status}")
275275

276276
"""
277277
A cluster and its instances must be in a valid state to send the stop request.
@@ -282,7 +282,7 @@ def execute(self, context: Context, event: dict[str, Any] | None = None, **kwarg
282282
"""
283283

284284
try:
285-
self.hook.conn.stop_db_cluster(DBClusterIdentifier=self.cluster_id)
285+
self.hook.conn.stop_db_cluster(DBClusterIdentifier=self.db_cluster_id)
286286

287287
# cluster must be in available state to stop it
288288
except ClientError as ex:
@@ -302,12 +302,12 @@ def execute(self, context: Context, event: dict[str, Any] | None = None, **kwarg
302302
raise
303303

304304
if should_defer(deferrable=self.deferrable, wait_for_completion=self.wait_for_completion):
305-
self.log.info("Deferring for cluster stop: %s", self.cluster_id)
305+
self.log.info("Deferring for cluster stop: %s", self.db_cluster_id)
306306

307307
self.defer(
308308
trigger=NeptuneClusterStoppedTrigger(
309309
aws_conn_id=self.aws_conn_id,
310-
db_cluster_id=self.cluster_id,
310+
db_cluster_id=self.db_cluster_id,
311311
waiter_delay=self.waiter_delay,
312312
waiter_max_attempts=self.waiter_max_attempts,
313313
region_name=self.region_name,
@@ -318,11 +318,13 @@ def execute(self, context: Context, event: dict[str, Any] | None = None, **kwarg
318318
)
319319

320320
elif self.wait_for_completion:
321-
self.log.info("Waiting for Neptune cluster %s to stop.", self.cluster_id)
321+
self.log.info("Waiting for Neptune cluster %s to stop.", self.db_cluster_id)
322322

323-
self.hook.wait_for_cluster_stopped(self.cluster_id, self.waiter_delay, self.waiter_max_attempts)
323+
self.hook.wait_for_cluster_stopped(
324+
self.db_cluster_id, self.waiter_delay, self.waiter_max_attempts
325+
)
324326

325-
return {"db_cluster_id": self.cluster_id}
327+
return {"db_cluster_id": self.db_cluster_id}
326328

327329
def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> dict[str, str]:
328330
validated_event = validate_execute_complete_event(event)

‎scripts/ci/prek/validate_operators_init_exemptions.txt‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,4 @@
55
#
66
# Fixing a class MUST remove its entry in the same PR — the hook fails on stale entries.
77
# Burn-down tracked at https://github.com/apache/airflow/issues/70296
8-
providers/amazon/src/airflow/providers/amazon/aws/operators/neptune.py::NeptuneStartDbClusterOperator
9-
providers/amazon/src/airflow/providers/amazon/aws/operators/neptune.py::NeptuneStopDbClusterOperator
108
providers/google/src/airflow/providers/google/cloud/operators/dataproc.py::DataprocCreateClusterOperator

0 commit comments

Comments
 (0)