Skip to content

Commit fd80b0d

Browse files
committed
Remove local_settings change, split into separate PR apache#65191
1 parent 10e7ba0 commit fd80b0d

3 files changed

Lines changed: 111 additions & 53 deletions

File tree

‎airflow-core/src/airflow/config_templates/airflow_local_settings.py‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -241,25 +241,19 @@ def _default_conn_name_from(mod_path, hook_name):
241241
)
242242
_handler_kwargs = {}
243243
elif remote_base_log_folder.startswith("stackdriver://"):
244-
from airflow.providers.google.cloud.log.stackdriver_task_handler import StackdriverRemoteLogIO
245-
246244
key_path = conf.get_mandatory_value("logging", "GOOGLE_KEY_PATH", fallback=None)
247245
# stackdriver:///airflow-tasks => airflow-tasks
248246
log_name = urlsplit(remote_base_log_folder).path[1:]
247+
STACKDRIVER_REMOTE_HANDLERS = {
248+
"task": {
249+
"class": "airflow.providers.google.cloud.log.stackdriver_task_handler.StackdriverTaskHandler",
250+
"formatter": "airflow",
251+
"gcp_log_name": log_name,
252+
"gcp_key_path": key_path,
253+
}
254+
}
249255

250-
REMOTE_TASK_LOG = StackdriverRemoteLogIO(
251-
**cast(
252-
"dict[str, Any]",
253-
{
254-
"base_log_folder": BASE_LOG_FOLDER,
255-
"gcp_log_name": log_name,
256-
"gcp_key_path": key_path,
257-
"delete_local_copy": delete_local_copy,
258-
}
259-
| _handler_kwargs,
260-
)
261-
)
262-
_handler_kwargs = {}
256+
DEFAULT_LOGGING_CONFIG["handlers"].update(STACKDRIVER_REMOTE_HANDLERS)
263257
elif remote_base_log_folder.startswith("oss://"):
264258
from airflow.providers.alibaba.cloud.log.oss_task_handler import OSSRemoteLogIO
265259

‎providers/google/src/airflow/providers/google/cloud/log/stackdriver_task_handler.py‎

Lines changed: 40 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@
2525
import shutil
2626
import warnings
2727
from collections.abc import Collection
28+
from datetime import datetime
2829
from functools import cached_property
30+
from logging import getLogRecordFactory
2931
from pathlib import Path
3032
from typing import TYPE_CHECKING
3133
from urllib.parse import urlencode
@@ -87,7 +89,7 @@ class StackdriverRemoteLogIO(LoggingMixin):
8789
labels: dict[str, str] | None = None
8890

8991
@cached_property
90-
def _credentials_and_project(self) -> tuple[Credentials, str]:
92+
def credentials_and_project(self) -> tuple[Credentials, str]:
9193
credentials, project = get_credentials_and_project_id(
9294
key_path=self.gcp_key_path, scopes=self.scopes, disable_logging=True
9395
)
@@ -96,7 +98,7 @@ def _credentials_and_project(self) -> tuple[Credentials, str]:
9698
@cached_property
9799
def _client(self) -> gcp_logging.Client:
98100
"""The Cloud Library API client."""
99-
credentials, project = self._credentials_and_project
101+
credentials, project = self.credentials_and_project
100102
return gcp_logging.Client(
101103
credentials=credentials,
102104
project=project,
@@ -106,29 +108,26 @@ def _client(self) -> gcp_logging.Client:
106108
@cached_property
107109
def _logging_service_client(self) -> LoggingServiceV2Client:
108110
"""The Cloud logging service v2 client."""
109-
credentials, _ = self._credentials_and_project
111+
credentials, _ = self.credentials_and_project
110112
return LoggingServiceV2Client(
111113
credentials=credentials,
112114
client_info=CLIENT_INFO,
113115
)
114116

115117
@cached_property
116-
def _transport(self) -> Transport:
118+
def transport(self) -> Transport:
117119
"""Object responsible for sending data to Stackdriver."""
118120
return self.transport_type(self._client, self.gcp_log_name)
119121

120122
@cached_property
121123
def processors(self) -> tuple[structlog.typing.Processor, ...]:
122-
from datetime import datetime
123-
from logging import getLogRecordFactory
124-
125124
import structlog.stdlib
126125

127-
logRecordFactory = getLogRecordFactory()
128-
_transport = self._transport
129-
130126
from airflow.sdk.log import relative_path_from_logger
131127

128+
log_record_factory = getLogRecordFactory()
129+
_transport = self.transport
130+
132131
def proc(
133132
logger: structlog.typing.WrappedLogger,
134133
method_name: str,
@@ -144,7 +143,7 @@ def proc(
144143
if ts := msg.pop("timestamp", None):
145144
with contextlib.suppress(Exception):
146145
created = datetime.fromisoformat(ts)
147-
record = logRecordFactory(
146+
record = log_record_factory(
148147
name,
149148
level,
150149
pathname="",
@@ -159,16 +158,21 @@ def proc(
159158
ct = created.timestamp()
160159
record.created = ct
161160
record.msecs = int((ct - int(ct)) * 1000) + 0.0
162-
_transport.send(
163-
record, str(msg.get("event", "")), resource=self.resource, labels=self.labels or {}
164-
)
161+
162+
ti = getattr(record, "task_instance", None)
163+
labels: dict[str, str] = {}
164+
if self.labels:
165+
labels.update(self.labels)
166+
if ti:
167+
labels.update(_task_instance_to_labels(ti))
168+
_transport.send(record, str(msg.get("event", "")), resource=self.resource, labels=labels)
165169
return event
166170

167171
return (proc,)
168172

169173
def upload(self, path: os.PathLike | str, ti: RuntimeTI) -> None:
170174
"""Flush the transport and optionally delete local log files."""
171-
self._transport.flush()
175+
self.transport.flush()
172176
if self.delete_local_copy:
173177
base = self.base_log_folder.resolve()
174178
raw = Path(path)
@@ -189,19 +193,19 @@ def upload(self, path: os.PathLike | str, ti: RuntimeTI) -> None:
189193
def read(self, relative_path: str, ti: RuntimeTI) -> LogResponse:
190194
"""Read logs from Stackdriver Logging using task instance labels."""
191195
ti_labels = _task_instance_to_labels(ti)
192-
log_filter = self._prepare_log_filter(ti_labels)
193-
messages, end_of_log, _ = self._read_logs(log_filter, next_page_token=None, all_pages=True)
196+
log_filter = self.prepare_log_filter(ti_labels)
197+
messages, end_of_log, _ = self.read_logs(log_filter, next_page_token=None, all_pages=True)
194198
return [f"Reading remote log from Stackdriver for {relative_path}"], [messages] if messages else []
195199

196-
def _prepare_log_filter(self, ti_labels: dict[str, str]) -> str:
200+
def prepare_log_filter(self, ti_labels: dict[str, str]) -> str:
197201
def escape_label_key(key: str) -> str:
198202
return f'"{key}"' if "." in key else key
199203

200204
def escape_label_value(value: str) -> str:
201205
escaped_value = value.replace("\\", "\\\\").replace('"', '\\"')
202206
return f'"{escaped_value}"'
203207

204-
_, project = self._credentials_and_project
208+
_, project = self.credentials_and_project
205209
log_filters = [
206210
f"resource.type={escape_label_value(self.resource.type)}",
207211
f'logName="projects/{project}/logs/{self.gcp_log_name}"',
@@ -214,7 +218,7 @@ def escape_label_value(value: str) -> str:
214218
log_filters.append(f"labels.{escape_label_key(key)}={escape_label_value(value)}")
215219
return "\n".join(log_filters)
216220

217-
def _read_logs(
221+
def read_logs(
218222
self, log_filter: str, next_page_token: str | None, all_pages: bool
219223
) -> tuple[str, bool, str | None]:
220224
messages = []
@@ -229,8 +233,6 @@ def _read_logs(
229233
log_filter=log_filter, page_token=next_page_token
230234
)
231235
messages.append(new_messages)
232-
if not messages:
233-
break
234236

235237
end_of_log = True
236238
next_page_token = None
@@ -239,7 +241,7 @@ def _read_logs(
239241
return "\n".join(messages), end_of_log, next_page_token
240242

241243
def _read_single_logs_page(self, log_filter: str, page_token: str | None = None) -> tuple[str, str]:
242-
_, project = self._credentials_and_project
244+
_, project = self.credentials_and_project
243245
request = ListLogEntriesRequest(
244246
resource_names=[f"projects/{project}"],
245247
filter=log_filter,
@@ -301,6 +303,7 @@ class StackdriverTaskHandler(logging.Handler):
301303
:param labels: (Optional) Mapping of labels for the entry.
302304
"""
303305

306+
# Re-export module-level constants for back-compat with external code reading them off the class
304307
LABEL_TASK_ID = LABEL_TASK_ID
305308
LABEL_DAG_ID = LABEL_DAG_ID
306309
LABEL_LOGICAL_DATE = LABEL_LOGICAL_DATE
@@ -376,7 +379,7 @@ def emit(self, record: logging.LogRecord) -> None:
376379
if not AIRFLOW_V_3_0_PLUS and getattr(record, ctx_indiv_trigger.name, None):
377380
ti = getattr(record, "task_instance", None) # trigger context
378381
labels = self._get_labels(ti)
379-
self.io._transport.send(record, message, resource=self.resource, labels=labels)
382+
self.io.transport.send(record, message, resource=self.resource, labels=labels)
380383

381384
def set_context(self, task_instance: TaskInstance) -> None:
382385
"""
@@ -415,11 +418,11 @@ def read(
415418
else:
416419
del ti_labels[LABEL_TRY_NUMBER]
417420

418-
log_filter = self.io._prepare_log_filter(ti_labels)
421+
log_filter = self.io.prepare_log_filter(ti_labels)
419422
next_page_token = metadata.get("next_page_token", None)
420423
all_pages = "download_logs" in metadata and metadata["download_logs"]
421424

422-
messages, end_of_log, next_page_token = self.io._read_logs(log_filter, next_page_token, all_pages)
425+
messages, end_of_log, next_page_token = self.io.read_logs(log_filter, next_page_token, all_pages)
423426

424427
new_metadata: dict[str, str | bool] = {"end_of_log": end_of_log}
425428

@@ -430,7 +433,14 @@ def read(
430433

431434
@classmethod
432435
def _task_instance_to_labels(cls, ti: TaskInstance) -> dict[str, str]:
433-
return _task_instance_to_labels(ti)
436+
return {
437+
cls.LABEL_TASK_ID: ti.task_id,
438+
cls.LABEL_DAG_ID: ti.dag_id,
439+
cls.LABEL_LOGICAL_DATE: str(ti.logical_date.isoformat())
440+
if AIRFLOW_V_3_0_PLUS
441+
else str(ti.execution_date.isoformat()),
442+
cls.LABEL_TRY_NUMBER: str(ti.try_number),
443+
}
434444

435445
@property
436446
def log_name(self):
@@ -455,12 +465,12 @@ def get_external_log_url(self, task_instance: TaskInstance, try_number: int) ->
455465
:param try_number: task instance try_number to read logs from
456466
:return: URL to the external log collection service
457467
"""
458-
_, project_id = self.io._credentials_and_project
468+
_, project_id = self.io.credentials_and_project
459469

460470
ti_labels = _task_instance_to_labels(task_instance)
461471
ti_labels[LABEL_TRY_NUMBER] = str(try_number)
462472

463-
log_filter = self.io._prepare_log_filter(ti_labels)
473+
log_filter = self.io.prepare_log_filter(ti_labels)
464474

465475
url_query_string = {
466476
"project": project_id,
@@ -473,4 +483,4 @@ def get_external_log_url(self, task_instance: TaskInstance, try_number: int) ->
473483
return url
474484

475485
def close(self) -> None:
476-
self.io._transport.flush()
486+
self.io.transport.flush()

‎providers/google/tests/unit/google/cloud/log/test_stackdriver_task_handler.py‎

Lines changed: 62 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
from __future__ import annotations
1818

1919
import logging
20-
import tempfile
2120
from contextlib import nullcontext
2221
from pathlib import Path
2322
from unittest import mock
@@ -57,8 +56,9 @@ def clean_stackdriver_handlers():
5756

5857

5958
class TestStackdriverRemoteLogIO:
60-
def setup_method(self):
61-
self.local_log_location = str(Path(tempfile.gettempdir()) / "local/stackdriver/logs")
59+
@pytest.fixture(autouse=True)
60+
def _setup(self, tmp_path):
61+
self.local_log_location = str(tmp_path / "local/stackdriver/logs")
6262
self.io = StackdriverRemoteLogIO(
6363
base_log_folder=self.local_log_location,
6464
gcp_key_path="KEY_PATH",
@@ -141,12 +141,12 @@ def test_transport_init(self, mock_client, mock_get_creds_and_project_id):
141141
gcp_log_name="test-log",
142142
transport_type=transport_type,
143143
)
144-
_ = io._transport
144+
_ = io.transport
145145
transport_type.assert_called_once_with(mock_client.return_value, "test-log")
146146

147147
@mock.patch("shutil.rmtree")
148148
@mock.patch(
149-
"airflow.providers.google.cloud.log.stackdriver_task_handler.StackdriverRemoteLogIO._transport",
149+
"airflow.providers.google.cloud.log.stackdriver_task_handler.StackdriverRemoteLogIO.transport",
150150
new_callable=PropertyMock,
151151
)
152152
def test_upload_flushes_transport_and_deletes_local(self, mock_transport_prop, mock_rmtree):
@@ -172,7 +172,7 @@ def test_upload_flushes_transport_and_deletes_local(self, mock_transport_prop, m
172172
mock_rmtree.assert_called_once_with(log_dir.resolve(), ignore_errors=True)
173173

174174
@mock.patch(
175-
"airflow.providers.google.cloud.log.stackdriver_task_handler.StackdriverRemoteLogIO._transport",
175+
"airflow.providers.google.cloud.log.stackdriver_task_handler.StackdriverRemoteLogIO.transport",
176176
new_callable=PropertyMock,
177177
)
178178
def test_upload_no_delete(self, mock_transport_prop):
@@ -198,7 +198,7 @@ def test_prepare_log_filter(self, mock_get_creds_and_project_id):
198198
"dag_id": "test_dag",
199199
"try_number": "1",
200200
}
201-
log_filter = self.io._prepare_log_filter(ti_labels)
201+
log_filter = self.io.prepare_log_filter(ti_labels)
202202

203203
assert 'resource.type="global"' in log_filter
204204
assert 'logName="projects/project_id/logs/airflow"' in log_filter
@@ -220,12 +220,66 @@ def test_prepare_log_filter_with_custom_resource(self, mock_get_creds_and_projec
220220
},
221221
),
222222
)
223-
log_filter = io._prepare_log_filter({"task_id": "test"})
223+
log_filter = io.prepare_log_filter({"task_id": "test"})
224224

225225
assert 'resource.type="cloud_composer_environment"' in log_filter
226226
assert 'resource.labels."environment.name"="test-instance"' in log_filter
227227
assert 'resource.labels.location="europe-west-3"' in log_filter
228228

229+
@pytest.mark.skipif(not AIRFLOW_V_3_0_PLUS, reason="airflow.sdk.log only exists in Airflow 3+")
230+
@mock.patch("airflow.providers.google.cloud.log.stackdriver_task_handler.get_credentials_and_project_id")
231+
@mock.patch("airflow.providers.google.cloud.log.stackdriver_task_handler.gcp_logging.Client")
232+
def test_processors_sends_to_transport(self, mock_client, mock_get_creds_and_project_id):
233+
mock_get_creds_and_project_id.return_value = ("creds", "project_id")
234+
235+
mock_transport_type = mock.MagicMock()
236+
with mock.patch("airflow.sdk.log.relative_path_from_logger", return_value="dag/task/1.log"):
237+
io = StackdriverRemoteLogIO(
238+
base_log_folder=self.local_log_location,
239+
gcp_log_name="airflow",
240+
labels={"env": "test"},
241+
transport_type=mock_transport_type,
242+
)
243+
processors = io.processors
244+
assert len(processors) == 1
245+
246+
proc = processors[0]
247+
mock_logger = mock.MagicMock()
248+
249+
event = {
250+
"event": "hello world",
251+
"logger_name": "airflow.task",
252+
"timestamp": "2026-01-15T10:30:00+00:00",
253+
}
254+
result = proc(mock_logger, "info", event)
255+
256+
assert result is event
257+
mock_transport = mock_transport_type.return_value
258+
mock_transport.send.assert_called_once()
259+
record = mock_transport.send.call_args[0][0]
260+
assert record.levelno == logging.INFO
261+
262+
@pytest.mark.skipif(not AIRFLOW_V_3_0_PLUS, reason="airflow.sdk.log only exists in Airflow 3+")
263+
@mock.patch("airflow.providers.google.cloud.log.stackdriver_task_handler.get_credentials_and_project_id")
264+
@mock.patch("airflow.providers.google.cloud.log.stackdriver_task_handler.gcp_logging.Client")
265+
def test_processors_skips_non_task_logger(self, mock_client, mock_get_creds_and_project_id):
266+
mock_get_creds_and_project_id.return_value = ("creds", "project_id")
267+
268+
mock_transport_type = mock.MagicMock()
269+
with mock.patch("airflow.sdk.log.relative_path_from_logger", return_value=None):
270+
io = StackdriverRemoteLogIO(
271+
base_log_folder=self.local_log_location,
272+
gcp_log_name="airflow",
273+
transport_type=mock_transport_type,
274+
)
275+
proc = io.processors[0]
276+
277+
event = {"event": "should not be sent"}
278+
result = proc(mock.MagicMock(), "info", event)
279+
280+
assert result is event
281+
mock_transport_type.return_value.send.assert_not_called()
282+
229283

230284
@pytest.mark.usefixtures("clean_stackdriver_handlers")
231285
@mock.patch("airflow.providers.google.cloud.log.stackdriver_task_handler.get_credentials_and_project_id")

0 commit comments

Comments
 (0)