Skip to content

Commit 689160f

Browse files
committed
Remove local_settings change, split into separate PR apache#65191
1 parent 10e7ba0 commit 689160f

3 files changed

Lines changed: 89 additions & 46 deletions

File tree

‎airflow-core/src/airflow/config_templates/airflow_local_settings.py‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -241,25 +241,19 @@ def _default_conn_name_from(mod_path, hook_name):
241241
)
242242
_handler_kwargs = {}
243243
elif remote_base_log_folder.startswith("stackdriver://"):
244-
from airflow.providers.google.cloud.log.stackdriver_task_handler import StackdriverRemoteLogIO
245-
246244
key_path = conf.get_mandatory_value("logging", "GOOGLE_KEY_PATH", fallback=None)
247245
# stackdriver:///airflow-tasks => airflow-tasks
248246
log_name = urlsplit(remote_base_log_folder).path[1:]
247+
STACKDRIVER_REMOTE_HANDLERS = {
248+
"task": {
249+
"class": "airflow.providers.google.cloud.log.stackdriver_task_handler.StackdriverTaskHandler",
250+
"formatter": "airflow",
251+
"gcp_log_name": log_name,
252+
"gcp_key_path": key_path,
253+
}
254+
}
249255

250-
REMOTE_TASK_LOG = StackdriverRemoteLogIO(
251-
**cast(
252-
"dict[str, Any]",
253-
{
254-
"base_log_folder": BASE_LOG_FOLDER,
255-
"gcp_log_name": log_name,
256-
"gcp_key_path": key_path,
257-
"delete_local_copy": delete_local_copy,
258-
}
259-
| _handler_kwargs,
260-
)
261-
)
262-
_handler_kwargs = {}
256+
DEFAULT_LOGGING_CONFIG["handlers"].update(STACKDRIVER_REMOTE_HANDLERS)
263257
elif remote_base_log_folder.startswith("oss://"):
264258
from airflow.providers.alibaba.cloud.log.oss_task_handler import OSSRemoteLogIO
265259

‎providers/google/src/airflow/providers/google/cloud/log/stackdriver_task_handler.py‎

Lines changed: 23 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@
2525
import shutil
2626
import warnings
2727
from collections.abc import Collection
28+
from datetime import datetime
2829
from functools import cached_property
30+
from logging import getLogRecordFactory
2931
from pathlib import Path
3032
from typing import TYPE_CHECKING
3133
from urllib.parse import urlencode
@@ -87,7 +89,7 @@ class StackdriverRemoteLogIO(LoggingMixin):
8789
labels: dict[str, str] | None = None
8890

8991
@cached_property
90-
def _credentials_and_project(self) -> tuple[Credentials, str]:
92+
def credentials_and_project(self) -> tuple[Credentials, str]:
9193
credentials, project = get_credentials_and_project_id(
9294
key_path=self.gcp_key_path, scopes=self.scopes, disable_logging=True
9395
)
@@ -96,7 +98,7 @@ def _credentials_and_project(self) -> tuple[Credentials, str]:
9698
@cached_property
9799
def _client(self) -> gcp_logging.Client:
98100
"""The Cloud Library API client."""
99-
credentials, project = self._credentials_and_project
101+
credentials, project = self.credentials_and_project
100102
return gcp_logging.Client(
101103
credentials=credentials,
102104
project=project,
@@ -106,29 +108,26 @@ def _client(self) -> gcp_logging.Client:
106108
@cached_property
107109
def _logging_service_client(self) -> LoggingServiceV2Client:
108110
"""The Cloud logging service v2 client."""
109-
credentials, _ = self._credentials_and_project
111+
credentials, _ = self.credentials_and_project
110112
return LoggingServiceV2Client(
111113
credentials=credentials,
112114
client_info=CLIENT_INFO,
113115
)
114116

115117
@cached_property
116-
def _transport(self) -> Transport:
118+
def transport(self) -> Transport:
117119
"""Object responsible for sending data to Stackdriver."""
118120
return self.transport_type(self._client, self.gcp_log_name)
119121

120122
@cached_property
121123
def processors(self) -> tuple[structlog.typing.Processor, ...]:
122-
from datetime import datetime
123-
from logging import getLogRecordFactory
124-
125124
import structlog.stdlib
126125

127-
logRecordFactory = getLogRecordFactory()
128-
_transport = self._transport
129-
130126
from airflow.sdk.log import relative_path_from_logger
131127

128+
log_record_factory = getLogRecordFactory()
129+
_transport = self.transport
130+
132131
def proc(
133132
logger: structlog.typing.WrappedLogger,
134133
method_name: str,
@@ -144,7 +143,7 @@ def proc(
144143
if ts := msg.pop("timestamp", None):
145144
with contextlib.suppress(Exception):
146145
created = datetime.fromisoformat(ts)
147-
record = logRecordFactory(
146+
record = log_record_factory(
148147
name,
149148
level,
150149
pathname="",
@@ -168,7 +167,7 @@ def proc(
168167

169168
def upload(self, path: os.PathLike | str, ti: RuntimeTI) -> None:
170169
"""Flush the transport and optionally delete local log files."""
171-
self._transport.flush()
170+
self.transport.flush()
172171
if self.delete_local_copy:
173172
base = self.base_log_folder.resolve()
174173
raw = Path(path)
@@ -189,19 +188,19 @@ def upload(self, path: os.PathLike | str, ti: RuntimeTI) -> None:
189188
def read(self, relative_path: str, ti: RuntimeTI) -> LogResponse:
190189
"""Read logs from Stackdriver Logging using task instance labels."""
191190
ti_labels = _task_instance_to_labels(ti)
192-
log_filter = self._prepare_log_filter(ti_labels)
193-
messages, end_of_log, _ = self._read_logs(log_filter, next_page_token=None, all_pages=True)
191+
log_filter = self.prepare_log_filter(ti_labels)
192+
messages, end_of_log, _ = self.read_logs(log_filter, next_page_token=None, all_pages=True)
194193
return [f"Reading remote log from Stackdriver for {relative_path}"], [messages] if messages else []
195194

196-
def _prepare_log_filter(self, ti_labels: dict[str, str]) -> str:
195+
def prepare_log_filter(self, ti_labels: dict[str, str]) -> str:
197196
def escape_label_key(key: str) -> str:
198197
return f'"{key}"' if "." in key else key
199198

200199
def escape_label_value(value: str) -> str:
201200
escaped_value = value.replace("\\", "\\\\").replace('"', '\\"')
202201
return f'"{escaped_value}"'
203202

204-
_, project = self._credentials_and_project
203+
_, project = self.credentials_and_project
205204
log_filters = [
206205
f"resource.type={escape_label_value(self.resource.type)}",
207206
f'logName="projects/{project}/logs/{self.gcp_log_name}"',
@@ -214,7 +213,7 @@ def escape_label_value(value: str) -> str:
214213
log_filters.append(f"labels.{escape_label_key(key)}={escape_label_value(value)}")
215214
return "\n".join(log_filters)
216215

217-
def _read_logs(
216+
def read_logs(
218217
self, log_filter: str, next_page_token: str | None, all_pages: bool
219218
) -> tuple[str, bool, str | None]:
220219
messages = []
@@ -229,8 +228,6 @@ def _read_logs(
229228
log_filter=log_filter, page_token=next_page_token
230229
)
231230
messages.append(new_messages)
232-
if not messages:
233-
break
234231

235232
end_of_log = True
236233
next_page_token = None
@@ -239,7 +236,7 @@ def _read_logs(
239236
return "\n".join(messages), end_of_log, next_page_token
240237

241238
def _read_single_logs_page(self, log_filter: str, page_token: str | None = None) -> tuple[str, str]:
242-
_, project = self._credentials_and_project
239+
_, project = self.credentials_and_project
243240
request = ListLogEntriesRequest(
244241
resource_names=[f"projects/{project}"],
245242
filter=log_filter,
@@ -376,7 +373,7 @@ def emit(self, record: logging.LogRecord) -> None:
376373
if not AIRFLOW_V_3_0_PLUS and getattr(record, ctx_indiv_trigger.name, None):
377374
ti = getattr(record, "task_instance", None) # trigger context
378375
labels = self._get_labels(ti)
379-
self.io._transport.send(record, message, resource=self.resource, labels=labels)
376+
self.io.transport.send(record, message, resource=self.resource, labels=labels)
380377

381378
def set_context(self, task_instance: TaskInstance) -> None:
382379
"""
@@ -415,11 +412,11 @@ def read(
415412
else:
416413
del ti_labels[LABEL_TRY_NUMBER]
417414

418-
log_filter = self.io._prepare_log_filter(ti_labels)
415+
log_filter = self.io.prepare_log_filter(ti_labels)
419416
next_page_token = metadata.get("next_page_token", None)
420417
all_pages = "download_logs" in metadata and metadata["download_logs"]
421418

422-
messages, end_of_log, next_page_token = self.io._read_logs(log_filter, next_page_token, all_pages)
419+
messages, end_of_log, next_page_token = self.io.read_logs(log_filter, next_page_token, all_pages)
423420

424421
new_metadata: dict[str, str | bool] = {"end_of_log": end_of_log}
425422

@@ -455,12 +452,12 @@ def get_external_log_url(self, task_instance: TaskInstance, try_number: int) ->
455452
:param try_number: task instance try_number to read logs from
456453
:return: URL to the external log collection service
457454
"""
458-
_, project_id = self.io._credentials_and_project
455+
_, project_id = self.io.credentials_and_project
459456

460457
ti_labels = _task_instance_to_labels(task_instance)
461458
ti_labels[LABEL_TRY_NUMBER] = str(try_number)
462459

463-
log_filter = self.io._prepare_log_filter(ti_labels)
460+
log_filter = self.io.prepare_log_filter(ti_labels)
464461

465462
url_query_string = {
466463
"project": project_id,
@@ -473,4 +470,4 @@ def get_external_log_url(self, task_instance: TaskInstance, try_number: int) ->
473470
return url
474471

475472
def close(self) -> None:
476-
self.io._transport.flush()
473+
self.io.transport.flush()

‎providers/google/tests/unit/google/cloud/log/test_stackdriver_task_handler.py‎

Lines changed: 57 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -141,12 +141,12 @@ def test_transport_init(self, mock_client, mock_get_creds_and_project_id):
141141
gcp_log_name="test-log",
142142
transport_type=transport_type,
143143
)
144-
_ = io._transport
144+
_ = io.transport
145145
transport_type.assert_called_once_with(mock_client.return_value, "test-log")
146146

147147
@mock.patch("shutil.rmtree")
148148
@mock.patch(
149-
"airflow.providers.google.cloud.log.stackdriver_task_handler.StackdriverRemoteLogIO._transport",
149+
"airflow.providers.google.cloud.log.stackdriver_task_handler.StackdriverRemoteLogIO.transport",
150150
new_callable=PropertyMock,
151151
)
152152
def test_upload_flushes_transport_and_deletes_local(self, mock_transport_prop, mock_rmtree):
@@ -172,7 +172,7 @@ def test_upload_flushes_transport_and_deletes_local(self, mock_transport_prop, m
172172
mock_rmtree.assert_called_once_with(log_dir.resolve(), ignore_errors=True)
173173

174174
@mock.patch(
175-
"airflow.providers.google.cloud.log.stackdriver_task_handler.StackdriverRemoteLogIO._transport",
175+
"airflow.providers.google.cloud.log.stackdriver_task_handler.StackdriverRemoteLogIO.transport",
176176
new_callable=PropertyMock,
177177
)
178178
def test_upload_no_delete(self, mock_transport_prop):
@@ -198,7 +198,7 @@ def test_prepare_log_filter(self, mock_get_creds_and_project_id):
198198
"dag_id": "test_dag",
199199
"try_number": "1",
200200
}
201-
log_filter = self.io._prepare_log_filter(ti_labels)
201+
log_filter = self.io.prepare_log_filter(ti_labels)
202202

203203
assert 'resource.type="global"' in log_filter
204204
assert 'logName="projects/project_id/logs/airflow"' in log_filter
@@ -220,12 +220,64 @@ def test_prepare_log_filter_with_custom_resource(self, mock_get_creds_and_projec
220220
},
221221
),
222222
)
223-
log_filter = io._prepare_log_filter({"task_id": "test"})
223+
log_filter = io.prepare_log_filter({"task_id": "test"})
224224

225225
assert 'resource.type="cloud_composer_environment"' in log_filter
226226
assert 'resource.labels."environment.name"="test-instance"' in log_filter
227227
assert 'resource.labels.location="europe-west-3"' in log_filter
228228

229+
@mock.patch("airflow.providers.google.cloud.log.stackdriver_task_handler.get_credentials_and_project_id")
230+
@mock.patch("airflow.providers.google.cloud.log.stackdriver_task_handler.gcp_logging.Client")
231+
def test_processors_sends_to_transport(self, mock_client, mock_get_creds_and_project_id):
232+
mock_get_creds_and_project_id.return_value = ("creds", "project_id")
233+
234+
mock_transport_type = mock.MagicMock()
235+
with mock.patch("airflow.sdk.log.relative_path_from_logger", return_value="dag/task/1.log"):
236+
io = StackdriverRemoteLogIO(
237+
base_log_folder=self.local_log_location,
238+
gcp_log_name="airflow",
239+
labels={"env": "test"},
240+
transport_type=mock_transport_type,
241+
)
242+
processors = io.processors
243+
assert len(processors) == 1
244+
245+
proc = processors[0]
246+
mock_logger = mock.MagicMock()
247+
248+
event = {
249+
"event": "hello world",
250+
"logger_name": "airflow.task",
251+
"timestamp": "2026-01-15T10:30:00+00:00",
252+
}
253+
result = proc(mock_logger, "info", event)
254+
255+
assert result is event
256+
mock_transport = mock_transport_type.return_value
257+
mock_transport.send.assert_called_once()
258+
record = mock_transport.send.call_args[0][0]
259+
assert record.levelno == logging.INFO
260+
261+
@mock.patch("airflow.providers.google.cloud.log.stackdriver_task_handler.get_credentials_and_project_id")
262+
@mock.patch("airflow.providers.google.cloud.log.stackdriver_task_handler.gcp_logging.Client")
263+
def test_processors_skips_non_task_logger(self, mock_client, mock_get_creds_and_project_id):
264+
mock_get_creds_and_project_id.return_value = ("creds", "project_id")
265+
266+
mock_transport_type = mock.MagicMock()
267+
with mock.patch("airflow.sdk.log.relative_path_from_logger", return_value=None):
268+
io = StackdriverRemoteLogIO(
269+
base_log_folder=self.local_log_location,
270+
gcp_log_name="airflow",
271+
transport_type=mock_transport_type,
272+
)
273+
proc = io.processors[0]
274+
275+
event = {"event": "should not be sent"}
276+
result = proc(mock.MagicMock(), "info", event)
277+
278+
assert result is event
279+
mock_transport_type.return_value.send.assert_not_called()
280+
229281

230282
@pytest.mark.usefixtures("clean_stackdriver_handlers")
231283
@mock.patch("airflow.providers.google.cloud.log.stackdriver_task_handler.get_credentials_and_project_id")

0 commit comments

Comments
 (0)