2525import shutil
2626import warnings
2727from collections .abc import Collection
28+ from datetime import datetime
2829from functools import cached_property
30+ from logging import getLogRecordFactory
2931from pathlib import Path
3032from typing import TYPE_CHECKING
3133from urllib .parse import urlencode
@@ -87,7 +89,7 @@ class StackdriverRemoteLogIO(LoggingMixin):
8789 labels : dict [str , str ] | None = None
8890
8991 @cached_property
90- def _credentials_and_project (self ) -> tuple [Credentials , str ]:
92+ def credentials_and_project (self ) -> tuple [Credentials , str ]:
9193 credentials , project = get_credentials_and_project_id (
9294 key_path = self .gcp_key_path , scopes = self .scopes , disable_logging = True
9395 )
@@ -96,7 +98,7 @@ def _credentials_and_project(self) -> tuple[Credentials, str]:
9698 @cached_property
9799 def _client (self ) -> gcp_logging .Client :
98100 """The Cloud Library API client."""
99- credentials , project = self ._credentials_and_project
101+ credentials , project = self .credentials_and_project
100102 return gcp_logging .Client (
101103 credentials = credentials ,
102104 project = project ,
@@ -106,29 +108,26 @@ def _client(self) -> gcp_logging.Client:
106108 @cached_property
107109 def _logging_service_client (self ) -> LoggingServiceV2Client :
108110 """The Cloud logging service v2 client."""
109- credentials , _ = self ._credentials_and_project
111+ credentials , _ = self .credentials_and_project
110112 return LoggingServiceV2Client (
111113 credentials = credentials ,
112114 client_info = CLIENT_INFO ,
113115 )
114116
115117 @cached_property
116- def _transport (self ) -> Transport :
118+ def transport (self ) -> Transport :
117119 """Object responsible for sending data to Stackdriver."""
118120 return self .transport_type (self ._client , self .gcp_log_name )
119121
120122 @cached_property
121123 def processors (self ) -> tuple [structlog .typing .Processor , ...]:
122- from datetime import datetime
123- from logging import getLogRecordFactory
124-
125124 import structlog .stdlib
126125
127- logRecordFactory = getLogRecordFactory ()
128- _transport = self ._transport
129-
130126 from airflow .sdk .log import relative_path_from_logger
131127
128+ log_record_factory = getLogRecordFactory ()
129+ _transport = self .transport
130+
132131 def proc (
133132 logger : structlog .typing .WrappedLogger ,
134133 method_name : str ,
@@ -144,7 +143,7 @@ def proc(
144143 if ts := msg .pop ("timestamp" , None ):
145144 with contextlib .suppress (Exception ):
146145 created = datetime .fromisoformat (ts )
147- record = logRecordFactory (
146+ record = log_record_factory (
148147 name ,
149148 level ,
150149 pathname = "" ,
@@ -159,16 +158,21 @@ def proc(
159158 ct = created .timestamp ()
160159 record .created = ct
161160 record .msecs = int ((ct - int (ct )) * 1000 ) + 0.0
162- _transport .send (
163- record , str (msg .get ("event" , "" )), resource = self .resource , labels = self .labels or {}
164- )
161+
162+ ti = getattr (record , "task_instance" , None )
163+ labels : dict [str , str ] = {}
164+ if self .labels :
165+ labels .update (self .labels )
166+ if ti :
167+ labels .update (_task_instance_to_labels (ti ))
168+ _transport .send (record , str (msg .get ("event" , "" )), resource = self .resource , labels = labels )
165169 return event
166170
167171 return (proc ,)
168172
169173 def upload (self , path : os .PathLike | str , ti : RuntimeTI ) -> None :
170174 """Flush the transport and optionally delete local log files."""
171- self ._transport .flush ()
175+ self .transport .flush ()
172176 if self .delete_local_copy :
173177 base = self .base_log_folder .resolve ()
174178 raw = Path (path )
@@ -189,19 +193,19 @@ def upload(self, path: os.PathLike | str, ti: RuntimeTI) -> None:
189193 def read (self , relative_path : str , ti : RuntimeTI ) -> LogResponse :
190194 """Read logs from Stackdriver Logging using task instance labels."""
191195 ti_labels = _task_instance_to_labels (ti )
192- log_filter = self ._prepare_log_filter (ti_labels )
193- messages , end_of_log , _ = self ._read_logs (log_filter , next_page_token = None , all_pages = True )
196+ log_filter = self .prepare_log_filter (ti_labels )
197+ messages , end_of_log , _ = self .read_logs (log_filter , next_page_token = None , all_pages = True )
194198 return [f"Reading remote log from Stackdriver for { relative_path } " ], [messages ] if messages else []
195199
196- def _prepare_log_filter (self , ti_labels : dict [str , str ]) -> str :
200+ def prepare_log_filter (self , ti_labels : dict [str , str ]) -> str :
197201 def escape_label_key (key : str ) -> str :
198202 return f'"{ key } "' if "." in key else key
199203
200204 def escape_label_value (value : str ) -> str :
201205 escaped_value = value .replace ("\\ " , "\\ \\ " ).replace ('"' , '\\ "' )
202206 return f'"{ escaped_value } "'
203207
204- _ , project = self ._credentials_and_project
208+ _ , project = self .credentials_and_project
205209 log_filters = [
206210 f"resource.type={ escape_label_value (self .resource .type )} " ,
207211 f'logName="projects/{ project } /logs/{ self .gcp_log_name } "' ,
@@ -214,7 +218,7 @@ def escape_label_value(value: str) -> str:
214218 log_filters .append (f"labels.{ escape_label_key (key )} ={ escape_label_value (value )} " )
215219 return "\n " .join (log_filters )
216220
217- def _read_logs (
221+ def read_logs (
218222 self , log_filter : str , next_page_token : str | None , all_pages : bool
219223 ) -> tuple [str , bool , str | None ]:
220224 messages = []
@@ -229,8 +233,6 @@ def _read_logs(
229233 log_filter = log_filter , page_token = next_page_token
230234 )
231235 messages .append (new_messages )
232- if not messages :
233- break
234236
235237 end_of_log = True
236238 next_page_token = None
@@ -239,7 +241,7 @@ def _read_logs(
239241 return "\n " .join (messages ), end_of_log , next_page_token
240242
241243 def _read_single_logs_page (self , log_filter : str , page_token : str | None = None ) -> tuple [str , str ]:
242- _ , project = self ._credentials_and_project
244+ _ , project = self .credentials_and_project
243245 request = ListLogEntriesRequest (
244246 resource_names = [f"projects/{ project } " ],
245247 filter = log_filter ,
@@ -301,6 +303,7 @@ class StackdriverTaskHandler(logging.Handler):
301303 :param labels: (Optional) Mapping of labels for the entry.
302304 """
303305
306+ # Re-export module-level constants for back-compat with external code reading them off the class
304307 LABEL_TASK_ID = LABEL_TASK_ID
305308 LABEL_DAG_ID = LABEL_DAG_ID
306309 LABEL_LOGICAL_DATE = LABEL_LOGICAL_DATE
@@ -376,7 +379,7 @@ def emit(self, record: logging.LogRecord) -> None:
376379 if not AIRFLOW_V_3_0_PLUS and getattr (record , ctx_indiv_trigger .name , None ):
377380 ti = getattr (record , "task_instance" , None ) # trigger context
378381 labels = self ._get_labels (ti )
379- self .io ._transport .send (record , message , resource = self .resource , labels = labels )
382+ self .io .transport .send (record , message , resource = self .resource , labels = labels )
380383
381384 def set_context (self , task_instance : TaskInstance ) -> None :
382385 """
@@ -415,11 +418,11 @@ def read(
415418 else :
416419 del ti_labels [LABEL_TRY_NUMBER ]
417420
418- log_filter = self .io ._prepare_log_filter (ti_labels )
421+ log_filter = self .io .prepare_log_filter (ti_labels )
419422 next_page_token = metadata .get ("next_page_token" , None )
420423 all_pages = "download_logs" in metadata and metadata ["download_logs" ]
421424
422- messages , end_of_log , next_page_token = self .io ._read_logs (log_filter , next_page_token , all_pages )
425+ messages , end_of_log , next_page_token = self .io .read_logs (log_filter , next_page_token , all_pages )
423426
424427 new_metadata : dict [str , str | bool ] = {"end_of_log" : end_of_log }
425428
@@ -430,7 +433,14 @@ def read(
430433
431434 @classmethod
432435 def _task_instance_to_labels (cls , ti : TaskInstance ) -> dict [str , str ]:
433- return _task_instance_to_labels (ti )
436+ return {
437+ cls .LABEL_TASK_ID : ti .task_id ,
438+ cls .LABEL_DAG_ID : ti .dag_id ,
439+ cls .LABEL_LOGICAL_DATE : str (ti .logical_date .isoformat ())
440+ if AIRFLOW_V_3_0_PLUS
441+ else str (ti .execution_date .isoformat ()),
442+ cls .LABEL_TRY_NUMBER : str (ti .try_number ),
443+ }
434444
435445 @property
436446 def log_name (self ):
@@ -455,12 +465,12 @@ def get_external_log_url(self, task_instance: TaskInstance, try_number: int) ->
455465 :param try_number: task instance try_number to read logs from
456466 :return: URL to the external log collection service
457467 """
458- _ , project_id = self .io ._credentials_and_project
468+ _ , project_id = self .io .credentials_and_project
459469
460470 ti_labels = _task_instance_to_labels (task_instance )
461471 ti_labels [LABEL_TRY_NUMBER ] = str (try_number )
462472
463- log_filter = self .io ._prepare_log_filter (ti_labels )
473+ log_filter = self .io .prepare_log_filter (ti_labels )
464474
465475 url_query_string = {
466476 "project" : project_id ,
@@ -473,4 +483,4 @@ def get_external_log_url(self, task_instance: TaskInstance, try_number: int) ->
473483 return url
474484
475485 def close (self ) -> None :
476- self .io ._transport .flush ()
486+ self .io .transport .flush ()
0 commit comments