2525import shutil
2626import warnings
2727from collections .abc import Collection
28+ from datetime import datetime
2829from functools import cached_property
30+ from logging import getLogRecordFactory
2931from pathlib import Path
3032from typing import TYPE_CHECKING
3133from urllib .parse import urlencode
@@ -87,7 +89,7 @@ class StackdriverRemoteLogIO(LoggingMixin):
8789 labels : dict [str , str ] | None = None
8890
8991 @cached_property
90- def _credentials_and_project (self ) -> tuple [Credentials , str ]:
92+ def credentials_and_project (self ) -> tuple [Credentials , str ]:
9193 credentials , project = get_credentials_and_project_id (
9294 key_path = self .gcp_key_path , scopes = self .scopes , disable_logging = True
9395 )
@@ -96,7 +98,7 @@ def _credentials_and_project(self) -> tuple[Credentials, str]:
9698 @cached_property
9799 def _client (self ) -> gcp_logging .Client :
98100 """The Cloud Library API client."""
99- credentials , project = self ._credentials_and_project
101+ credentials , project = self .credentials_and_project
100102 return gcp_logging .Client (
101103 credentials = credentials ,
102104 project = project ,
@@ -106,29 +108,26 @@ def _client(self) -> gcp_logging.Client:
106108 @cached_property
107109 def _logging_service_client (self ) -> LoggingServiceV2Client :
108110 """The Cloud logging service v2 client."""
109- credentials , _ = self ._credentials_and_project
111+ credentials , _ = self .credentials_and_project
110112 return LoggingServiceV2Client (
111113 credentials = credentials ,
112114 client_info = CLIENT_INFO ,
113115 )
114116
115117 @cached_property
116- def _transport (self ) -> Transport :
118+ def transport (self ) -> Transport :
117119 """Object responsible for sending data to Stackdriver."""
118120 return self .transport_type (self ._client , self .gcp_log_name )
119121
120122 @cached_property
121123 def processors (self ) -> tuple [structlog .typing .Processor , ...]:
122- from datetime import datetime
123- from logging import getLogRecordFactory
124-
125124 import structlog .stdlib
126125
127- logRecordFactory = getLogRecordFactory ()
128- _transport = self ._transport
129-
130126 from airflow .sdk .log import relative_path_from_logger
131127
128+ log_record_factory = getLogRecordFactory ()
129+ _transport = self .transport
130+
132131 def proc (
133132 logger : structlog .typing .WrappedLogger ,
134133 method_name : str ,
@@ -144,7 +143,7 @@ def proc(
144143 if ts := msg .pop ("timestamp" , None ):
145144 with contextlib .suppress (Exception ):
146145 created = datetime .fromisoformat (ts )
147- record = logRecordFactory (
146+ record = log_record_factory (
148147 name ,
149148 level ,
150149 pathname = "" ,
@@ -168,7 +167,7 @@ def proc(
168167
169168 def upload (self , path : os .PathLike | str , ti : RuntimeTI ) -> None :
170169 """Flush the transport and optionally delete local log files."""
171- self ._transport .flush ()
170+ self .transport .flush ()
172171 if self .delete_local_copy :
173172 base = self .base_log_folder .resolve ()
174173 raw = Path (path )
@@ -189,19 +188,19 @@ def upload(self, path: os.PathLike | str, ti: RuntimeTI) -> None:
189188 def read (self , relative_path : str , ti : RuntimeTI ) -> LogResponse :
190189 """Read logs from Stackdriver Logging using task instance labels."""
191190 ti_labels = _task_instance_to_labels (ti )
192- log_filter = self ._prepare_log_filter (ti_labels )
193- messages , end_of_log , _ = self ._read_logs (log_filter , next_page_token = None , all_pages = True )
191+ log_filter = self .prepare_log_filter (ti_labels )
192+ messages , end_of_log , _ = self .read_logs (log_filter , next_page_token = None , all_pages = True )
194193 return [f"Reading remote log from Stackdriver for { relative_path } " ], [messages ] if messages else []
195194
196- def _prepare_log_filter (self , ti_labels : dict [str , str ]) -> str :
195+ def prepare_log_filter (self , ti_labels : dict [str , str ]) -> str :
197196 def escape_label_key (key : str ) -> str :
198197 return f'"{ key } "' if "." in key else key
199198
200199 def escape_label_value (value : str ) -> str :
201200 escaped_value = value .replace ("\\ " , "\\ \\ " ).replace ('"' , '\\ "' )
202201 return f'"{ escaped_value } "'
203202
204- _ , project = self ._credentials_and_project
203+ _ , project = self .credentials_and_project
205204 log_filters = [
206205 f"resource.type={ escape_label_value (self .resource .type )} " ,
207206 f'logName="projects/{ project } /logs/{ self .gcp_log_name } "' ,
@@ -214,7 +213,7 @@ def escape_label_value(value: str) -> str:
214213 log_filters .append (f"labels.{ escape_label_key (key )} ={ escape_label_value (value )} " )
215214 return "\n " .join (log_filters )
216215
217- def _read_logs (
216+ def read_logs (
218217 self , log_filter : str , next_page_token : str | None , all_pages : bool
219218 ) -> tuple [str , bool , str | None ]:
220219 messages = []
@@ -229,8 +228,6 @@ def _read_logs(
229228 log_filter = log_filter , page_token = next_page_token
230229 )
231230 messages .append (new_messages )
232- if not messages :
233- break
234231
235232 end_of_log = True
236233 next_page_token = None
@@ -239,7 +236,7 @@ def _read_logs(
239236 return "\n " .join (messages ), end_of_log , next_page_token
240237
241238 def _read_single_logs_page (self , log_filter : str , page_token : str | None = None ) -> tuple [str , str ]:
242- _ , project = self ._credentials_and_project
239+ _ , project = self .credentials_and_project
243240 request = ListLogEntriesRequest (
244241 resource_names = [f"projects/{ project } " ],
245242 filter = log_filter ,
@@ -376,7 +373,7 @@ def emit(self, record: logging.LogRecord) -> None:
376373 if not AIRFLOW_V_3_0_PLUS and getattr (record , ctx_indiv_trigger .name , None ):
377374 ti = getattr (record , "task_instance" , None ) # trigger context
378375 labels = self ._get_labels (ti )
379- self .io ._transport .send (record , message , resource = self .resource , labels = labels )
376+ self .io .transport .send (record , message , resource = self .resource , labels = labels )
380377
381378 def set_context (self , task_instance : TaskInstance ) -> None :
382379 """
@@ -415,11 +412,11 @@ def read(
415412 else :
416413 del ti_labels [LABEL_TRY_NUMBER ]
417414
418- log_filter = self .io ._prepare_log_filter (ti_labels )
415+ log_filter = self .io .prepare_log_filter (ti_labels )
419416 next_page_token = metadata .get ("next_page_token" , None )
420417 all_pages = "download_logs" in metadata and metadata ["download_logs" ]
421418
422- messages , end_of_log , next_page_token = self .io ._read_logs (log_filter , next_page_token , all_pages )
419+ messages , end_of_log , next_page_token = self .io .read_logs (log_filter , next_page_token , all_pages )
423420
424421 new_metadata : dict [str , str | bool ] = {"end_of_log" : end_of_log }
425422
@@ -455,12 +452,12 @@ def get_external_log_url(self, task_instance: TaskInstance, try_number: int) ->
455452 :param try_number: task instance try_number to read logs from
456453 :return: URL to the external log collection service
457454 """
458- _ , project_id = self .io ._credentials_and_project
455+ _ , project_id = self .io .credentials_and_project
459456
460457 ti_labels = _task_instance_to_labels (task_instance )
461458 ti_labels [LABEL_TRY_NUMBER ] = str (try_number )
462459
463- log_filter = self .io ._prepare_log_filter (ti_labels )
460+ log_filter = self .io .prepare_log_filter (ti_labels )
464461
465462 url_query_string = {
466463 "project" : project_id ,
@@ -473,4 +470,4 @@ def get_external_log_url(self, task_instance: TaskInstance, try_number: int) ->
473470 return url
474471
475472 def close (self ) -> None :
476- self .io ._transport .flush ()
473+ self .io .transport .flush ()
0 commit comments