77
88import json
99import io
10+ import logging
1011import pandas as pd
1112from azure .identity import DefaultAzureCredential , ManagedIdentityCredential
1213from azure .storage .filedatalake import DataLakeServiceClient
1314from typing import List , Dict
15+ import time
16+ import functools
17+ import uuid
18+
19+ # Configure logging for observability
20+ logging .basicConfig (
21+ level = logging .INFO ,
22+ format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
23+ )
24+ logger = logging .getLogger (__name__ )
25+
26+
27+ def retry_with_timeout (max_retries : int = 3 , timeout_seconds : float = 30.0 ):
28+ """
29+ Decorator for retry logic with exponential backoff and timeout.
30+
31+ Args:
32+ max_retries: Maximum number of retry attempts
33+ timeout_seconds: Timeout in seconds for each attempt
34+ """
35+ def decorator (func ):
36+ @functools .wraps (func )
37+ async def wrapper (* args , ** kwargs ):
38+ attempt_id = str (uuid .uuid4 ())[:8 ]
39+ logger .info (f"Starting operation { func .__name__ } (attempt_id: { attempt_id } )" )
40+
41+ last_exception = None
42+ for attempt in range (max_retries ):
43+ try :
44+ # Add timeout to the operation
45+ import asyncio
46+ result = await asyncio .wait_for (
47+ func (* args , ** kwargs ),
48+ timeout = timeout_seconds
49+ )
50+ logger .info (f"Operation { func .__name__ } succeeded on attempt { attempt + 1 } " )
51+ return result
52+ except asyncio .TimeoutError :
53+ last_exception = TimeoutError (f"Operation timed out after { timeout_seconds } s" )
54+ wait_time = 2 ** attempt # Exponential backoff
55+ logger .warning (
56+ f"Attempt { attempt + 1 } timed out for { func .__name__ } . "
57+ f"Retrying in { wait_time } s... (attempt_id: { attempt_id } )"
58+ )
59+ if attempt < max_retries - 1 :
60+ await asyncio .sleep (wait_time )
61+ except Exception as e :
62+ last_exception = e
63+ wait_time = 2 ** attempt # Exponential backoff
64+ logger .warning (
65+ f"Attempt { attempt + 1 } failed for { func .__name__ } : { str (e )} . "
66+ f"Retrying in { wait_time } s... (attempt_id: { attempt_id } )"
67+ )
68+ if attempt < max_retries - 1 :
69+ await asyncio .sleep (wait_time )
70+
71+ # All retries exhausted
72+ logger .error (
73+ f"Operation { func .__name__ } failed after { max_retries } attempts "
74+ f"(attempt_id: { attempt_id } ): { str (last_exception )} "
75+ )
76+ raise last_exception
77+ return wrapper
78+ return decorator
1479
1580
1681class FabricLakehouseCredential :
@@ -54,7 +119,9 @@ async def list_lakehouse_files(
54119
55120 Args:
56121 workspace_id: The Fabric workspace ID (GUID)
122+ Example: "12345678-1234-5678-1234-567812345678"
57123 lakehouse_id: The lakehouse ID (GUID)
124+ Example: "87654321-4321-8765-4321-876543218765"
58125 path: Path within the lakehouse (default: "Files")
59126 file_extension: Optional filter by file extension (e.g., ".csv")
60127 limit: Maximum number of files to return (default: 100)
@@ -63,21 +130,33 @@ async def list_lakehouse_files(
63130 Returns:
64131 JSON string containing list of files with pagination info
65132 """
133+ request_id = str (uuid .uuid4 ())[:8 ]
134+ logger .info (
135+ f"[{ request_id } ] Listing files in workspace { workspace_id } , "
136+ f"lakehouse { lakehouse_id } , path { path } "
137+ )
138+
66139 try :
67140 # Create credential
68141 credential = FabricLakehouseCredential ()
142+ logger .debug (f"[{ request_id } ] Created FabricLakehouseCredential" )
69143
70144 # Connect to OneLake (Fabric's data lake)
71145 datalake_client = DataLakeServiceClient (
72146 account_url = "https://onelake.dfs.fabric.microsoft.com" ,
73147 credential = credential
74148 )
149+ logger .debug (f"[{ request_id } ] Connected to OneLake" )
75150
76151 # Get the file system client for the workspace
77152 fs_client = datalake_client .get_file_system_client (workspace_id )
153+ logger .debug (
154+ f"[{ request_id } ] Got file system client for workspace { workspace_id } "
155+ )
78156
79157 # Build the full path
80158 full_path = f"{ lakehouse_id } /{ path } "
159+ logger .debug (f"[{ request_id } ] Listing path: { full_path } " )
81160
82161 # List paths
83162 paths = fs_client .get_paths (path = full_path )
@@ -104,6 +183,11 @@ async def list_lakehouse_files(
104183 paginated_files = all_files [offset :offset + limit ]
105184 has_more = (offset + limit ) < total_count
106185
186+ logger .info (
187+ f"[{ request_id } ] Listed { len (paginated_files )} of "
188+ f"{ total_count } files (offset: { offset } , has_more: { has_more } )"
189+ )
190+
107191 return json .dumps ({
108192 "success" : True ,
109193 "file_count" : len (paginated_files ),
@@ -115,6 +199,7 @@ async def list_lakehouse_files(
115199 })
116200
117201 except Exception as e :
202+ logger .error (f"[{ request_id } ] Error listing files: { str (e )} " , exc_info = True )
118203 return json .dumps ({
119204 "success" : False ,
120205 "error" : str (e )
@@ -137,36 +222,57 @@ async def read_csv_file(
137222 Returns:
138223 JSON string containing CSV data or error
139224 """
225+ request_id = str (uuid .uuid4 ())[:8 ]
226+ logger .info (
227+ f"[{ request_id } ] Reading CSV file from workspace { workspace_id } , "
228+ f"lakehouse { lakehouse_id } , file { file_path } "
229+ )
230+
140231 try :
141232 # Create credential
142233 credential = FabricLakehouseCredential ()
234+ logger .debug (f"[{ request_id } ] Created FabricLakehouseCredential" )
143235
144236 # Connect to OneLake
145237 datalake_client = DataLakeServiceClient (
146238 account_url = "https://onelake.dfs.fabric.microsoft.com" ,
147239 credential = credential
148240 )
241+ logger .debug (f"[{ request_id } ] Connected to OneLake" )
149242
150243 # Get the file system client
151244 fs_client = datalake_client .get_file_system_client (workspace_id )
245+ logger .debug (
246+ f"[{ request_id } ] Got file system client for workspace "
247+ f"{ workspace_id } "
248+ )
152249
153250 # Build the full path
154251 full_path = f"{ lakehouse_id } /{ file_path } "
252+ logger .debug (f"[{ request_id } ] Full path: { full_path } " )
155253
156254 # Get file client
157255 file_client = fs_client .get_file_client (full_path )
256+ logger .debug (f"[{ request_id } ] Got file client" )
158257
159258 # Download the file
160259 download = file_client .download_file ()
161260 content = download .readall ()
261+ logger .debug (f"[{ request_id } ] Downloaded file ({ len (content )} bytes)" )
162262
163263 # Use pandas for robust CSV parsing
164264 # Handles: quoted fields, escaped commas, various encodings, line breaks
165265 df = pd .read_csv (io .BytesIO (content ))
266+ logger .debug (f"[{ request_id } ] Parsed CSV with { len (df )} rows" )
166267
167268 # Convert to dictionary format for JSON serialization
168269 data = df .head (100 ).to_dict (orient = 'records' )
169270
271+ logger .info (
272+ f"[{ request_id } ] Successfully read CSV file with "
273+ f"{ len (df )} rows, returning { len (data )} records"
274+ )
275+
170276 return json .dumps ({
171277 "success" : True ,
172278 "row_count" : len (df ),
@@ -175,6 +281,10 @@ async def read_csv_file(
175281 })
176282
177283 except Exception as e :
284+ logger .error (
285+ f"[{ request_id } ] Error reading CSV file: { str (e )} " ,
286+ exc_info = True
287+ )
178288 return json .dumps ({
179289 "success" : False ,
180290 "error" : str (e )
0 commit comments