Skip to content

Commit 720b650

Browse files
vivekrnvvivekrnv
authored
[debug dump util] Module implementation Logic and Port Module (#1667)
* Port Module and UT Added What I did HLD for Dump Utility: HLD. For More Info on adding new modules, Check this section in the HLD: MatchInfra Signed-off-by: Vivek Reddy Karri <[email protected]>
1 parent 3e8626e commit 720b650

File tree

11 files changed

+556
-121
lines changed

11 files changed

+556
-121
lines changed

‎dump/match_infra.py‎

Lines changed: 118 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
import json, fnmatch
1+
import json
2+
import fnmatch
23
from abc import ABC, abstractmethod
34
from dump.helper import verbose_print
45
from swsscommon.swsscommon import SonicV2Connector, SonicDBConfig
@@ -12,24 +13,25 @@
1213
"NO_SRC": "Either one of db or file in the request should be non-empty",
1314
"NO_TABLE": "No 'table' name provided",
1415
"NO_KEY": "'key_pattern' cannot be empty",
15-
"NO_VALUE" : "Field is provided, but no value is provided to compare with",
16+
"NO_VALUE": "Field is provided, but no value is provided to compare with",
1617
"SRC_VAGUE": "Only one of db or file should be provided",
17-
"CONN_ERR" : "Connection Error",
18+
"CONN_ERR": "Connection Error",
1819
"JUST_KEYS_COMPAT": "When Just_keys is set to False, return_fields should be empty",
1920
"BAD_FORMAT_RE_FIELDS": "Return Fields should be of list type",
2021
"NO_ENTRIES": "No Keys found after applying the filtering criteria",
2122
"FILE_R_EXEP": "Exception Caught While Reading the json cfg file provided",
2223
"INV_NS": "Namespace is invalid"
2324
}
2425

26+
2527
class MatchRequest:
26-
"""
27-
Request Object which should be passed to the MatchEngine
28-
28+
"""
29+
Request Object which should be passed to the MatchEngine
30+
2931
Attributes:
3032
"table" : A Valid Table Name
3133
"key_pattern" : Pattern of the redis-key to match. Defaults to "*". Eg: "*" will match all the keys.
32-
Supports these glob style patterns. https://redis.io/commands/KEYS
34+
Supports these glob style patterns. https://redis.io/commands/KEYS
3335
"field" : Field to check for a match,Defaults to None
3436
"value" : Value to match, Defaults to None
3537
"return_fields" : An iterable type, where each element woudld imply a field to return from all the filtered keys
@@ -38,9 +40,10 @@ class MatchRequest:
3840
Only one of the db/file fields should have a non-empty string.
3941
"just_keys" : If true, Only Returns the keys matched. Does not return field-value pairs. Defaults to True
4042
"ns" : namespace argument, if nothing is provided, default namespace is used
41-
"match_entire_list" : When this arg is set to true, entire list is matched incluing the ",".
43+
"match_entire_list" : When this arg is set to true, entire list is matched incluing the ",".
4244
When False, the values are split based on "," and individual items are matched with
4345
"""
46+
4447
def __init__(self, **kwargs):
4548
self.table = kwargs["table"] if "table" in kwargs else None
4649
self.key_pattern = kwargs["key_pattern"] if "key_pattern" in kwargs else "*"
@@ -56,16 +59,15 @@ def __init__(self, **kwargs):
5659
verbose_print(str(err))
5760
if err:
5861
raise Exception("Static Checks for the MatchRequest Failed, Reason: \n" + err)
59-
60-
62+
6163
def __static_checks(self):
62-
64+
6365
if not self.db and not self.file:
6466
return EXCEP_DICT["NO_SRC"]
65-
67+
6668
if self.db and self.file:
6769
return EXCEP_DICT["SRC_VAGUE"]
68-
70+
6971
if not self.db:
7072
try:
7173
with open(self.file) as f:
@@ -75,32 +77,32 @@ def __static_checks(self):
7577

7678
if not self.file and self.db not in SonicDBConfig.getDbList():
7779
return EXCEP_DICT["INV_DB"]
78-
80+
7981
if not self.table:
8082
return EXCEP_DICT["NO_TABLE"]
81-
83+
8284
if not isinstance(self.return_fields, list):
8385
return EXCEP_DICT["BAD_FORMAT_RE_FIELDS"]
84-
86+
8587
if not self.just_keys and self.return_fields:
8688
return EXCEP_DICT["JUST_KEYS_COMPAT"]
87-
89+
8890
if self.field and not self.value:
8991
return EXCEP_DICT["NO_VALUE"]
90-
92+
9193
if self.ns != DEFAULT_NAMESPACE and self.ns not in multi_asic.get_namespace_list():
9294
return EXCEP_DICT["INV_NS"] + " Choose From {}".format(multi_asic.get_namespace_list())
93-
95+
9496
verbose_print("MatchRequest Checks Passed")
95-
97+
9698
return ""
97-
99+
98100
def __str__(self):
99101
str = "----------------------- \n MatchRequest: \n"
100102
if self.db:
101103
str += "db:{} , ".format(self.db)
102104
if self.file:
103-
str += "file:{} , ".format(self.file)
105+
str += "file:{} , ".format(self.file)
104106
if self.table:
105107
str += "table:{} , ".format(self.table)
106108
if self.key_pattern:
@@ -116,78 +118,76 @@ def __str__(self):
116118
if len(self.return_fields) > 0:
117119
str += "return_fields: " + ",".join(self.return_fields) + " "
118120
if self.ns:
119-
str += "namespace: , " + self.ns
121+
str += "namespace: , " + self.ns
120122
if self.match_entire_list:
121123
str += "match_list: True , "
122124
else:
123125
str += "match_list: False , "
124126
return str
125-
127+
128+
126129
class SourceAdapter(ABC):
127130
""" Source Adaptor offers unified interface to Data Sources """
128-
131+
129132
def __init__(self):
130133
pass
131-
134+
132135
@abstractmethod
133136
def connect(self, db, ns):
134137
""" Return True for Success, False for failure """
135138
return False
136-
139+
137140
@abstractmethod
138141
def getKeys(self, db, table, key_pattern):
139142
return []
140-
143+
141144
@abstractmethod
142145
def get(self, db, key):
143146
return {}
144-
147+
145148
@abstractmethod
146149
def hget(self, db, key, field):
147150
return ""
148-
151+
149152
@abstractmethod
150153
def get_separator(self, db):
151154
return ""
152-
155+
156+
153157
class RedisSource(SourceAdapter):
154158
""" Concrete Adaptor Class for connecting to Redis Data Sources """
155-
156-
def __init__(self):
157-
self.conn = None
158-
159+
160+
def __init__(self, conn_pool):
161+
self.conn = None
162+
self.pool = conn_pool
163+
159164
def connect(self, db, ns):
160165
try:
161-
if not SonicDBConfig.isInit():
162-
if multi_asic.is_multi_asic():
163-
SonicDBConfig.load_sonic_global_db_config()
164-
else:
165-
SonicDBConfig.load_sonic_db_config()
166-
self.conn = SonicV2Connector(namespace=ns, use_unix_socket_path=True)
167-
self.conn.connect(db)
166+
self.conn = self.pool.get(db, ns)
168167
except Exception as e:
169168
verbose_print("RedisSource: Connection Failed\n" + str(e))
170169
return False
171170
return True
172-
171+
173172
def get_separator(self, db):
174173
return self.conn.get_db_separator(db)
175-
176-
def getKeys(self, db, table, key_pattern):
174+
175+
def getKeys(self, db, table, key_pattern):
177176
return self.conn.keys(db, table + self.get_separator(db) + key_pattern)
178-
177+
179178
def get(self, db, key):
180179
return self.conn.get_all(db, key)
181-
180+
182181
def hget(self, db, key, field):
183182
return self.conn.get(db, key, field)
184183

184+
185185
class JsonSource(SourceAdapter):
186186
""" Concrete Adaptor Class for connecting to JSON Data Sources """
187-
187+
188188
def __init__(self):
189189
self.json_data = None
190-
190+
191191
def connect(self, db, ns):
192192
try:
193193
with open(db) as f:
@@ -196,67 +196,114 @@ def connect(self, db, ns):
196196
verbose_print("JsonSource: Loading the JSON file failed" + str(e))
197197
return False
198198
return True
199-
199+
200200
def get_separator(self, db):
201201
return SonicDBConfig.getSeparator("CONFIG_DB")
202-
202+
203203
def getKeys(self, db, table, key_pattern):
204204
if table not in self.json_data:
205205
return []
206206
# https://docs.python.org/3.7/library/fnmatch.html
207207
kp = key_pattern.replace("[^", "[!")
208208
kys = fnmatch.filter(self.json_data[table].keys(), kp)
209209
return [table + self.get_separator(db) + ky for ky in kys]
210-
210+
211211
def get(self, db, key):
212212
sep = self.get_separator(db)
213213
table, key = key.split(sep, 1)
214214
return self.json_data.get(table, {}).get(key, {})
215-
215+
216216
def hget(self, db, key, field):
217217
sep = self.get_separator(db)
218218
table, key = key.split(sep, 1)
219219
return self.json_data.get(table, "").get(key, "").get(field, "")
220-
220+
221+
222+
class ConnectionPool:
223+
""" Caches SonicV2Connector objects for effective reuse """
224+
def __init__(self):
225+
self.cache = dict() # Pool of SonicV2Connector objects
226+
227+
def initialize_connector(self, ns):
228+
if not SonicDBConfig.isInit():
229+
if multi_asic.is_multi_asic():
230+
SonicDBConfig.load_sonic_global_db_config()
231+
else:
232+
SonicDBConfig.load_sonic_db_config()
233+
return SonicV2Connector(namespace=ns, use_unix_socket_path=True)
234+
235+
def get(self, db_name, ns, update=False):
236+
""" Returns a SonicV2Connector Object and caches it for further requests """
237+
if ns not in self.cache:
238+
self.cache[ns] = {}
239+
self.cache[ns]["conn"] = self.initialize_connector(ns)
240+
self.cache[ns]["connected_to"] = set()
241+
if update or db_name not in self.cache[ns]["connected_to"]:
242+
self.cache[ns]["conn"].connect(db_name)
243+
self.cache[ns]["connected_to"].add(db_name)
244+
return self.cache[ns]["conn"]
245+
246+
def clear(self, namespace=None):
247+
if not namespace:
248+
self.cache.clear()
249+
elif namespace in self.cache:
250+
del self.cache[namespace]
251+
252+
221253
class MatchEngine:
222-
""" Pass in a MatchRequest, to fetch the Matched dump from the Data sources """
223-
254+
"""
255+
Provide a MatchRequest to fetch the relevant keys/fv's from the data source
256+
Usage Guidelines:
257+
1) Instantiate the class once for the entire execution,
258+
to effectively use the caching of redis connection objects
259+
"""
260+
def __init__(self, pool=None):
261+
if not isinstance(pool, ConnectionPool):
262+
self.conn_pool = ConnectionPool()
263+
else:
264+
self.conn_pool = pool
265+
266+
def clear_cache(self, ns):
267+
self.conn_pool(ns)
268+
224269
def __get_source_adapter(self, req):
225270
src = None
226271
d_src = ""
227272
if req.db:
228273
d_src = req.db
229-
src = RedisSource()
274+
src = RedisSource(self.conn_pool)
230275
else:
231276
d_src = req.file
232277
src = JsonSource()
233278
return d_src, src
234-
279+
235280
def __create_template(self):
236-
return {"error" : "", "keys" : [], "return_values" : {}}
237-
281+
return {"error": "", "keys": [], "return_values": {}}
282+
238283
def __display_error(self, err):
239284
template = self.__create_template()
240285
template['error'] = err
241286
verbose_print("MatchEngine: \n" + template['error'])
242287
return template
243-
288+
244289
def __filter_out_keys(self, src, req, all_matched_keys):
245290
# TODO: Custom Callbacks for Complex Matching Criteria
246291
if not req.field:
247292
return all_matched_keys
248-
293+
249294
filtered_keys = []
250295
for key in all_matched_keys:
251296
f_values = src.hget(req.db, key, req.field)
297+
if not f_values:
298+
continue
252299
if "," in f_values and not req.match_entire_list:
253300
f_value = f_values.split(",")
254301
else:
255302
f_value = [f_values]
256303
if req.value in f_value:
257304
filtered_keys.append(key)
258305
return filtered_keys
259-
306+
260307
def __fill_template(self, src, req, filtered_keys, template):
261308
for key in filtered_keys:
262309
temp = {}
@@ -266,35 +313,34 @@ def __fill_template(self, src, req, filtered_keys, template):
266313
elif len(req.return_fields) > 0:
267314
template["keys"].append(key)
268315
template["return_values"][key] = {}
269-
for field in req.return_fields:
316+
for field in req.return_fields:
270317
template["return_values"][key][field] = src.hget(req.db, key, field)
271318
else:
272319
template["keys"].append(key)
273320
verbose_print("Return Values:" + str(template["return_values"]))
274321
return template
275-
322+
276323
def fetch(self, req):
277324
""" Given a request obj, find its match in the data source provided """
278325
if not isinstance(req, MatchRequest):
279326
return self.__display_error(EXCEP_DICT["INV_REQ"])
280-
327+
281328
verbose_print(str(req))
282-
329+
283330
if not req.key_pattern:
284331
return self.__display_error(EXCEP_DICT["NO_KEY"])
285-
332+
286333
d_src, src = self.__get_source_adapter(req)
287334
if not src.connect(d_src, req.ns):
288335
return self.__display_error(EXCEP_DICT["CONN_ERR"])
289-
336+
290337
template = self.__create_template()
291338
all_matched_keys = src.getKeys(req.db, req.table, req.key_pattern)
292339
if not all_matched_keys:
293340
return self.__display_error(EXCEP_DICT["NO_MATCHES"])
294-
341+
295342
filtered_keys = self.__filter_out_keys(src, req, all_matched_keys)
296343
verbose_print("Filtered Keys:" + str(filtered_keys))
297344
if not filtered_keys:
298345
return self.__display_error(EXCEP_DICT["NO_ENTRIES"])
299-
return self.__fill_template(src, req, filtered_keys, template)
300-
346+
return self.__fill_template(src, req, filtered_keys, template)

0 commit comments

Comments
 (0)