Merge branch 'master' into test_improv

config/config_mgmt.py

@@ -35,7 +35,7 @@ class ConfigMgmt():
  to verify config for the commands which are capable of change in config DB.
  '''
 
- def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True):
+ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True, sonicYangOptions=0):
  '''
  Initialise the class, --read the config, --load in data tree.
 
@@ -53,6 +53,7 @@ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True):
  self.configdbJsonOut = None
  self.source = source
  self.allowTablesWithoutYang = allowTablesWithoutYang
+ self.sonicYangOptions = sonicYangOptions
 
  # logging vars
  self.SYSLOG_IDENTIFIER = "ConfigMgmt"
@@ -67,7 +68,7 @@ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True):
  return
 
  def __init_sonic_yang(self):
- self.sy = sonic_yang.SonicYang(YANG_DIR, debug=self.DEBUG)
+ self.sy = sonic_yang.SonicYang(YANG_DIR, debug=self.DEBUG, sonic_yang_options=self.sonicYangOptions)
  # load yang models
  self.sy.loadYangModel()
  # load jIn from config DB or from config DB json file.
@@ -280,7 +281,7 @@ def get_module_name(yang_module_str):
  """
 
  # Instantiate new context since parse_module_mem() loads the module into context.
- sy = sonic_yang.SonicYang(YANG_DIR)
+ sy = sonic_yang.SonicYang(YANG_DIR, sonic_yang_options=self.sonicYangOptions)
  module = sy.ctx.parse_module_mem(yang_module_str, ly.LYS_IN_YANG)
  return module.name()
 

dump/main.py

@@ -7,6 +7,7 @@
 from sonic_py_common import multi_asic
 from utilities_common.constants import DEFAULT_NAMESPACE
 from dump.match_infra import RedisSource, JsonSource, MatchEngine
+from swsscommon.swsscommon import ConfigDBConnector
 from dump import plugins
 
 # Autocompletion Helper
@@ -81,7 +82,10 @@ def state(ctx, module, identifier, db, table, key_map, verbose, namespace):
  params['namespace'] = namespace
  for arg in ids:
  params[plugins.dump_modules[module].ARG_NAME] = arg
- collected_info[arg] = obj.execute(params)
+ try:
+ collected_info[arg] = obj.execute(params)
+ except ValueError as err:
+ click.fail(f"Failed to execute plugin: {err}")
 
  if len(db) > 0:
  collected_info = filter_out_dbs(db, collected_info)
@@ -174,9 +178,13 @@ def populate_fv(info, module, namespace, conn_pool):
 
 
 def get_dict_str(key_obj):
+ conn = ConfigDBConnector()
  table = []
- for pair in key_obj.items():
- table.append(list(pair))
+ key_obj = conn.raw_to_typed(key_obj)
+ for field, value in key_obj.items():
+ if isinstance(value, list):
+ value = "\n".join(value)
+ table.append((field, value))
  return tabulate(table, headers=["field", "value"], tablefmt="psql")
 
 

dump/match_helper.py

@@ -44,7 +44,7 @@ def fetch_vlan_oid(match_engine, vlan_name, ns):
  vlan_num = int(vlan_name[4:])
 
  # Find the table named "ASIC_STATE:SAI_OBJECT_TYPE_VLAN:*" in which SAI_VLAN_ATTR_VLAN_ID = vlan_num
- req = MatchRequest(db="ASIC_DB", table="ASIC_STATE:SAI_OBJECT_TYPE_VLAN", key_pattern="*", field="SAI_VLAN_ATTR_VLAN_ID", 
+ req = MatchRequest(db="ASIC_DB", table="ASIC_STATE:SAI_OBJECT_TYPE_VLAN", key_pattern="*", field="SAI_VLAN_ATTR_VLAN_ID",
  value=str(vlan_num), ns=ns)
  ret = match_engine.fetch(req)
  vlan_oid = ""
@@ -100,3 +100,15 @@ def fetch_lag_oid(match_engine, lag_name, ns):
  lag_oids matched {}".format(lag_name, lag_type_oids), lag_type_oids[-1])
  lag_type_oid = lag_type_oids[-1]
  return lag_type_oid
+
+# ACL helpers
+
+def fetch_acl_counter_oid(match_engine, acl_table_name, acl_rule_name, ns):
+ """
+ Fetch ACL counter OID from COUNTERS DB for a particular rule
+ """
+ counters_db = match_engine.get_redis_source_adapter()
+ counters_db.connect("COUNTERS_DB", ns)
+ counters = counters_db.hgetall("COUNTERS_DB", "ACL_COUNTER_RULE_MAP")
+ counter_oid = counters.get(f"{acl_table_name}{counters_db.get_separator('COUNTERS_DB')}{acl_rule_name}")
+ return counter_oid

dump/match_infra.py

@@ -154,6 +154,10 @@ def hget(self, db, key, field):
  def get_separator(self, db):
  return ""
 
+ @abstractmethod
+ def hgetall(self, db, key):
+ raise NotImplementedError
+
 
 class RedisSource(SourceAdapter):
  """ Concrete Adaptor Class for connecting to Redis Data Sources """
@@ -182,6 +186,9 @@ def get(self, db, key):
  def hget(self, db, key, field):
  return self.conn.get(db, key, field)
 
+ def hgetall(self, db, key):
+ return self.conn.get_all(db, key)
+
 
 class JsonSource(SourceAdapter):
  """ Concrete Adaptor Class for connecting to JSON Data Sources """
@@ -219,6 +226,11 @@ def hget(self, db, key, field):
  table, key = key.split(sep, 1)
  return self.json_data.get(table, "").get(key, "").get(field, "")
 
+ def hgetall(self, db, key):
+ sep = self.get_separator(db)
+ table, key = key.split(sep, 1)
+ return self.json_data.get(table, {}).get(key)
+
 
 class ConnectionPool:
  """ Caches SonicV2Connector objects for effective reuse """
@@ -271,15 +283,21 @@ def __init__(self, pool=None):
  def clear_cache(self, ns):
  self.conn_pool(ns)
 
+ def get_redis_source_adapter(self):
+ return RedisSource(self.conn_pool)
+
+ def get_json_source_adapter(self):
+ return JsonSource()
+
  def __get_source_adapter(self, req):
  src = None
  d_src = ""
  if req.db:
  d_src = req.db
- src = RedisSource(self.conn_pool)
+ src = self.get_redis_source_adapter()
  else:
  d_src = req.file
- src = JsonSource()
+ src = self.get_json_source_adapter()
  return d_src, src
 
  def __create_template(self):

dump/plugins/acl_rule.py

@@ -0,0 +1,84 @@
+from dump.helper import create_template_dict
+from dump.match_infra import MatchRequest
+from swsscommon.swsscommon import SonicDBConfig
+
+from dump.match_helper import fetch_acl_counter_oid
+from .executor import Executor
+
+
+CFG_DB_SEPARATOR = SonicDBConfig.getSeparator("CONFIG_DB")
+ASIC_DB_SEPARATOR = SonicDBConfig.getSeparator("ASIC_DB")
+
+
+class Acl_Rule(Executor):
+ """
+ Debug Dump Plugin for ACL Rule Module
+ """
+ ARG_NAME = "acl_rule_name"
+
+ def __init__(self, match_engine=None):
+ super().__init__(match_engine)
+
+ def get_all_args(self, ns=""):
+ req = MatchRequest(db="CONFIG_DB", table="ACL_RULE", key_pattern="*", ns=ns)
+ ret = self.match_engine.fetch(req)
+ acl_rules = ret["keys"]
+ return [key.split(CFG_DB_SEPARATOR, 1)[-1] for key in acl_rules]
+
+ def execute(self, params):
+ self.ret_temp = create_template_dict(dbs=["CONFIG_DB", "ASIC_DB"])
+
+ try:
+ acl_table_name, acl_rule_name = params[self.ARG_NAME].split(CFG_DB_SEPARATOR, 1)
+ except ValueError:
+ raise ValueError(f"Invalid rule name passed {params[self.ARG_NAME]}")
+
+ self.ns = params["namespace"]
+ self.init_acl_rule_config_info(acl_table_name, acl_rule_name)
+ self.init_acl_rule_asic_info(acl_table_name, acl_rule_name)
+ return self.ret_temp
+
+ def init_acl_rule_config_info(self, acl_table_name, acl_rule_name):
+ req = MatchRequest(db="CONFIG_DB", table="ACL_RULE",
+ key_pattern=CFG_DB_SEPARATOR.join([acl_table_name, acl_rule_name]), ns=self.ns)
+ ret = self.match_engine.fetch(req)
+ self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"])
+
+ def init_acl_rule_asic_info(self, acl_table_name, acl_rule_name):
+ counter_oid = fetch_acl_counter_oid(self.match_engine, acl_table_name, acl_rule_name, self.ns)
+ if not counter_oid:
+ return
+
+ req = MatchRequest(db="ASIC_DB", table=ASIC_DB_SEPARATOR.join(["ASIC_STATE", "SAI_OBJECT_TYPE_ACL_COUNTER"]),
+ key_pattern=counter_oid, return_fields=["SAI_ACL_COUNTER_ATTR_TABLE_ID"], ns=self.ns)
+ ret = self.match_engine.fetch(req)
+ self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"])
+
+ return_values = ret["return_values"]
+ counter_object = return_values.get(ASIC_DB_SEPARATOR.join(["ASIC_STATE", "SAI_OBJECT_TYPE_ACL_COUNTER", counter_oid]), {})
+ table_oid = counter_object.get("SAI_ACL_COUNTER_ATTR_TABLE_ID")
+ if not table_oid:
+ raise Exception("Invalid counter object without table OID in ASIC_DB")
+
+ req = MatchRequest(db="ASIC_DB", table=ASIC_DB_SEPARATOR.join(["ASIC_STATE", "SAI_OBJECT_TYPE_ACL_ENTRY"]), key_pattern="*",
+ field="SAI_ACL_ENTRY_ATTR_TABLE_ID", value=table_oid,
+ return_fields=["SAI_ACL_ENTRY_ATTR_FIELD_ACL_RANGE_TYPE"], ns=self.ns)
+ ret = self.match_engine.fetch(req)
+ self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"])
+
+ range_oids = set()
+ for _, entry in ret["return_values"].items():
+ range_attr_value = entry.get("SAI_ACL_ENTRY_ATTR_FIELD_ACL_RANGE_TYPE")
+ if not range_attr_value:
+ continue
+ ranges_attr_value = range_attr_value.split(ASIC_DB_SEPARATOR, 1)
+ if len(range_attr_value) < 2:
+ raise Exception("Invalid SAI_ACL_ENTRY_ATTR_FIELD_ACL_RANGE_TYPE field format")
+ for oid in ranges_attr_value[1].split(','):
+ range_oids.add(oid)
+
+ for range_oid in range_oids:
+ req = MatchRequest(db="ASIC_DB", table=ASIC_DB_SEPARATOR.join(["ASIC_STATE", "SAI_OBJECT_TYPE_ACL_RANGE"]),
+ key_pattern=range_oid, ns=self.ns)
+ ret = self.match_engine.fetch(req)
+ self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"])

dump/plugins/acl_table.py

@@ -0,0 +1,91 @@
+from dump.helper import create_template_dict
+from dump.match_infra import MatchRequest
+from swsscommon.swsscommon import SonicDBConfig
+
+from dump.match_helper import fetch_acl_counter_oid
+from .executor import Executor
+
+
+CFG_DB_SEPARATOR = SonicDBConfig.getSeparator("CONFIG_DB")
+ASIC_DB_SEPARATOR = SonicDBConfig.getSeparator("ASIC_DB")
+
+
+class Acl_Table(Executor):
+ """
+ Debug Dump Plugin for ACL Table Module
+ """
+ ARG_NAME = "acl_table_name"
+
+ def __init__(self, match_engine=None):
+ super().__init__(match_engine)
+
+ def get_all_args(self, ns=""):
+ req = MatchRequest(db="CONFIG_DB", table="ACL_TABLE", key_pattern="*", ns=ns)
+ ret = self.match_engine.fetch(req)
+ acl_tables = ret["keys"]
+ return [key.split(CFG_DB_SEPARATOR)[-1] for key in acl_tables]
+
+ def execute(self, params):
+ self.ret_temp = create_template_dict(dbs=["CONFIG_DB", "ASIC_DB"])
+ acl_table_name = params[self.ARG_NAME]
+ self.ns = params["namespace"]
+ self.init_acl_table_config_info(acl_table_name)
+ self.init_acl_table_asic_info(acl_table_name)
+ return self.ret_temp
+
+ def init_acl_table_config_info(self, acl_table_name):
+ req = MatchRequest(db="CONFIG_DB", table="ACL_TABLE", key_pattern=acl_table_name, return_fields=["type"], ns=self.ns)
+ ret = self.match_engine.fetch(req)
+ self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"])
+
+ # Find corresponding ACL table type in CONFIG DB
+ return_values = ret["return_values"]
+ acl_table_type_name = return_values.get(CFG_DB_SEPARATOR.join(["ACL_TABLE", acl_table_name]), {}).get("type")
+ req = MatchRequest(db="CONFIG_DB", table="ACL_TABLE_TYPE", key_pattern=acl_table_type_name, ns=self.ns)
+ ret = self.match_engine.fetch(req)
+ # If not found don't add it to the table, it might be a default table type
+ if ret["keys"]:
+ self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"])
+
+ def init_acl_table_asic_info(self, acl_table_name):
+ req = MatchRequest(db="CONFIG_DB", table="ACL_RULE", key_pattern=CFG_DB_SEPARATOR.join([acl_table_name, "*"]), ns=self.ns)
+ ret = self.match_engine.fetch(req)
+ acl_rules = ret["keys"]
+ if not acl_rules:
+ return
+ acl_rule_name = acl_rules[0].split(CFG_DB_SEPARATOR)[-1]
+
+ counter_oid = fetch_acl_counter_oid(self.match_engine, acl_table_name, acl_rule_name, self.ns)
+ if not counter_oid:
+ return
+
+ req = MatchRequest(db="ASIC_DB", table=ASIC_DB_SEPARATOR.join(["ASIC_STATE", "SAI_OBJECT_TYPE_ACL_COUNTER"]),
+ key_pattern=counter_oid, return_fields=["SAI_ACL_COUNTER_ATTR_TABLE_ID"], ns=self.ns)
+ ret = self.match_engine.fetch(req)
+
+ return_values = ret["return_values"]
+ counter_object = return_values.get(ASIC_DB_SEPARATOR.join(["ASIC_STATE", "SAI_OBJECT_TYPE_ACL_COUNTER", counter_oid]), {})
+ table_oid = counter_object.get("SAI_ACL_COUNTER_ATTR_TABLE_ID")
+ if not table_oid:
+ raise Exception("Invalid counter object without table OID in ASIC_DB")
+
+ req = MatchRequest(db="ASIC_DB", table=ASIC_DB_SEPARATOR.join(["ASIC_STATE", "SAI_OBJECT_TYPE_ACL_TABLE"]),
+ key_pattern=table_oid, ns=self.ns)
+ ret = self.match_engine.fetch(req)
+ self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"])
+
+ req = MatchRequest(db="ASIC_DB", table=ASIC_DB_SEPARATOR.join(["ASIC_STATE", "SAI_OBJECT_TYPE_ACL_TABLE_GROUP_MEMBER"]),
+ key_pattern="*", field="SAI_ACL_TABLE_GROUP_MEMBER_ATTR_ACL_TABLE_ID",
+ value=table_oid, return_fields=["SAI_ACL_TABLE_GROUP_MEMBER_ATTR_ACL_TABLE_GROUP_ID"], ns=self.ns)
+ ret = self.match_engine.fetch(req)
+ self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"])
+
+ group_oids = set()
+ for _, entry in ret["return_values"].items():
+ group_oids.add(entry.get("SAI_ACL_TABLE_GROUP_MEMBER_ATTR_ACL_TABLE_GROUP_ID"))
+
+ for group_oid in group_oids:
+ req = MatchRequest(db="ASIC_DB", table=ASIC_DB_SEPARATOR.join(["ASIC_STATE", "SAI_OBJECT_TYPE_ACL_TABLE_GROUP"]),
+ key_pattern=group_oid, ns=self.ns)
+ ret = self.match_engine.fetch(req)
+ self.add_to_ret_template(req.table, req.db, ret["keys"], ret["error"])

generic_config_updater/gu_common.py

@@ -750,6 +750,19 @@ def _get_path_tokens_from_leaf(self, model, token_index, xpath_tokens, config):
  # leaf_list_name = match.group(1)
  leaf_list_value = match.group(1)
  list_config = config[leaf_list_name]
+ # Workaround for those fields who is defined as leaf-list in YANG model but have string value in config DB
+ # No need to lookup the item index in ConfigDb since the list is represented as a string, return path to string immediately
+ # Example:
+ # xpath: /sonic-buffer-port-egress-profile-list:sonic-buffer-port-egress-profile-list/BUFFER_PORT_EGRESS_PROFILE_LIST/BUFFER_PORT_EGRESS_PROFILE_LIST_LIST[port='Ethernet9']/profile_list[.='egress_lossy_profile']
+ # path: /BUFFER_PORT_EGRESS_PROFILE_LIST/Ethernet9/profile_list
+ if isinstance(list_config, str):
+ return [leaf_list_name]
+
+ if not isinstance(list_config, list):
+ raise ValueError(f"list_config is expected to be of type list or string. Found {type(list_config)}.\n " + \
+ f"model: {model}\n token_index: {token_index}\n " + \
+ f"xpath_tokens: {xpath_tokens}\n config: {config}")
+
  list_idx = list_config.index(leaf_list_value)
  return [leaf_list_name, list_idx]
 

scripts/portconfig

@@ -129,6 +129,9 @@ class portconfig(object):
  self.db.mod_entry(PORT_TABLE_NAME, port, {PORT_FEC_CONFIG_FIELD_NAME: fec})
 
  def set_mtu(self, port, mtu):
+ port_tables = self.db.get_table(PORT_TABLE_NAME)
+ if port not in port_tables:
+ raise Exception("Invalid port %s" % (port))
  if self.verbose:
  print("Setting mtu %s on port %s" % (mtu, port))
  self.db.mod_entry(PORT_TABLE_NAME, port, {PORT_MTU_CONFIG_FIELD_NAME: mtu})